[
  {
    "path": ".githooks/pre-commit.d/00-gofmt",
    "content": "#!/bin/bash\n# Copyright 2012 The Go Authors. All rights reserved.\n# Use of this source code is governed by a BSD-style\n# license that can be found in the LICENSE file.\n\n# git gofmt pre-commit hook\n#\n# To use, store as .git/hooks/pre-commit inside your repository and make sure\n# it has execute permissions.\n#\n# This script does not handle file names that contain spaces.\n\nif [ -z \"$(command -v gofmt)\" ]; then\n    echo >&2 \"WARNING: Cannot check/enforce Go code formatting: can't find gofmt.\"\n    echo >&2 \"WARNING: Please consider installing gofmt.\"\n    exit 0\nfi\n\ngofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '\\.go$')\n[ -z \"$gofiles\" ] && exit 0\n\n# shellcheck disable=SC2086\nunformatted=$(gofmt -l $gofiles)\n[ -z \"$unformatted\" ] && exit 0\n\n# Some files are not gofmt'd. Print message and fail.\n\necho >&2 \"Go files must be formatted with gofmt. Please run:\"\nfor fn in $unformatted; do\n\techo >&2 \"  gofmt -w $PWD/$fn\"\ndone\n\nexit 1\n"
  },
  {
    "path": ".githooks/pre-commit.d/10-shellcheck",
    "content": "#!/bin/bash\n\n# git shellcheck pre-commit hook\n#\n# To use, store as .git/hooks/pre-commit/shellcheck inside your repository\n# and make sure it has execute permissions.\n#\n# This script does not handle file names that contain spaces.\n#\n\nif [ -z \"$(command -v shellcheck)\" ]; then\n    echo >&2 \"WARNING: Cannot shellcheck scripts: can't find shellcheck.\"\n    echo >&2 \"WARNING: Please consider installing shellcheck.\"\n    exit 0\nfi\n\nshfiles=$(git diff --cached --name-only --diff-filter=ACM -- '*.sh' '*.bash')\n#echo >&2 \"[$0: shfiles: $shfiles]\"\n\nfor f in $(git diff --cached --name-only --diff-filter=ACM); do\n    if grep -EHn '^#!/bin/.*sh *' \"$f\" | grep -q ':1:#!'; then\n        shfiles=\"$shfiles $f\"\n    fi\ndone\nshfiles=\"$(echo \"$shfiles\" | tr -s '\\t ' '\\n' | sort | uniq)\"\n#echo >&2 \"[$0: shfiles: $shfiles]\"\n\n# shellcheck disable=SC2086\nif  [ -z \"$shfiles\" ] || shellcheck $shfiles; then\n    exit 0\nfi\n\n# Some files do not pass ShellCheck. Print message and fail.\necho >&2 \"shell scripts must pass ShellCheck. Please fix them.\"\nexit 1\n"
  },
  {
    "path": ".githooks/pre-commit.d/20-go-version",
    "content": "#!/bin/bash\n\nWORKFLOWS=\".github/workflows/verify.yml\"\n\nif git diff --cached go.mod | grep -q '^+go '; then\n    gomod=$(go list -m -f '{{.GoVersion}}')\nelse\n    exit 0\nfi\n\nstatus=0\nfor wf in $WORKFLOWS; do\n    workflow=$(grep 'go-version:' $wf | sed 's/^.*: //')\n    if [ \"$gomod\" != \"$workflow\" ]; then\n        echo >&2 \"ERROR: inconsistent golang versions, $gomod in go.mod but $workflow in $wf...\"\n        status=1\n    fi\ndone\n\nif [ \"$status\" != 0 ]; then\n    echo >&2 \"Please consider fixing these inconsistencies before committing...\"\nfi\n\nexit $status\n"
  },
  {
    "path": ".githooks/run-hooks",
    "content": "#!/bin/bash\n\ntype=${0##*/}\nhdir=$0.d\norig=${0%/*}/../.git/hooks/$type\n\nexec 1>&2\n\nfor hlet in \"$hdir\"/???*; do\n    case $hlet in\n        *~|*.swp)\n            continue\n            ;;\n        [0-9][0-9]-*)\n            ;;\n    esac\n    if [ ! -x \"$hlet\" ]; then\n        continue\n    fi\n\n    echo \"<checking $type/${hlet##*/}>\"\n    $hlet\n    r=$?\n    if [ $r != 0 ]; then\n        exit $r\n    fi\ndone\n\nif [ -x \"$orig\" ]; then\n    echo \"<checking .git/hooks/$type>\"\n    $orig\n    exit $?\nfi\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/bug_report.md",
    "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\n<!-- A clear and concise description of what the bug is\n-->\n\n**Expected behavior**\n<!-- A clear and concise description of what you expected to happen\n-->\n\n**To Reproduce**\n<!-- Steps to reproduce the behavior\n-->\n\n**Environment**\n<!-- OS, kernel, container runtime, Kubernetes version\n-->\n\n**Additional context**\n<!-- Add any other context about the problem here\n-->\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature_request.md",
    "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the solution you'd like**\n<!--\nA clear and concise description of what you want to happen\n-->\n\n**Why this is needed**\n<!--\nA clear and concise description of the use case and/or justification for the feature\n-->\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/new-release.md",
    "content": "---\nname: New release\nabout: Propose a new release\ntitle: Release v0.0.0\nlabels: ''\nassignees: ''\n\n---\n\n## Release Process\n<!--\nIf making adjustments to the checklist please also file a PR against this issue\ntemplate (.github/ISSUE_TEMPLATE/new-release.md) to incorporate the changes for\nfuture releases.\n-->\n- [ ] In the issue description, add a changelog section, describing changes since the last release.\n- Local release preparations\n  - [ ] Perform mandatory internal release checks and preparations.\n  - [ ] Run `make release-tests` to run an extended set of tests prior to a release.\n  - [ ] Sync/tidy up dependencies.\n    - [ ] Run `go mod tidy`.\n    - [ ] Run `git commit -m 'go.mod,go.sum: update dependencies.' go.{mod,sum}`, if necessary.\n  - [ ] Run `git tag -a -m \"CRI Resource Manager release $VERSION\" $VERSION`.\n- Publishing\n  - [ ] Push the tag with `git push $VERSION`. This will automatically build container images and release assets and upload the release assets to a new draft release,\n  - [ ] Check that release assets were created for the tag\n    - Container images are published\n      - https://hub.docker.com/r/intel/cri-resmgr-agent/tags\n      - https://hub.docker.com/r/intel/cri-resmgr-webhook/tags\n    - Release assets are uploaded to the draft release\n      - RPM packages\n      - DEB package\n      - Binary tarball\n      - Source+dependencies tarball (vendored dist)\n  - [ ] Update the automatically created draft release corresponding to the tag.\n    - [ ] Write the change log to the release.\n    - [ ] Mark the release as a non-production pre-release if necessary.\n    - [ ] Save as draft.\n  - [ ] Get the change log OK'd by other maintainers.\n  - [ ] Publish the draft as a release.\n  - [ ] Add a link to the tagged release in this issue.\n- [ ] Close this issue.\n\n\n## Changelog\n<!--\nCapture changes since the last release here.\nFor major releases have separate sections for major changes and a more detailed changelog.\nFor minor releases list the most important bug fixes and other improvements.\n-->\n### Major changes\n\n### Detailed changelog\n"
  },
  {
    "path": ".github/workflows/common-build-docs.yaml",
    "content": "name: Build documentation\non:\n  workflow_call:\n    inputs:\n      publish:\n        default: false\n        required: false\n        type: boolean\n\npermissions:\n  contents: read\n\njobs:\n  update-gh-pages:\n    runs-on: ubuntu-22.04\n    permissions:\n      contents: write\n    steps:\n    - uses: actions/checkout@v4\n\n    - name: Fetch gh-pages\n      run: git fetch --no-tags --prune --depth=1 origin refs/heads/gh-pages:refs/heads/gh-pages\n\n    - name: Install build dependencies\n      run: |\n        pip3 install --user -r docs/requirements.txt\n        echo \"`python3 -m site --user-base`/bin\" >> $GITHUB_PATH\n\n    - name: Add docs from this revision to gh-pages\n      run: |\n        git config user.name \"Github\"\n        git config user.email \"no-reply@github.com\"\n        ./scripts/build/update-gh-pages.sh\n\n    - name: Publish gh-pages\n      if: ${{ inputs.publish }}\n      shell: bash\n      env:\n        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n      run: |\n        git push https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git gh-pages\n"
  },
  {
    "path": ".github/workflows/common-build-images.yaml",
    "content": "name: Build container images\n\non:\n  workflow_call:\n    inputs:\n      image-tag:\n        default: ${{ github.ref_name }}\n        required: false\n        type: string\n      publish:\n        default: false\n        required: false\n        type: boolean\n      github-environment:\n        default: null\n        required: false\n        type: string\n\npermissions:\n  contents: read\n\njobs:\n  build-images:\n    name: Build and publish container images\n    runs-on: ubuntu-22.04\n    environment: ${{ inputs.github-environment }}\n    env:\n      IMAGE_REPO: intel\n      IMAGE_VERSION: ${{ inputs.image-tag }}\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v4\n\n    - name: Build images\n      run: \"make images IMAGE_VERSION=${IMAGE_VERSION}  Q=\"\n\n    - name: Login to Docker Hub\n      if: ${{ inputs.publish }}\n      uses: docker/login-action@v3\n      with:\n        username: ${{ secrets.DOCKERHUB_USERNAME }}\n        password: ${{ secrets.DOCKERHUB_TOKEN }}\n\n    - name: Push images\n      if: ${{ inputs.publish }}\n      run: \"make images-push IMAGE_VERSION=${IMAGE_VERSION} Q=\"\n\n"
  },
  {
    "path": ".github/workflows/common-codeql.yaml",
    "content": "name: CodeQL scanning\non:\n  workflow_call:\n    inputs:\n      export-report:\n        default: false\n        required: false\n        type: boolean\n\npermissions:\n  contents: read\n\njobs:\n  codeql-scan:\n    runs-on: ubuntu-22.04\n    permissions:\n      security-events: write\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v4\n\n    - name: Set up Go\n      uses: actions/setup-go@v5\n      with:\n        go-version-file: go.mod\n\n    - name: Initialize CodeQL\n      uses: github/codeql-action/init@v3\n      with:\n        languages: go\n\n    - name: Perform CodeQL Analysis\n      uses: github/codeql-action/analyze@v3\n\n    - name: Generate CodeQL Security Report\n      if: ${{ inputs.export-report }}\n      uses: rsdmike/github-security-report-action@v3.0.4\n      with:\n        template: report\n        token: ${{ secrets.GITHUB_TOKEN }}\n\n    - name: Upload PDF report as an artifact\n      if: ${{ inputs.export-report }}\n      uses: actions/upload-artifact@v4\n      with:\n        name: codeql-report\n        path: report.pdf\n"
  },
  {
    "path": ".github/workflows/common-trivy.yaml",
    "content": "name: Trivy scanning\non:\n  workflow_call:\n    inputs:\n      upload-to-github-security-tab:\n        default: false\n        required: false\n        type: boolean\n      export-csv:\n        default: false\n        required: false\n        type: boolean\n\npermissions:\n  contents: read\n\njobs:\n  trivy-scan-licenses:\n    runs-on: ubuntu-22.04\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v4\n\n    - name: Run Trivy in fs mode\n      uses: aquasecurity/trivy-action@master\n      with:\n        scan-type: fs\n        scan-ref: .\n        exit-code: 1\n        scanners: license\n        severity: \"UNKNOWN,MEDIUM,HIGH,CRITICAL\"\n\n  trivy-scan-vulns:\n    runs-on: ubuntu-22.04\n    permissions:\n      security-events: write\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v4\n\n    - name: Run Trivy in fs mode\n      continue-on-error: true\n      uses: aquasecurity/trivy-action@master\n      with:\n        scan-type: fs\n        scan-ref: .\n        exit-code: 1\n        list-all-pkgs: true\n        format: json\n        output: trivy-report.json\n\n    - name: Show report in human-readable format\n      uses: aquasecurity/trivy-action@master\n      with:\n        scan-type: convert\n        vuln-type: ''\n        severity: ''\n        image-ref: trivy-report.json\n        format: table\n\n    - name: Convert report to sarif\n      if: ${{ inputs.upload-to-github-security-tab }}\n      uses: aquasecurity/trivy-action@master\n      with:\n        scan-type: convert\n        vuln-type: ''\n        severity: ''\n        image-ref: trivy-report.json\n        format: sarif\n        output: trivy-report.sarif\n\n    - name: Upload sarif report to GitHub Security tab\n      if: ${{ inputs.upload-to-github-security-tab }}\n      uses: github/codeql-action/upload-sarif@v3\n      with:\n       sarif_file: trivy-report.sarif\n\n    - name: Convert report to csv\n      if: ${{ inputs.export-csv }}\n      uses: aquasecurity/trivy-action@master\n      with:\n        scan-type: convert\n        vuln-type: ''\n        severity: ''\n        image-ref: trivy-report.json\n        format: template\n        template: \"@.github/workflows/trivy-csv.tpl\"\n        output: trivy-report.csv\n\n    - name: Upload CSV report as an artifact\n      if: ${{ inputs.export-csv }}\n      uses: actions/upload-artifact@v4\n      with:\n        name: trivy-report\n        path: trivy-report.csv\n"
  },
  {
    "path": ".github/workflows/common-verify-code.yaml",
    "content": "name: Verify code\n\non:\n  - workflow_call\n\npermissions:\n  contents: read\n\njobs:\n  build-and-test:\n    runs-on: ubuntu-22.04\n    steps:\n    - name: Check out code\n      uses: actions/checkout@v4\n\n    - name: Set up Go\n      uses: actions/setup-go@v5\n      with:\n        go-version-file: go.mod\n      id: go\n\n    - name: Install golangci-lint\n      run: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.7\n\n    - name: Gofmt\n      run: make format\n\n    - name: Build\n      run: make\n\n    - name: Test\n      run: make test\n\n    - name: Golangci-lint\n      run: |\n        export PATH=$PATH:$(go env GOPATH)/bin\n        make golangci-lint\n\n    - name: Codecov report\n      run: bash <(curl -s https://codecov.io/bash)\n\n  trivy-scan:\n    uses: \"./.github/workflows/common-trivy.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n    with:\n      upload-to-github-security-tab: true\n\n  codeql-scan:\n    uses: \"./.github/workflows/common-codeql.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n"
  },
  {
    "path": ".github/workflows/publish-devel-images.yaml",
    "content": "name: Build and publish devel container images\n\non:\n  push:\n    branches: [\"master\"]\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.workflow }}-${{ github.ref_name }}\n  cancel-in-progress: true\n\njobs:\n  trivy-scan:\n    uses: \"./.github/workflows/common-trivy.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n\n  publish-images:\n    uses: \"./.github/workflows/common-build-images.yaml\"\n    needs: [trivy-scan]\n    secrets: inherit\n    with:\n      publish: true\n      image-tag: \"devel\"\n      github-environment: \"staging\"\n\n"
  },
  {
    "path": ".github/workflows/publish-docs.yml",
    "content": "name: Publish documentation\n\non:\n  push:\n    branches:\n        - master\n        - release-*\n    # Path filters are ignored for tags\n    paths:\n      - \"docs/**\"\n      - \"Makefile\"\n    tags:\n        - v*\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.workflow }}\n  cancel-in-progress: false\n\njobs:\n  update-gh-pages:\n    uses: \"./.github/workflows/common-build-docs.yaml\"\n    permissions:\n      contents: write\n    with:\n      publish: true\n"
  },
  {
    "path": ".github/workflows/release.yaml",
    "content": "name: Build and publish release artifacts\n\non:\n  push:\n    tags: [ 'v*' ]\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.workflow }}-${{ github.ref_name }}\n  cancel-in-progress: true\n\njobs:\n  trivy-scan:\n    uses: \"./.github/workflows/common-trivy.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n    with:\n      export-csv: true\n\n  codeql:\n    uses: \"./.github/workflows/common-codeql.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n    with:\n      export-report: true\n\n  publish-images:\n    uses: \"./.github/workflows/common-build-images.yaml\"\n    needs: [trivy-scan]\n    secrets: inherit\n    with:\n      publish: true\n      image-tag: ${{ github.ref_name }}\n      github-environment: \"release\"\n\n  build-packages:\n    needs: [trivy-scan]\n    permissions:\n      contents: write\n    runs-on: ubuntu-22.04\n    steps:\n    - name: Checkout\n      uses: actions/checkout@v4\n\n    - name: Build packages\n      run: \"make cross-packages  Q=\"\n\n    - name: Build vendored dist tarball\n      run: \"make vendored-dist  Q=\"\n\n    - name: Upload release assets\n      uses: softprops/action-gh-release@v1\n      with:\n        name: ${{ github.ref_name }}\n        draft: true\n        append_body: true\n        files: |\n          packages/release-assets/*\n          vendored-cri-resource-manager-*.tar.gz\n"
  },
  {
    "path": ".github/workflows/trivy-csv.tpl",
    "content": "{{ range . }}\nTrivy Vulnerability Scan Results ({{- .Target -}})\nVulnerabilityID,Severity,CVSS Score,Title,Library,Vulnerable Version,Fixed Version,Information URL,Triage Information\n{{ range .Vulnerabilities }}\n    {{- .VulnerabilityID }},\n    {{- .Severity }},\n    {{- range $key, $value := .CVSS }}\n        {{- if (eq $key \"nvd\") }}\n            {{- .V3Score -}}\n        {{- end }}\n    {{- end }},\n    {{- quote .Title }},\n    {{- quote .PkgName }},\n    {{- quote .InstalledVersion }},\n    {{- quote .FixedVersion }},\n    {{- .PrimaryURL }}\n{{ else -}}\n    No vulnerabilities found at this time.\n{{ end }}\nTrivy Dependency Scan Results ({{ .Target }})\nID,Name,Version,Notes\n{{ range .Packages -}}\n    {{- quote .ID }},\n    {{- quote .Name }},\n    {{- quote .Version }}\n{{ else -}}\n    No dependencies found at this time.\n{{ end }}\n{{ end }}\n"
  },
  {
    "path": ".github/workflows/verify-periodic.yaml",
    "content": "name: Verify branches periodic\n\non:\n  schedule:\n    - cron: '30 2 * * 0'\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.workflow }}-${{ github.ref }}\n  cancel-in-progress: true\n\njobs:\n  verify-code:\n    uses: \"./.github/workflows/common-verify-code.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n"
  },
  {
    "path": ".github/workflows/verify-pr-code.yaml",
    "content": "name: Verify code\n\non:\n  pull_request\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}\n  cancel-in-progress: true\n\njobs:\n  verify:\n    uses: \"./.github/workflows/common-verify-code.yaml\"\n    permissions:\n      contents: read\n      security-events: write\n"
  },
  {
    "path": ".github/workflows/verify-pr-docs.yaml",
    "content": "name: Verify documentation\n\non:\n  pull_request:\n    paths:\n      - \"docs/**\"\n      - \"Makefile\"\n\npermissions:\n  contents: read\n\nconcurrency:\n  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}\n  cancel-in-progress: true\n\njobs:\n  verify-docs:\n    uses: \"./.github/workflows/common-build-docs.yaml\"\n    permissions:\n      contents: write\n      security-events: write\n"
  },
  {
    "path": ".gitignore",
    "content": "*~\n*.swp\n\n*_gendata.go\n/bin\ncoverage.html\ncoverage.txt\n.git-hooks.redirected\n\n*.tar\n*.tar.*\n*.spec\n.static.*\n/debian\n/packages\n\n/_build\n/_work\n*.stamp\ntest/e2e/**/output\n"
  },
  {
    "path": "CODEOWNERS",
    "content": "* @kad @klihub @marquiz @mythi @askervin @jukkar @fmuyassarov\n"
  },
  {
    "path": "Jenkinsfile",
    "content": "pipeline {\n    agent {\n        label \"cri-rm\"\n    }\n\n    environment {\n        IMAGE_REPO = \"cloud-native-image-registry.westus.cloudapp.azure.com\"\n    }\n\n    stages {\n        stage('Build and push images') {\n            steps {\n                script {\n                    withDockerRegistry([credentialsId: \"${env.DOCKER_REGISTRY}\", url: \"https://${env.IMAGE_REPO}\"]) {\n                        if (env.BRANCH_NAME == 'master') {\n                            sh \"make images-push IMAGE_REPO=${env.IMAGE_REPO} IMAGE_VERSION=devel Q=\"\n                        } else {\n                            sh \"make images-push IMAGE_REPO=${env.IMAGE_REPO} Q=\"\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "LICENSE",
    "content": "\n                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "Makefile",
    "content": "# We use bashisms in this Makefile.\nSHELL := /bin/bash\n\n# Go compiler/toolchain and extra related binaries we ues/need.\nGO_PARALLEL :=\nGO_CMD      := go\nGO_BUILD    := $(GO_CMD) build $(GO_PARALLEL)\nGO_GEN      := $(GO_CMD) generate -x\nGO_INSTALL  := $(GO_CMD) install\nGO_FMT      := gofmt\nGO_CYCLO    := gocyclo\nGO_LINT     := golint\nGO_CILINT   := golangci-lint\nGO_VERSION  ?= 1.24.1\nGOLICENSES_VERSION  ?= v1.5.0\n\n# TEST_TAGS is the set of extra build tags passed for tests.\n# We disable AVX collector for tests by default.\nTEST_TAGS := noavx,test\nGO_TEST   := $(GO_CMD) test $(GO_PARALLEL) -tags $(TEST_TAGS)\nGO_VET    := $(GO_CMD) vet -tags $(TEST_TAGS)\n\nTEST_SETUP   := test-setup.sh\nTEST_CLEANUP := test-cleanup.sh\n\n# Disable some golangci_lint checkers for now until we have an more acceptable baseline...\nGO_CILINT_CHECKERS := -D unused,staticcheck,errcheck,deadcode,structcheck,gosimple,revive -E gofmt\nGO_CILINT_RUNFLAGS := --build-tags $(TEST_TAGS)\n\n# Protoc compiler and protobuf definitions we might need to recompile.\nPROTOC    := $(shell command -v protoc;)\nPROTOBUFS  = $(shell find cmd pkg -name \\*.proto)\nPROTOCODE := $(patsubst %.proto,%.pb.go,$(PROTOBUFS))\n\nPROTO_INCLUDE = -I$(PWD):/usr/local/include:/usr/include\nPROTO_OPTIONS = --proto_path=. $(PROTO_INCLUDE) \\\n    --go_opt=paths=source_relative --go_out=. \\\n    --go-grpc_opt=paths=source_relative --go-grpc_out=.\nPROTO_COMPILE = $(PROTOC) $(PROTO_OPTIONS)\n\n\n# ShellCheck for checking shell scripts.\nSHELLCHECK := shellcheck\n\nCLANG := clang\nKERNEL_VERSION ?= $(shell uname -r)\nKERNEL_HEADERS_DIR ?= /lib/modules/$(KERNEL_VERSION)/source\nKERNEL_BUILD_DIR ?= /lib/modules/$(KERNEL_VERSION)/build\n# Directory for full kernel sources\nKERNEL_SRC_DIR ?= /usr/src/linux\n\n# Binaries and directories for installation.\nINSTALL    := install\nPREFIX     ?= /usr\nBINDIR     ?= $(PREFIX)/bin\nUNITDIR    ?= $(PREFIX)/lib/systemd/system\nDOCDIR     ?= $(PREFIX)/share/doc/cri-resource-manager\nSYSCONFDIR ?= /etc\nCONFIGDIR  ?= /etc/cri-resmgr\nDEFAULTDIR ?= $(shell \\\n    [ -d /etc/rpm ] && { echo /etc/sysconfig; exit 0; };  \\\n    [ -f /etc/debian_version ] && { echo /etc/default; exit 0; }; \\\n    echo unknown; exit 1)\n\n# Directories (in cmd) with go code we'll want to build and install.\nBUILD_DIRS = $(shell find cmd -name \\*.go | sed 's:cmd/::g;s:/.*::g' | uniq)\nBUILD_BINS = $(foreach dir,$(BUILD_DIRS),bin/$(dir))\n\n# Directories (in cmd) with go code we'll want to create Docker images from.\nIMAGE_DIRS  = $(shell find cmd -name Dockerfile | sed 's:cmd/::g;s:/.*::g' | uniq)\nIMAGE_VERSION  := $(shell git describe --dirty 2> /dev/null || echo unknown)\nifdef IMAGE_REPO\n    override IMAGE_REPO := $(IMAGE_REPO)/\nendif\n\n# List of our active go modules.\nGO_LIST_MODULES := $(GO_CMD) list ./... | grep -v vendor/\nGO_PKG_SRC = $(shell find pkg -name \\*.go)\n\n# List of visualizer collateral files to go generate.\nUI_ASSETS := $(shell for i in pkg/cri/resource-manager/visualizer/*; do \\\n        if [ -d \"$$i\" -a -e \"$$i/assets_generate.go\" ]; then \\\n            echo $$i/assets_gendata.go; \\\n        fi; \\\n    done)\n\n# Right now we don't depend on libexec/%.o on purpose so make sure the file\n# is always up-to-date when elf/avx512.c is changed.\nGEN_TARGETS := pkg/avx/programbytes_gendata.go $(PROTOCODE)\n\n# Determine binary version and buildid, and versions for rpm, deb, and tar packages.\nBUILD_VERSION := $(shell scripts/build/get-buildid --version --shell=no)\nBUILD_BUILDID := $(shell scripts/build/get-buildid --buildid --shell=no)\nRPM_VERSION   := $(shell scripts/build/get-buildid --rpm --shell=no)\nDEB_VERSION   := $(shell scripts/build/get-buildid --deb --shell=no)\nTAR_VERSION   := $(shell scripts/build/get-buildid --tar --shell=no)\n\n# Kubernetes version we pull in as modules and our external API versions.\nKUBERNETES_VERSION := $(shell grep 'k8s.io/kubernetes ' go.mod | sed 's/^.* //')\nRESMGR_API_VERSION := $(shell ls pkg/apis/resmgr | grep '^v[0-9]*')\n\n# Git (tagged) version and revisions we'll use to linker-tag our binaries with.\nRANDOM_ID := \"$(shell head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')\"\n\nifdef STATIC\n    STATIC_LDFLAGS:=-extldflags=-static\n    BUILD_TAGS:=-tags osusergo,netgo\nendif\n\nLDFLAGS    = \\\n    -ldflags \"$(STATIC_LDFLAGS) -X=github.com/intel/cri-resource-manager/pkg/version.Version=$(BUILD_VERSION) \\\n             -X=github.com/intel/cri-resource-manager/pkg/version.Build=$(BUILD_BUILDID) \\\n             -B 0x$(RANDOM_ID)\"\n\n# Build non-optimized version for debugging on make DEBUG=1.\nDEBUG ?= 0\nifeq ($(DEBUG),1)\n    GCFLAGS=-gcflags \"all=-N -l\"\nelse\n    GCFLAGS=\nendif\n\n# Release/end-to-end testing. Specify E2E_TESTS to override the default test set.\nE2E_RUN := reinstall_cri_resmgr=1 test/e2e/run_tests.sh\n\n# tar-related commands and options.\nTAR        := tar\nTAR_UPDATE := $(TAR) -uf\nGZIP       := gzip\nGZIP_DC    := gzip -dc\nGZEXT      := .gz\n\n# Metadata for packages, changelog, etc.\nUSER_NAME  ?= $(shell git config user.name)\nUSER_EMAIL ?= $(shell git config user.email)\nBUILD_DATE ?= $(shell date -R)\n\n# RPM spec files we might want to generate.\nSPEC_FILES = $(shell find packaging -name \\*.spec.in | sed 's/.spec.in/.spec/g' | uniq)\n\n# Systemd collateral.\nSYSTEMD_DIRS = $(shell find cmd -name \\*.service -o -name \\*.socket | sed 's:cmd/::g;s:/.*::g'|uniq)\nSYSCONF_DIRS = $(shell find cmd -name \\*.sysconf | sed 's:cmd/::g;s:/.*::g' | uniq)\n\nDOCKER := docker\n\n# Extra options to pass to docker (for instance --network host).\nDOCKER_OPTIONS =\n\n# Set this to empty to prevent 'docker build' from trying to pull all image refs.\nDOCKER_PULL := --pull\n\n# Docker boilerplate/commands to build debian/ubuntu packages.\nDOCKER_DEB_BUILD := \\\n    cd /build && \\\n    tar -xvf /build/input/cri-resource-manager-$(TAR_VERSION).tar.gz && \\\n    cd cri-resource-manager-$(TAR_VERSION) && \\\n    cp -r /build/input/debian . && \\\n    dpkg-buildpackage -uc && \\\n    cp ../*.{buildinfo,changes,deb,dsc} /output\n\n# Docker boilerplate/commands to build rpm packages.\nDOCKER_RPM_BUILD := \\\n    mkdir -p ~/rpmbuild/{SOURCES,SPECS} && \\\n    cp -v /build/input/*.spec ~/rpmbuild/SPECS && \\\n    cp -v /build/input/*.tar.* ~/rpmbuild/SOURCES && \\\n    for spec in ~/rpmbuild/SPECS/*.spec; do \\\n        rpmbuild -bb $$spec; \\\n    done && \\\n    cp -v $$(rpm --eval %{_rpmdir}/%{_arch})/*.rpm /output\n\n# Docker boilerplate/commands to build binary tarballs.\nDOCKER_TAR_BUILD := \\\n    cd ~ && \\\n    $(GZIP_DC) /build/input/cri-resource-manager-$(TAR_VERSION).tar$(GZ_EXT) | \\\n        $(TAR) -xf - && \\\n    cd cri-resource-manager-$(TAR_VERSION) && \\\n    $(MAKE) OUTPUT=/output/ binary-dist\n\n# Docker boilerplate/commands to build binaries.\nDOCKER_BIN_BUILD := \\\n    mkdir ~/build && cd ~/build && \\\n    tar -xvzf /build/input/cri-resource-manager-$(TAR_VERSION).tar$(GZEXT) && \\\n    cd cri-resource-manager-$(TAR_VERSION) && \\\n    make && \\\n    cp -v bin/* /output\n\n# Documentation-related variables\nSPHINXOPTS    ?= -W\nSPHINXBUILD   = sphinx-build\nSITE_BUILDDIR ?= _build\n\n# Docker base command for working with html documentation.\nDOCKER_SITE_BUILDER_IMAGE := cri-resmgr-site-builder\nDOCKER_SITE_CMD := $(DOCKER) run --rm -v \"`pwd`:/docs\" --user=`id -u`:`id -g` \\\n\t-p 8081:8081 \\\n\t-e SITE_BUILDDIR=$(SITE_BUILDDIR) -e SPHINXOPTS=$(SPHINXOPTS)\n\n\n# Supported distros with debian native packaging format.\nSUPPORTED_DEB_DISTROS := $(shell \\\n    grep -l 'apt-get ' dockerfiles/cross-build/Dockerfile.* | \\\n    egrep -v '((~)|(swp))$$' | \\\n    sed 's:^.*Dockerfile.::g')\n\n# Supported distros with rpm native packaging format.\nSUPPORTED_RPM_DISTROS := $(shell \\\n    egrep -l '(dnf )|(yum )|(zypper )' dockerfiles/cross-build/Dockerfile.* | \\\n    egrep -v '((~)|(swp))$$' | \\\n    sed 's:^.*Dockerfile.::g')\n\n# Directory to leave built distro packages and collateral in.\nPACKAGES_DIR := packages\n\n# Directory to leave build distro binaries in.\nBINARIES_DIR := binaries\n\n# Directory to use to build distro packages.\nBUILD_DIR := build\n\n# dist tarball target name\nifneq ($(wildcard .git/.),)\n    DIST_TARGET = dist-git\nelse\n    DIST_TARGET = dist-cwd\nendif\n\n# Paths to exclude from tarballs generated by dist-cwd.\nDIST_EXCLUDE := \\\n    --exclude=\"./$$tarball*\" \\\n    --exclude='./cri-resource-manager-*' \\\n    --exclude='./$(PACKAGES_DIR)*' \\\n    --exclude='./$(BUILD_DIR)*'\n\n# Path name transformations for tarballs generated by dist-cwd.\nDIST_TRANSFORM := \\\n    --transform='s:^.:cri-resource-manager-$(TAR_VERSION):'\n\n# Determine distro ID, version and package type.\nDISTRO_ID      := $(shell . /etc/os-release; echo \"$${ID:-unknown}\")\nDISTRO_VERSION := $(shell . /etc/os-release; echo \"$${VERSION_ID:-unknown}\")\nDISTRO_PACKAGE := $(shell echo $(DISTRO_ID) | tr -d ' \\t' | \\\n    sed -E 's/.*((fedora)|(suse)).*/rpm/;s/.*((ubuntu)|(debian)).*/deb/')\n\n# Be quiet by default but let folks override it with Q= or V=1 on the command line.\nifneq ($(V),1)\n  Q := @\nendif\n\n# Default target: just build everything.\nall: build\n\n#\n# Generic targets: build, install, clean, build images.\n#\n\nbuild: $(BUILD_BINS)\n\nbuild-static:\n\t$(MAKE) STATIC=1 build\n\ninstall: $(BUILD_BINS) $(foreach dir,$(BUILD_DIRS),install-bin-$(dir)) \\\n    $(foreach dir,$(BUILD_DIRS),install-systemd-$(dir)) \\\n    $(foreach dir,$(BUILD_DIRS),install-sysconf-$(dir)) \\\n    $(foreach dir,$(BUILD_DIRS),install-config-$(dir))\n\n\nclean: clean-bin clean-spec clean-deb clean-ui-assets clean-html\n\nimages: $(foreach dir,$(IMAGE_DIRS),image-$(dir))\n\nimages-push: $(foreach dir,$(IMAGE_DIRS),image-push-$(dir))\n\n#\n# Rules for building and installing binaries, or building docker images, and cleaning up.\n#\n\nKERNEL_INCLUDE_DIRS = /include \\\n                      /include/uapi \\\n                      /include/generated/uapi \\\n                      /arch/x86/include \\\n                      /arch/x86/include/uapi \\\n                      /arch/x86/include/generated/uapi\n\nKERNEL_INCLUDES := $(strip $(foreach kernel_dir,$(KERNEL_HEADERS_DIR) $(KERNEL_BUILD_DIR),$(addprefix -I,$(wildcard $(addprefix $(kernel_dir),$(KERNEL_INCLUDE_DIRS))))))\n\nlibexec/%.o: elf/%.c\n\t$(Q)if [ -z \"$(KERNEL_INCLUDES)\" ]; then echo \"Cannot build $@: invalid KERNEL_HEADERS_DIR=$(KERNEL_HEADERS_DIR)\"; exit 1; fi\n\t$(Q)echo \"Building $@\"\n\t$(Q)mkdir -p libexec\n\t$(Q)$(CLANG) -nostdinc -D __KERNEL__ $(KERNEL_INCLUDES) -O2 -Wall -target bpf -c $< -o $@\n\nbin/%: .static.%.$(STATIC)\n\t$(Q)bin=$(notdir $@); src=./cmd/$$bin; \\\n\techo \"Building $$([ -n \"$(STATIC)\" ] && echo 'static ')$@ (version $(BUILD_VERSION), build $(BUILD_BUILDID))...\"; \\\n\tmkdir -p bin && \\\n\t$(GO_BUILD) $(BUILD_TAGS) $(LDFLAGS) $(GCFLAGS) -o bin/ $$src\n\n.static.%.$(STATIC):\n\t$(Q)if [ ! -f \"$@\" ]; then \\\n\t    touch \"$@\"; \\\n\tfi; \\\n\told=\"$@\"; old=\"$${old%.*}\"; \\\n        if [ -n \"$(STATIC)\" ]; then \\\n\t    rm -f \"$$old.\"; \\\n\telse \\\n\t    rm -f \"$$old.1\"; \\\n\tfi\n\n.PRECIOUS: $(foreach dir,$(BUILD_DIRS),.static.$(dir).1 .static.$(dir).)\n\ninstall-bin-%: bin/%\n\t$(Q)bin=$(patsubst install-bin-%,%,$@); dir=cmd/$$bin; \\\n\techo \"Installing $$bin in $(DESTDIR)$(BINDIR)...\"; \\\n\t$(INSTALL) -d $(DESTDIR)$(BINDIR) && \\\n\t$(INSTALL) -m 0755 -t $(DESTDIR)$(BINDIR) bin/$$bin; \\\n\ninstall-systemd-%:\n\t$(Q)bin=$(patsubst install-systemd-%,%,$@); dir=cmd/$$bin; \\\n\techo \"Installing systemd collateral for $$bin...\"; \\\n\t$(INSTALL) -d $(DESTDIR)$(UNITDIR) && \\\n\tfor f in $$(find $$dir -name \\*.service -o -name \\*.socket); do \\\n\t    echo \"  $$f in $(DESTDIR)$(UNITDIR)...\"; \\\n\t    $(INSTALL) -m 0644 -t $(DESTDIR)$(UNITDIR) $$f.in; \\\n\tdone; \\\n\tfor f in $$(find $$dir -name \\*.service.in -o -name \\*.socket.in); do \\\n\t    echo \"  $$f in $(DESTDIR)$(UNITDIR)...\"; \\\n\t    df=$${f##*/}; df=$${df%.in}; \\\n\t    $(INSTALL) -m 0644 -T $$f $(DESTDIR)$(UNITDIR)/$$df; \\\n\t    sed -E -i -e \"s:__DEFAULTDIR__:$(DEFAULTDIR):g\" \\\n\t              -e \"s:__BINDIR__:$(BINDIR):g\" $(DESTDIR)$(UNITDIR)/$$df; \\\n\tdone\n\ninstall-sysconf-%:\n\t$(Q)bin=$(patsubst install-sysconf-%,%,$@); dir=cmd/$$bin; \\\n\techo \"Installing sysconf/default collateral for $$bin...\"; \\\n\t$(INSTALL) -d $(DESTDIR)$(DEFAULTDIR) && \\\n\tfor f in $$(find $$dir -name \\*.sysconf); do \\\n\t    echo \"  $$f in $(DESTDIR)$(DEFAULTDIR)...\"; \\\n\t    df=$${f##*/}; df=$${df%.sysconf}; \\\n\t    $(INSTALL) -m 0644 -T $$f $(DESTDIR)$(DEFAULTDIR)/$$df; \\\n\tdone\n\ninstall-config-%:\n\t$(Q)bin=$(patsubst install-config-%,%,$@); dir=cmd/$$bin; \\\n\techo \"Installing sample configuration collateral for $$bin...\"; \\\n\t$(INSTALL) -d $(DESTDIR)$(CONFIGDIR) && \\\n\tfor f in $$(find $$dir -name \\*.cfg.sample); do \\\n\t    echo \"  $$f in $(DESTDIR)$(CONFIGDIR)...\"; \\\n\t    df=$${f##*/}; \\\n\t    $(INSTALL) -m 0644 -T $$f $(DESTDIR)$(CONFIGDIR)/$${df}; \\\n\tdone\n\ninstall-minimal-docs:\n\t$(Q)echo \"Installing minimal documentation to $(DOCDIR)...\"; \\\n\t$(INSTALL) -d $(DESTDIR)$(DOCDIR) && \\\n\tfor f in LICENSE docs/security.md; do \\\n\t    echo \"  $$f in $(DESTDIR)$(DOCDIR)...\"; \\\n\t    df=$${f##*/}; \\\n\t    $(INSTALL) -m 0644 -T $$f $(DESTDIR)$(DOCDIR)/$${df}; \\\n\tdone\n\ninstall-licenses:\n\t$(Q)for cmd in $(BUILD_DIRS); do \\\n\t    install -D LICENSE $(DESTDIR)/licenses/$$cmd/LICENSE && \\\n\t    go-licenses save ./cmd/$$cmd \\\n\t        --ignore github.com/intel/cri-resource-manager \\\n\t        --save_path $(DESTDIR)/licenses/$$cmd/go-licenses; \\\n\tdone\n\nclean-bin: $(foreach dir,$(BUILD_DIRS),clean-$(dir))\n\t$(Q)rm -f .static.*\n\nclean-%:\n\t$(Q)bin=$(patsubst clean-%,%,$@); src=cmd/$$bin; \\\n\techo \"Cleaning up $$bin...\"; \\\n\trm -f bin/$$bin\n\nclean-gen:\n\t$(Q)rm -f $(GEN_TARGETS)\n\nimage-%:\n\t$(Q)bin=$(patsubst image-%,%,$@); \\\n\t    $(DOCKER) build . -f \"cmd/$$bin/Dockerfile\" \\\n\t    --build-arg GO_VERSION=$(GO_VERSION) \\\n\t    --build-arg GOLICENSES_VERSION=$(GOLICENSES_VERSION) \\\n\t    -t $(IMAGE_REPO)$$bin:$(IMAGE_VERSION)\n\nimage-push-%:\n\t$(Q)bin=$(patsubst image-push-%,%,$@); \\\n\t\tif [ -z \"$(IMAGE_REPO)\" ]; then echo \"ERROR: no IMAGE_REPO specified\"; exit 1; fi; \\\n\t\t$(DOCKER) push $(IMAGE_REPO)$$bin:$(IMAGE_VERSION)\n\n#\n# Rules for format checking, various code quality and complexity checks and measures.\n#\n\nformat:\n\t$(Q)report=`$(GO_FMT) -s -d -w $$(find cmd pkg test/functional -name \\*.go)`; \\\n\tif [ -n \"$$report\" ]; then \\\n\t    echo \"$$report\"; \\\n\t    exit 1; \\\n\tfi\n\nvet:\n\t$(Q)$(GO_VET) $(shell $(GO_LIST_MODULES))\n\ncyclomatic-check:\n\t$(Q)report=`$(GO_CYCLO) -over 15 cmd pkg`; \\\n\tif [ -n \"$$report\" ]; then \\\n\t    echo \"Complexity is over 15 in\"; \\\n\t    echo \"$$report\"; \\\n\t    exit 1; \\\n\tfi\n\nlint:\n\t$(Q)rc=0; \\\n\tfor f in $$(find -name \\*.go | grep -v \\.\\/vendor); do \\\n\t    $(GO_LINT) -set_exit_status $$f || rc=1; \\\n\tdone; \\\n\texit $$rc\n\ngolangci-lint:\n\t$(Q)$(GO_CILINT) run $(GO_CILINT_RUNFLAGS) $(GO_CILINT_CHECKERS)\n\nshellcheck:\n\t$(Q)for f in $$(git grep -n '^#!/bin/.*sh *' | grep ':1:#!' | sed 's/:1:.*//'); do \\\n\t    echo \"shellchecking $$f...\"; \\\n\t    $(SHELLCHECK) $$f; \\\n\tdone\n\n\n#\n# Rules for running unit/module tests.\n#\n\ntest: test-setup test-run test-cleanup\nrace-test racetest: test-setup racetest-run test-cleanup\n\ntest-setup:\n\t$(Q)for i in $$(find . -name $(TEST_SETUP)); do \\\n\t    echo \"+ Running test setup $$i...\"; \\\n\t    (cd $${i%/*}; \\\n\t        if [ -x \"$(TEST_SETUP)\" ]; then \\\n\t            ./$(TEST_SETUP); \\\n\t        fi); \\\n\tdone\n\ntest-cleanup:\n\t$(Q)for i in $$(find . -name $(TEST_CLEANUP)); do \\\n\t    echo \"- Running test cleanup $$i...\"; \\\n\t    (cd $${i%/*}; \\\n\t        if [ -x \"$(TEST_CLEANUP)\" ]; then \\\n\t            ./$(TEST_CLEANUP); \\\n\t        fi); \\\n\tdone\n\ntest-run:\nifndef WHAT\n\t$(Q)$(GO_TEST) -race -coverprofile=coverage.txt -covermode=atomic \\\n\t    $(shell $(GO_LIST_MODULES))\nelse\n\t$(Q)if [ -n '$(TESTS)' ]; then \\\n\t        run=\"-run $(TESTS)\"; \\\n\t    fi; \\\n\tcd $(WHAT) && \\\n            $(GO_TEST) $$run -v -cover -coverprofile cover.out || rc=1; \\\n            $(GO_CMD) tool cover -html=cover.out -o coverage.html; \\\n            rm cover.out; \\\n            echo \"Coverage report: file://$$(realpath coverage.html)\"; \\\n            exit $$rc\nendif\n\nracetest-run:\nifndef WHAT\n\t$(Q)$(GO_TEST) -race -coverprofile=coverage.txt -covermode=atomic \\\n\t    $(shell $(GO_LIST_MODULES))\nelse\n\t$(Q)cd $(WHAT) && \\\n\t    $(GO_TEST) -race -coverprofile=cover.out -covermode=atomic || rc=1; \\\n            $(GO_CMD) tool cover -html=cover.out -o coverage.html; \\\n            rm cover.out; \\\n            echo \"Coverage report: file://$$(realpath coverage.html)\"; \\\n            exit $$rc\nendif\n\nrelease-tests: e2e-tests\n\ne2e-tests: build-static\n\t$(Q)tests=\"$(if $(E2E_TESTS),$(E2E_TESTS),test/e2e/policies.test-suite)\"; \\\n\t$(E2E_RUN) $$tests; \\\n\tif [ \"$$?\" != \"0\" ]; then \\\n\t    echo \"You drop into interactive mode upon failures if you run e2e tests as\"; \\\n\t    echo \"    on_verify_fail=interactive $(E2E_RUN) $$tests\"; \\\n\t    exit 1; \\\n\tfi\n\npackaging-tests: cross-packages\n\t$(Q)cleanup=1 omit_agent=1 $(E2E_RUN) test/e2e/packages.test-suite\n\n#\n# Rules for building distro packages.\n#\n\nifneq ($(DISTRO_ID),fedora)\n    packages: cross-$(DISTRO_PACKAGE).$(DISTRO_ID)-$(DISTRO_VERSION)\nelse\n    packages: cross-$(DISTRO_PACKAGE).$(DISTRO_ID)\nendif\n\ncross-packages: cross-rpm cross-deb cross-tar\n\ncross-rpm: $(foreach d,$(SUPPORTED_RPM_DISTROS),cross-rpm.$(d))\n\ncross-deb: $(foreach d,$(SUPPORTED_DEB_DISTROS),cross-deb.$(d))\n\ncross-bin: $(foreach d,$(SUPPORTED_RPM_DISTROS),cross-bin.$(d)) \\\n           $(foreach d,$(SUPPORTED_DEB_DISTROS),cross-bin.$(d))\n\n#\n# Rules for building dist-tarballs, rpm, and deb packages.\n#\n\ndist: $(DIST_TARGET)\n\ndist-git:\n\t$(Q)echo \"Using git to create dist tarball $(TAR_VERSION) from $(BUILD_BUILDID)...\"; \\\n\ttardir=cri-resource-manager-$(TAR_VERSION) && \\\n\ttarball=cri-resource-manager-$(TAR_VERSION).tar && \\\n\tgit archive --format=tar --prefix=$$tardir/ HEAD > $$tarball && \\\n\tmkdir -p $$tardir && \\\n\t    echo $(BUILD_VERSION) > $$tardir/version && \\\n\t    echo $(BUILD_BUILDID) > $$tardir/buildid && \\\n\t$(TAR) -uf $$tarball $$tardir && \\\n\trm -f $$tarball.* && \\\n\t$(GZIP) $$tarball && \\\n\trm -fr $$tardir\n\ndist-cwd:\n\t$(Q)echo \"Using tar to create dist tarball $(TAR_VERSION) from $$(pwd)...\"; \\\n\ttardir=cri-resource-manager-$(TAR_VERSION) && \\\n\ttarball=cri-resource-manager-$(TAR_VERSION).tar && \\\n\t$(TAR) $(DIST_EXCLUDE) $(DIST_TRANSFORM) -cvf - . > $$tarball && \\\n\tmkdir -p $$tardir && \\\n\t    echo $(BUILD_VERSION) > $$tardir/version && \\\n\t    echo $(BUILD_BUILDID) > $$tardir/buildid && \\\n\t$(TAR_UPDATE) $$tarball $$tardir && \\\n\trm -f $$tarball.* && \\\n\t$(GZIP) $$tarball && \\\n\trm -fr $$tardir\n\nvendored-dist: dist\n\t$(Q)echo \"Creating vendored dist tarball $(TAR_VERSION)...\"; \\\n\ttardir=cri-resource-manager-$(TAR_VERSION) && \\\n\ttarball=cri-resource-manager-$(TAR_VERSION).tar && \\\n\tcp $$tarball$(GZEXT) vendored-$$tarball$(GZEXT) && \\\n\t$(GZIP_DC) vendored-$$tarball$(GZEXT) | tar -xf - && \\\n\tgo mod vendor -v && \\\n\tmkdir -p $$tardir && \\\n\t  mv vendor $$tardir && \\\n\trm -f vendored-$$tarball* && \\\n\t$(TAR) -cf vendored-$$tarball $$tardir && \\\n\t$(GZIP) vendored-$$tarball && \\\n\trm -fr $$tardir\n\nbinary-dist:\n\t$(Q)tarball=$(OUTPUT)cri-resource-manager-$(TAR_VERSION).$$(uname -m).tar; \\\n\techo \"Creating binary dist tarball $$tarball...\"; \\\n\ttardir=binary-dist; \\\n\trm -fr $$tarball* $$tardir && \\\n\t$(MAKE) DESTDIR=$$tardir \\\n\t        BUILD_DIRS=cri-resmgr \\\n\t        PREFIX=/opt/intel \\\n\t        DEFAULTDIR=/etc/default \\\n\t        UNITDIR=$(SYSCONFDIR)/systemd/system install install-minimal-docs && \\\n\t$(MAKE) DESTDIR=$$tardir/opt/intel/ install-licenses && \\\n\t$(TAR) -C $$tardir -cf $$tarball . && \\\n\t$(GZIP) $$tarball && \\\n\trm -fr $$tardir\n\nspec: clean-spec $(SPEC_FILES)\n\n%.spec:\n\t$(Q)echo \"Generating RPM spec file $@...\"; \\\n\tcp $@.in $@ && \\\n\tsed -E -i -e \"s/__VERSION__/$(RPM_VERSION)/g\"    \\\n\t          -e \"s/__TARVERSION__/$(TAR_VERSION)/g\" \\\n\t          -e \"s/__BUILDID__/$(BUILD_BUILDID)/g\" $@\n\nclean-spec:\n\t$(Q)rm -f $(SPEC_FILES)\n\ncross-rpm.%: docker/cross-build/% clean-spec spec dist\n\t$(Q)distro=$(patsubst cross-rpm.%,%,$@); \\\n\tbuilddir=$(BUILD_DIR)/docker/$$distro; \\\n\toutdir=$(PACKAGES_DIR)/$$distro; \\\n\techo \"Docker cross-building $$distro packages...\"; \\\n\tmkdir -p $(PACKAGES_DIR)/$$distro && \\\n\trm -fr $$builddir && mkdir -p $$builddir/{input,build} && \\\n\tcp cri-resource-manager-$(TAR_VERSION).tar$(GZEXT) $$builddir/input && \\\n\tcp packaging/rpm/cri-resource-manager.spec $$builddir/input && \\\n\t$(DOCKER) run --rm $(DOCKER_OPTIONS) --user $$USER \\\n\t    --env USER_NAME=\"$(USER_NAME)\" --env USER_EMAIL=$(USER_EMAIL) \\\n\t    -v $$(pwd)/$$builddir:/build \\\n\t    -v $$(pwd)/$$outdir:/output \\\n\t    -v \"`go env GOMODCACHE`:/home/$$USER/go/pkg/mod\" \\\n\t    $$distro-build /bin/bash -c '$(DOCKER_RPM_BUILD)' && \\\n\trm -fr $$builddir && \\\n\tinstall -D -m644  $$outdir/cri-resource-manager-$(RPM_VERSION)-0.x86_64.rpm $(PACKAGES_DIR)/release-assets/cri-resource-manager-$(RPM_VERSION)-0.$$distro.x86_64.rpm\n\nsrc.rpm source-rpm: spec dist\n\tmkdir -p ~/rpmbuild/{SOURCES,SPECS} && \\\n\tcp packaging/rpm/cri-resource-manager.spec ~/rpmbuild/SPECS && \\\n\tcp cri-resource-manager-$(TAR_VERSION).tar$(GZEXT) ~/rpmbuild/SOURCES && \\\n\trpmbuild -bs ~/rpmbuild/SPECS/cri-resource-manager.spec\n\nrpm: source-rpm\n\trpmbuild -bb ~/rpmbuild/SPECS/cri-resource-manager.spec\n\ndebian/%: packaging/deb.in/%\n\t$(Q)echo \"Generating debian packaging file $@...\"; \\\n\ttardir=cri-resource-manager-$(TAR_VERSION) && \\\n\ttarball=cri-resource-manager-$(TAR_VERSION).tar && \\\n\tmkdir -p debian; \\\n\tcp $< $@ && \\\n\tsed -E -i -e \"s/__PACKAGE__/cri-resource-manager/g\" \\\n\t          -e \"s/__TARBALL__/$$tarball/g\"            \\\n\t          -e \"s/__VERSION__/$(DEB_VERSION)/g\"       \\\n\t          -e \"s/__AUTHOR__/$(USER_NAME)/g\"          \\\n\t          -e \"s/__EMAIL__/$(USER_EMAIL)/g\"          \\\n\t          -e \"s/__DATE__/$(BUILD_DATE)/g\" $@\n\nclean-deb:\n\t$(Q)rm -fr debian\n\ncross-deb.%: docker/cross-build/% \\\n    clean-deb debian/changelog debian/control debian/rules debian/compat dist\n\t$(Q)distro=$(patsubst cross-deb.%,%,$@); \\\n\techo \"Docker cross-building $$distro packages...\"; \\\n\tbuilddir=$(BUILD_DIR)/docker/$$distro; \\\n\toutdir=$(PACKAGES_DIR)/$$distro; \\\n\tmkdir -p $(PACKAGES_DIR)/$$distro && \\\n\trm -fr $$builddir && mkdir -p $$builddir/{input,build} && \\\n\tcp cri-resource-manager-$(TAR_VERSION).tar$(GZEXT) $$builddir/input && \\\n\tcp -r debian $$builddir/input && \\\n\t$(DOCKER) run --rm $(DOCKER_OPTIONS) --user $$USER \\\n\t    --env USER_NAME=\"$(USER_NAME)\" --env USER_EMAIL=$(USER_EMAIL) \\\n\t    -v $$(pwd)/$$builddir:/build \\\n\t    -v $$(pwd)/$$outdir:/output \\\n\t    -v \"`go env GOMODCACHE`:/home/$$USER/go/pkg/mod\" \\\n\t    $$distro-build /bin/bash -c '$(DOCKER_DEB_BUILD)' && \\\n\trm -fr $$builddir && \\\n\tinstall -D -m644 $$outdir/cri-resource-manager_$(DEB_VERSION)_amd64.deb $(PACKAGES_DIR)/release-assets/cri-resource-manager_$(DEB_VERSION)_$${distro}_amd64.deb\n\ndeb: debian/changelog debian/control debian/rules debian/compat dist\n\tdpkg-buildpackage -uc\n\ncross-bin.%: docker/cross-build/% dist\n\t$(Q)distro=$(patsubst cross-bin.%,%,$@); \\\n\techo \"Docker cross-building $$distro binaries...\"; \\\n\tbuilddir=$(BUILD_DIR)/docker/$$distro; \\\n\toutdir=$(BINARIES_DIR)/$$distro; \\\n\tmkdir -p $(BINARIES_DIR)/$$distro && \\\n\trm -fr $$builddir && mkdir -p $$builddir/{input,build} && \\\n\tcp cri-resource-manager-$(TAR_VERSION).tar$(GZEXT) $$builddir/input && \\\n\t$(DOCKER) run --rm $(DOCKER_OPTIONS) --user $$USER \\\n\t    --env USER_NAME=\"$(USER_NAME)\" --env USER_EMAIL=$(USER_EMAIL) \\\n\t    -v $$(pwd)/$$builddir:/build \\\n\t    -v $$(pwd)/$$outdir:/output \\\n\t    -v \"`go env GOMODCACHE`:/home/$$USER/go/pkg/mod\" \\\n\t    $$distro-build /bin/bash -c '$(DOCKER_BIN_BUILD)' && \\\n\trm -fr $$builddir\n\ncross-tar cross-tarball: dist docker/cross-build/fedora\n\t$(Q)distro=tarball; \\\n\tbuilddir=$(BUILD_DIR)/docker/$$distro; \\\n\toutdir=$(PACKAGES_DIR)/$$distro; \\\n\techo \"Docker cross-building $$distro packages...\"; \\\n\tmkdir -p $$outdir && \\\n\trm -fr $$builddir && mkdir -p $$builddir/{input,build} && \\\n\tcp cri-resource-manager-$(TAR_VERSION).tar$(GZEXT) $$builddir/input && \\\n\t$(DOCKER) run --rm $(DOCKER_OPTIONS) --user $$USER \\\n\t    --env USER_NAME=\"$(USER_NAME)\" --env USER_EMAIL=$(USER_EMAIL) \\\n\t    -v $$(pwd)/$$builddir:/build \\\n\t    -v $$(pwd)/$$outdir:/output \\\n\t    -v \"`go env GOMODCACHE`:/home/$$USER/go/pkg/mod\" \\\n\t    fedora-build /bin/bash -c '$(DOCKER_TAR_BUILD)' && \\\n\trm -fr $$builddir && \\\n\tinstall -D -m644 -t $(PACKAGES_DIR)/release-assets $$outdir/cri-resource-manager-$(TAR_VERSION).x86_64.tar.gz\n\n# Build a docker image (for distro cross-building).\ndocker/cross-build/%: dockerfiles/cross-build/Dockerfile.%\n\t$(Q)distro=$(patsubst docker/cross-build/%,%,$@) && \\\n\techo \"Building cross-build docker image for $$distro...\" && \\\n\timg=$${distro}-build && $(DOCKER) rm $$distro-build || : && \\\n\tscripts/build/docker-build-image $$distro-build \\\n\t    $(DOCKER_PULL) \\\n\t    --build-arg GO_VERSION=$(GO_VERSION) \\\n\t    --build-arg GOLICENSES_VERSION=$(GOLICENSES_VERSION) \\\n\t    $(DOCKER_OPTIONS)\n\n# Rule for recompiling a changed protobuf.\n%.pb.go: %.proto\n\t$(Q)if [ -n \"$(PROTOC)\" -o ! -e \"$@\" ]; then \\\n\t        echo \"Generating go code ($@) for updated protobuf $<...\"; \\\n\t\t$(PROTO_COMPILE) $<; \\\n\telse \\\n\t        echo \"WARNING: no protoc found, compiling with OUTDATED $@...\"; \\\n\tfi\n\n\n# Rule for installing in-repo git hooks.\ninstall-git-hooks:\n\t$(Q)if [ -d .git -a ! -e .git-hooks.redirected ]; then \\\n\t    echo -n \"Redirecting git hooks to .githooks...\"; \\\n\t    git config core.hookspath .githooks && \\\n\t    touch .git-hooks.redirected && \\\n\t    echo \"done.\"; \\\n\tfi\n\n# Rules for installing protoc and related utilities.\ninstall-protoc:\n\t$(Q)./scripts/hack/install-protobuf\n\ninstall-protoc-gen-go:\n\t$(Q)$(GO_INSTALL) google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0\n\ninstall-protoc-gen-go-grpc:\n\t$(Q)$(GO_INSTALL) google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0\n\ninstall-protoc-tools: install-protoc install-protoc-gen-go install-protoc-gen-go-grpc\n\n#\n# go dependencies for our binaries (careful with that axe, Eugene...)\n#\n\nbin/cri-resmgr: $(wildcard cmd/cri-resmgr/*.go) $(UI_ASSETS) $(GEN_TARGETS) \\\n    $(shell for dir in \\\n                  $(shell go list -f '{{ join .Deps  \"\\n\"}}' ./cmd/cri-resmgr/... | \\\n                          grep cri-resource-manager/pkg/ | \\\n                          sed 's#github.com/intel/cri-resource-manager/##g'); do \\\n                find $$dir -name \\*.go; \\\n            done | sort | uniq)\n\nbin/cri-resmgr-agent: $(wildcard cmd/cri-resmgr-agent/*.go) \\\n    $(shell for dir in \\\n                  $(shell go list -f '{{ join .Deps  \"\\n\"}}' ./cmd/cri-resmgr-agent/... | \\\n                          grep cri-resource-manager/pkg/ | \\\n                          sed 's#github.com/intel/cri-resource-manager/##g'); do \\\n                find $$dir -name \\*.go; \\\n            done | sort | uniq)\n\nbin/webhook: $(wildcard cmd/cri-resmgr-webhook/*.go) \\\n    $(shell for dir in \\\n                  $(shell go list -f '{{ join .Deps  \"\\n\"}}' ./cmd/cri-resmgr-webhook/... | \\\n                          grep cri-resource-manager/pkg/ | \\\n                          sed 's#github.com/intel/cri-resource-manager/##g'); do \\\n                find $$dir -name \\*.go; \\\n            done | sort | uniq)\n\n#\n# rules to run go generators\n#\nclean-ui-assets:\n\t$(Q)echo \"Cleaning up generated UI assets...\"; \\\n\tfor i in $(UI_ASSETS); do \\\n\t    echo \"  - $$i\"; \\\n\t    rm -f $$i; \\\n\tdone\n\n%_gendata.go::\n\t$(Q)echo \"Generating $@...\"; \\\n\tcd $(dir $@) && \\\n\t    $(GO_GEN) || exit 1 && \\\n\tcd - > /dev/null\n\npkg/sysfs/sst_types%.go: pkg/sysfs/_sst_types%.go pkg/sysfs/gen_sst_types.sh\n\t$(Q)cd $(@D) && \\\n\t    KERNEL_SRC_DIR=$(KERNEL_SRC_DIR) $(GO_GEN)\n\n\n#\n# API generation\n#\n\n# unconditionally generate all apis\ngenerate-apis: generate-resmgr-api\n\n# unconditionally generate (external) resmgr api\ngenerate-resmgr-api:\n\t$(Q)$(call generate-api,resmgr,$(RESMGR_API_VERSION))\n\n# automatic update of generated code for resource-manager external api\npkg/apis/resmgr/$(RESMGR_API_VERSION)/zz_generated.deepcopy.go: \\\n    pkg/apis/resmgr/$(RESMGR_API_VERSION)/types.go\n\t$(Q)$(call generate-api,resmgr,$(RESMGR_API_VERSION))\n\n# macro to generate code for api $(1), version $(2)\ngenerate-api = \\\n\techo \"Generating '$(1)' api, version $(2)...\" && \\\n\t    KUBERNETES_VERSION=$(KUBERNETES_VERSION) \\\n\t    ./scripts/code-generator/generate-groups.sh all \\\n\t        github.com/intel/cri-resource-manager/pkg/apis/$(1)/generated \\\n\t        github.com/intel/cri-resource-manager/pkg/apis $(1):$(2) \\\n\t        --output-base $(shell pwd)/generate && \\\n\t    cp -r generate/github.com/intel/cri-resource-manager/pkg/apis/$(1) pkg/apis && \\\n\t        rm -fr generate/github.com/intel/cri-resource-manager/pkg/apis/$(1)\n\n\n#\n# dependencies for UI assets baked in using vfsgendev (can't come up with a working pattern rule)\n#\n\npkg/cri/resource-manager/visualizer/bubbles/assets_gendata.go:: \\\n\t$(wildcard pkg/cri/resource-manager/visualizer/bubbles/assets/*.html) \\\n\t$(wildcard pkg/cri/resource-manager/visualizer/bubbles/assets/js/*.js) \\\n\t$(wildcard pkg/cri/resource-manager/visualizer/bubbles/assets/css/*.css)\n\n\n# phony targets\n.PHONY: all build install clean test images images-push release-tests e2e-tests \\\n\tformat vet cyclomatic-check lint golangci-lint \\\n\tcross-packages cross-rpm cross-deb \\\n\n#\n# Rules for documentation\n#\n\nvhtml: _work/venv/.stamp\n\t. _work/venv/bin/activate && \\\n\t\tmake -C docs html && \\\n\t\tcp -r docs/_build .\n\nhtml: clean-html\n\t$(Q)BUILD_VERSION=$(BUILD_VERSION) \\\n\t\t$(SPHINXBUILD) -c docs . \"$(SITE_BUILDDIR)\" $(SPHINXOPTS)\n\tcp docs/index.html \"$(SITE_BUILDDIR)\"\n\tfor d in $$(find docs -name figures -type d); do \\\n\t    mkdir -p $(SITE_BUILDDIR)/$$d && cp $$d/* $(SITE_BUILDDIR)/$$d; \\\n\tdone\n\nserve-html: html\n\t$(Q)cd $(SITE_BUILDDIR) && python3 -m http.server 8081\n\nclean-html:\n\trm -rf $(SITE_BUILDDIR)\n\nsite-build: .$(DOCKER_SITE_BUILDER_IMAGE).image.stamp\n\t$(Q)$(DOCKER_SITE_CMD) $(DOCKER_SITE_BUILDER_IMAGE) make html\n\nsite-serve: .$(DOCKER_SITE_BUILDER_IMAGE).image.stamp\n\t$(Q)$(DOCKER_SITE_CMD) -it $(DOCKER_SITE_BUILDER_IMAGE) make serve-html\n\n.$(DOCKER_SITE_BUILDER_IMAGE).image.stamp: docs/Dockerfile docs/requirements.txt\n\tdocker build -t $(DOCKER_SITE_BUILDER_IMAGE) docs\n\ttouch $@\n\n# Set up a Python3 environment with the necessary tools for document creation.\n_work/venv/.stamp: docs/requirements.txt\n\trm -rf ${@D}\n\tpython3 -m venv ${@D}\n\t. ${@D}/bin/activate && pip install -r $<\n\ttouch $@\n"
  },
  {
    "path": "README.md",
    "content": "# CRI Resource Manager for Kubernetes\\*\n\n## ⚠️ The project is no longer maintained ⚠️\n\nThe CRI Resource manager project is no longer maintained. No further updates,\nbug fixes or releases are planned.\n\nWe recommend users migrate to\n[NRI Plugins](https://github.com/containers/nri-plugins), which provides\nsimilar functionality and is actively maintained.\n\nThank you for being part of this journey!\n\n### See our [Documentation][documentation] site for detailed documentation.\n\n[documentation]: https://intel.github.io/cri-resource-manager\n"
  },
  {
    "path": "SECURITY.md",
    "content": "# Security Policy\nIntel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation.\n\n## Reporting a Vulnerability\nPlease report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).\n"
  },
  {
    "path": "cmd/cri-resmgr/cri-resource-manager.service.in",
    "content": "[Unit]\nDescription=A CRI proxy with (hardware) resource aware container placement policies.\nDocumentation=https://github.com/intel/cri-resource-manager\nBefore=kubelet.service\nLogRateLimitIntervalSec=5\nLogRateLimitBurst=100000\n\n[Service]\nType=simple\nEnvironmentFile=__DEFAULTDIR__/cri-resource-manager\nExecStart=__BINDIR__/cri-resmgr $CONFIG_OPTIONS $POLICY_OPTIONS\nRestart=always\n\n[Install]\nWantedBy=multi-user.target\n"
  },
  {
    "path": "cmd/cri-resmgr/cri-resource-manager.sysconf",
    "content": "# Configuration options to pass to cri-resmgr when started via systemd.\n\n# Use a fallback file for configuration if/when we can't acquire one from the agent.\nCONFIG_OPTIONS=\"--fallback-config /etc/cri-resmgr/fallback.cfg\"\n\n# Enable this for preventing the active policy to be changed during startup.\n#POLICY_OPTIONS=\"--disable-policy-switch\"\n"
  },
  {
    "path": "cmd/cri-resmgr/fallback.cfg.sample",
    "content": "#\n# If you pass this file to cri-resmgr using the --fallback-config\n# command line option, it will be used if configuration cannot be\n# acquired from any other source (agent, or last configuration\n# stored in the cache).\n#\n# Switching Policies:\n#     Recent versions of cri-resmgr will allow changing the active\n#     policy during startup. If you want to prevent this from hap-\n#     pening you can pass the --disable-policy-switch option to\n#     cri-resmgr on the command line.\n#\n#     With the stock packaging you can control whether startup-\n#     phase policy switching is allowed using the POLICY_OPTIONS\n#     variable in the sysconf file.\n#\n#     If switching policies is disabled, you can still reset the\n#     active policy manually when cri-resmgr is not running. This\n#     allows cri-resmgr to start up next with a new policy. You\n#     do this by passing the --reset-policy command line option\n#     to cri-resmgr. The full sequence of switching policies this\n#     way is\n#         - stop cri-resmgr (systemctl stop cri-resource-manager),\n#         - reset the active policy (cri-resmgr --reset-policy),\n#         - start cri-resmgr (systemctl start cri-resource-manager)\n#\n\npolicy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: resource-manager,cache,resource-control\ndump:\n  Config: off:.*,full:((Create)|(Remove)|(Run)|(Update)|(Start)|(Stop)).*\n"
  },
  {
    "path": "cmd/cri-resmgr/main.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/intel/goresctrl/pkg/rdt\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\tversion \"github.com/intel/cri-resource-manager/pkg/version\"\n)\n\nvar log = logger.Default()\n\nfunc main() {\n\trate := logger.Rate{Limit: logger.Every(1 * time.Minute)}\n\tlogger.SetGrpcLogger(\"grpc\", &rate)\n\tlogger.SetStdLogger(\"stdlog\")\n\trdt.SetLogger(logger.Get(\"rdt\"))\n\n\tprintConfig := flag.Bool(\"print-config\", false, \"Print configuration and exit.\")\n\tlistPolicies := flag.Bool(\"list-policies\", false, \"List available policies.\")\n\tflag.Parse()\n\n\tswitch {\n\tcase *printConfig:\n\t\tconfig.Print(nil)\n\t\tos.Exit(0)\n\n\tcase *listPolicies:\n\t\tfmt.Printf(\"Available policies:\\n\")\n\t\tfor _, available := range policy.AvailablePolicies() {\n\t\t\tfmt.Printf(\"  * %s: %s\\n\", available.Name, available.Description)\n\t\t}\n\t\tos.Exit(0)\n\n\tdefault:\n\t\tif args := flag.Args(); len(args) > 0 {\n\t\t\tswitch args[0] {\n\t\t\tcase \"config-help\", \"help\":\n\t\t\t\tconfig.Describe(args[1:]...)\n\t\t\t\tos.Exit(0)\n\t\t\tdefault:\n\t\t\t\tlog.Error(\"unknown command line arguments: %s\", strings.Join(flag.Args(), \",\"))\n\t\t\t\tflag.Usage()\n\t\t\t\tos.Exit(1)\n\t\t\t}\n\t\t}\n\t}\n\n\tlogger.Flush()\n\tlogger.SetupDebugToggleSignal(syscall.SIGUSR1)\n\tlog.Info(\"cri-resmgr (version %s, build %s) starting...\", version.Version, version.Build)\n\n\tif err := instrumentation.Start(); err != nil {\n\t\tlog.Fatal(\"failed to set up instrumentation: %v\", err)\n\t}\n\tdefer instrumentation.Stop()\n\n\tm, err := resmgr.NewResourceManager()\n\tif err != nil {\n\t\tlog.Fatal(\"failed to create resource manager instance: %v\", err)\n\t}\n\n\tif err := m.Start(); err != nil {\n\t\tlog.Fatal(\"failed to start resource manager: %v\", err)\n\t}\n\n\tfor {\n\t\ttime.Sleep(15 * time.Second)\n\t}\n}\n"
  },
  {
    "path": "cmd/cri-resmgr-agent/Dockerfile",
    "content": "ARG GO_VERSION=1.24\n\nFROM golang:${GO_VERSION}-bullseye as builder\n\nARG GOLICENSES_VERSION\n\nWORKDIR /go/build\n\n# Fetch go dependencies in a separate layer for caching\nRUN go install github.com/google/go-licenses@${GOLICENSES_VERSION}\nCOPY go.mod go.sum ./\nCOPY pkg/topology/ pkg/topology/\nRUN go mod download -x\n\n# Build agent and agent-probe, fully statically linked binary\nCOPY . .\n\nRUN CGO_ENABLED=0 make build-static BUILD_DIRS=\"cri-resmgr-agent cri-resmgr-agent-probe\" && \\\n    install -D /go/build/bin/* -t /install_root/bin\n\n# Save licenses\nRUN make install-licenses BUILD_DIRS=\"cri-resmgr-agent cri-resmgr-agent-probe\" DESTDIR=/install_root\n\nFROM scratch as final\n\nCOPY --from=builder /install_root /\n\nENTRYPOINT [\"/bin/cri-resmgr-agent\"]\n"
  },
  {
    "path": "cmd/cri-resmgr-agent/agent-deployment.yaml",
    "content": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: cri-resmgr-agent\n  namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: cri-resmgr-agent\nrules:\n- apiGroups:\n  - \"\"\n  - criresmgr.intel.com\n  resources:\n  - nodes\n  - configmaps\n  - adjustments\n  - labels\n  - annotations\n  verbs:\n  - get\n  - patch\n  - update\n  - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: cri-resmgr-agent\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: cri-resmgr-agent\nsubjects:\n- kind: ServiceAccount\n  name: cri-resmgr-agent\n  namespace: kube-system\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  labels:\n    app: cri-resmgr-agent\n  name: cri-resmgr-agent\n  namespace: kube-system\nspec:\n  selector:\n    matchLabels:\n      app: cri-resmgr-agent\n  template:\n    metadata:\n      labels:\n        app: cri-resmgr-agent\n    spec:\n      serviceAccount: cri-resmgr-agent\n      containers:\n        - name: cri-resmgr-agent\n          env:\n          - name: NODE_NAME\n            valueFrom:\n              fieldRef:\n                fieldPath: spec.nodeName\n          image: IMAGE_PLACEHOLDER\n          imagePullPolicy: Always # for testing\n          securityContext:\n            allowPrivilegeEscalation: false\n            capabilities:\n              drop: [\"ALL\"]\n            readOnlyRootFilesystem: true\n          volumeMounts:\n          - name: resmgrsockets\n            mountPath: /var/run/cri-resmgr\n          resources:\n            limits:\n              cpu: 1\n              memory: 512Mi\n          livenessProbe:\n            exec:\n              command: [\"/bin/cri-resmgr-agent-probe\"]\n            initialDelaySeconds: 5\n            periodSeconds: 30\n          #\n          # Notes: This is NOT a readiness probe for the agent itself.\n          #\n          # We (mis)use this readiness probe to propagate information\n          # back to the control plane about any failure on the node to\n          # activate the last updated configuration. Since success or\n          # failure is reflected by whether the agent's pod on the node\n          # is marked Ready, any error in configuration should now be a\n          # watchable condition, at least indirectly. One can get more\n          # details about the specifics of any configuration errors by\n          # watching the readiness of the agent's and fetching its log\n          # messages if it ever becomes not ready.\n          #\n          readinessProbe:\n            exec:\n              command: [\"/bin/cri-resmgr-agent-probe\", \"-query\", \"config-status\"]\n            initialDelaySeconds: 5\n            periodSeconds: 30\n      volumes:\n      - name: resmgrsockets\n        hostPath:\n          path: /var/run/cri-resmgr\n"
  },
  {
    "path": "cmd/cri-resmgr-agent/main.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"flag\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/agent\"\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/version\"\n)\n\nfunc main() {\n\t// Disable buffering and make sure that all messages have been emitted at\n\t// program exit\n\tlog.Flush()\n\tdefer log.Flush()\n\n\tflag.Parse()\n\n\ta, err := agent.NewResourceManagerAgent()\n\tif err != nil {\n\t\tlog.Fatal(\"failed to create resource manager agent instance: %v\", err)\n\t}\n\n\tlog.Info(\"cri-resmgr agent (version %s, build %s) starting...\", version.Version, version.Build)\n\n\tif err := a.Run(); err != nil {\n\t\tlog.Fatal(\"%v\", err)\n\t}\n}\n"
  },
  {
    "path": "cmd/cri-resmgr-agent-probe/main.go",
    "content": "/*\nCopyright 2020 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"google.golang.org/grpc\"\n\n\tagent_v1 \"github.com/intel/cri-resource-manager/pkg/agent/api/v1\"\n\tv1 \"github.com/intel/cri-resource-manager/pkg/agent/api/v1\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/sockets\"\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nfunc main() {\n\tsocket := flag.String(\"agent-socket\", sockets.ResourceManagerAgent, \"Unix domain socket where agent is serving\")\n\tquery := flag.String(\"query\", \"\", fmt.Sprintf(\"query to send, use %q to query status of last config push to resmgr\", v1.ConfigStatus))\n\n\t// Disable logger buffering and make sure that everything has been flushed\n\t// when program exits\n\tlog.Flush()\n\tdefer log.Flush()\n\n\tflag.Parse()\n\n\t// Try to connect to agent\n\tdialOpts := []grpc.DialOption{\n\t\tgrpc.WithInsecure(),\n\t\tgrpc.WithDialer(func(sock string, timeout time.Duration) (net.Conn, error) {\n\t\t\treturn net.Dial(\"unix\", sock)\n\t\t}),\n\t}\n\tconn, err := grpc.Dial(*socket, dialOpts...)\n\tif err != nil {\n\t\tlog.Fatal(\"failed to connect to agent: %v\", err)\n\t}\n\tcli := agent_v1.NewAgentClient(conn)\n\n\t// Do health check\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\tdefer cancel()\n\n\trpl, err := cli.HealthCheck(ctx, &agent_v1.HealthCheckRequest{\n\t\tQuery: *query,\n\t})\n\tif err != nil {\n\t\tlog.Fatal(\"%v\", err)\n\t}\n\tif rpl.Error != \"\" {\n\t\tlog.Fatal(\"health check negative: %s\", rpl.Error)\n\t}\n\tlog.Info(\"Health check OK\")\n}\n"
  },
  {
    "path": "cmd/cri-resmgr-webhook/Dockerfile",
    "content": "ARG GO_VERSION=1.24\n\nFROM golang:${GO_VERSION}-bullseye as builder\n\nARG GOLICENSES_VERSION\n\nWORKDIR /go/build\n\n# Fetch go dependencies in a separate layer for caching\nRUN go install github.com/google/go-licenses@${GOLICENSES_VERSION}\nCOPY go.mod go.sum ./\nCOPY pkg/topology/ pkg/topology/\nRUN go mod download -x\n\n# Build webhook, fully statically linked binary\nCOPY . .\n\nRUN CGO_ENABLED=0 make build-static BUILD_DIRS=cri-resmgr-webhook && \\\n    install -D /go/build/bin/* -t /install_root/bin\n\n# Save licenses\nRUN make install-licenses BUILD_DIRS=cri-resmgr-webhook DESTDIR=/install_root\n\nFROM scratch as final\n\nUSER 65534:65534\n\nCOPY --from=builder /install_root /\n\nENTRYPOINT [\"/bin/cri-resmgr-webhook\"]\n\n"
  },
  {
    "path": "cmd/cri-resmgr-webhook/handlers.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"net/http\"\n\n\tadmissionv1 \"k8s.io/api/admission/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\t\"k8s.io/apimachinery/pkg/runtime/serializer\"\n\tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n\t\"sigs.k8s.io/yaml\"\n)\n\ntype jsonPatch struct {\n\tOp    string      `json:\"op\"`\n\tPath  string      `json:\"path\"`\n\tValue interface{} `json:\"value\"`\n}\n\ntype podResourceRequirements struct {\n\tInitContainers map[string]corev1.ResourceRequirements `json:\"initContainers\"`\n\tContainers     map[string]corev1.ResourceRequirements `json:\"containers\"`\n}\n\nvar scheme = runtime.NewScheme()\nvar codecs = serializer.NewCodecFactory(scheme)\n\n// Module inatialization\nfunc init() {\n\tutilruntime.Must(corev1.AddToScheme(scheme))\n\tutilruntime.Must(admissionv1.AddToScheme(scheme))\n}\n\n// Helper for creating an AdmissionResponse with an error\nfunc errResponse(err error) *admissionv1.AdmissionResponse {\n\treturn &admissionv1.AdmissionResponse{\n\t\tResult: &metav1.Status{\n\t\t\tMessage: err.Error(),\n\t\t},\n\t}\n}\n\n// Dump req/rsp in human-readable form\nfunc stringify(r interface{}) string {\n\tout, err := yaml.Marshal(r)\n\tif err != nil {\n\t\treturn fmt.Sprintf(\"!!!!!\\nUnable to stringify %T: %v\\n!!!!!\", r, err)\n\t}\n\treturn string(out)\n}\n\n// Handle HTTP requests\nfunc handle(w http.ResponseWriter, r *http.Request) {\n\tvar body []byte\n\tif r.Body != nil {\n\t\tif data, err := io.ReadAll(r.Body); err == nil {\n\t\t\tbody = data\n\t\t}\n\t}\n\n\t// Check Content-Type\n\tcontentType := r.Header.Get(\"Content-Type\")\n\tif contentType != \"application/json\" {\n\t\tlog.Printf(\"ERROR: incorrect Content-Type (received %s, expect application/json\", contentType)\n\t\treturn\n\t}\n\n\t// Deserialize AdmissionReview request and create an AdmissionReview response\n\tarReq := admissionv1.AdmissionReview{}\n\tarRsp := admissionv1.AdmissionReview{}\n\tdeserializer := codecs.UniversalDeserializer()\n\tif _, _, err := deserializer.Decode(body, nil, &arReq); err != nil {\n\t\tlog.Printf(\"ERROR: deserializing admission request failed: %v\", err)\n\t\tarRsp.Response = errResponse(err)\n\t} else if arReq.Request == nil {\n\t\tlog.Printf(\"REQUEST empty\")\n\t\tarRsp.Response = errResponse(errors.New(\"Empty request\"))\n\t} else {\n\t\tlog.Printf(\"REQUEST:\\n%s\", stringify(&arReq))\n\t\tif arReq.Request.Resource.Group != \"\" || arReq.Request.Resource.Version != \"v1\" {\n\t\t\tarRsp.Response = errResponse(fmt.Errorf(\"Unexpected resource group/version '%s/%s'\", arReq.Request.Resource.Group, arReq.Request.Resource.Version))\n\t\t} else {\n\t\t\tres := arReq.Request.Resource.Resource\n\t\t\tswitch res {\n\t\t\tcase \"pods\":\n\t\t\t\tarRsp.Kind = \"AdmissionReview\"\n\t\t\t\tarRsp.APIVersion = \"admission.k8s.io/v1\"\n\t\t\t\tarRsp.Response = mutatePodObject(&arReq.Request.Object)\n\t\t\tdefault:\n\t\t\t\tarRsp.Response = errResponse(fmt.Errorf(\"Unexpected resource %s\", arReq.Request.Resource))\n\t\t\t}\n\t\t}\n\t\t// Use the same UID in response that was used in the request\n\t\tarRsp.Response.UID = arReq.Request.UID\n\t}\n\n\tlog.Printf(\"RESPONSE:\\n%s\", stringify(arRsp.Response))\n\n\trespBytes, err := json.Marshal(arRsp)\n\tif err != nil {\n\t\tlog.Printf(\"ERROR: json marshal failed: %v\", err)\n\t}\n\tif _, err := w.Write(respBytes); err != nil {\n\t\tlog.Printf(\"ERROR: failed to write HTTP response: %v\", err)\n\t}\n}\n\n// Handle AdmissionReview requests for Pod objects\nfunc mutatePodObject(rawObj *runtime.RawExtension) *admissionv1.AdmissionResponse {\n\tpod := corev1.Pod{}\n\tdeserializer := codecs.UniversalDeserializer()\n\tif _, _, err := deserializer.Decode(rawObj.Raw, nil, &pod); err != nil {\n\t\tlog.Printf(\"ERROR: failed to deserialize Pod object: %v\", err)\n\t\treturn errResponse(err)\n\t}\n\n\treviewResponse := admissionv1.AdmissionResponse{}\n\treviewResponse.Allowed = true\n\n\tpatches := []jsonPatch{}\n\t// Add a patch to add an empty annotations object if no annotations are found\n\tif pod.ObjectMeta.Annotations == nil {\n\t\tpatches = append(patches, jsonPatch{Op: \"add\", Path: \"/metadata/annotations\", Value: map[string]string{}})\n\t}\n\n\tpatch, err := patchResourceAnnotation(&pod)\n\tif err != nil {\n\t\treturn errResponse(err)\n\t}\n\tpatches = append(patches, patch)\n\n\treviewResponse.Patch, err = json.Marshal(patches)\n\tif err != nil {\n\t\tlog.Printf(\"ERROR: failed to marshal Pod patch: %v\", err)\n\t\treturn errResponse(err)\n\t}\n\tpatchType := admissionv1.PatchTypeJSONPatch\n\treviewResponse.PatchType = &patchType\n\n\treturn &reviewResponse\n}\n\n// Create a Pod (JSON) patch adding resource annotation\nfunc patchResourceAnnotation(pod *corev1.Pod) (jsonPatch, error) {\n\tpatch := jsonPatch{Op: \"add\", Path: \"/metadata/annotations/intel.com~1resources\"}\n\n\t// Create annotation that includes all resources of all (init)containers\n\tresourceAnnotation := podResourceRequirements{InitContainers: map[string]corev1.ResourceRequirements{},\n\t\tContainers: map[string]corev1.ResourceRequirements{}}\n\tfor _, container := range pod.Spec.Containers {\n\t\tresourceAnnotation.Containers[container.Name] = container.Resources\n\t}\n\tfor _, container := range pod.Spec.InitContainers {\n\t\tresourceAnnotation.InitContainers[container.Name] = container.Resources\n\t}\n\tresourceAnnotationBytes, err := json.Marshal(resourceAnnotation)\n\tif err != nil {\n\t\tlog.Printf(\"ERROR: failed to marshal 'intel.com/resources' annotations: %v\", err)\n\t\treturn patch, err\n\t}\n\n\t// Patch Pod annotations to include the \"resources\" annotation\n\tpatch.Value = string(resourceAnnotationBytes)\n\n\treturn patch, nil\n}\n"
  },
  {
    "path": "cmd/cri-resmgr-webhook/main.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"log\"\n)\n\n// Parse command line\nfunc parseArgs() args {\n\targs := args{}\n\n\tflag.IntVar(&args.port, \"port\", 443, \"Port on which to listen for connections\")\n\tflag.StringVar(&args.certFile, \"cert-file\", \"\", \"x509 certificate used for authenticating connections\")\n\tflag.StringVar(&args.keyFile, \"key-file\", \"\", \"Private x509 key matching --cert-file\")\n\n\tflag.Parse()\n\n\treturn args\n}\n\nfunc main() {\n\targs := parseArgs()\n\n\tif err := Run(args); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n}\n"
  },
  {
    "path": "cmd/cri-resmgr-webhook/mutating-webhook-config.yaml",
    "content": "apiVersion: admissionregistration.k8s.io/v1\nkind: MutatingWebhookConfiguration\nmetadata:\n  name: cri-resmgr\nwebhooks:\n- name: cri-resmgr.intel.com\n  sideEffects: None\n  admissionReviewVersions: [\"v1\"]\n  rules:\n  - apiGroups:\n    - \"\"\n    apiVersions:\n    - v1\n    operations:\n    - CREATE\n    - UPDATE\n    resources:\n    - pods\n  clientConfig:\n    service:\n      namespace: cri-resmgr\n      name: cri-resmgr-webhook\n    caBundle: CA_BUNDLE_PLACEHOLDER\n"
  },
  {
    "path": "cmd/cri-resmgr-webhook/webhook-deployment.yaml",
    "content": "apiVersion: v1\nkind: Namespace\nmetadata:\n  name: cri-resmgr\n  labels:\n    name: cri-resmgr\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: cri-resmgr-webhook\n  namespace: cri-resmgr\n  labels:\n    app: cri-resmgr-webhook\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: cri-resmgr-webhook\n  template:\n    metadata:\n      labels:\n        app: cri-resmgr-webhook\n    spec:\n      containers:\n      - name: cri-resmgr-webhook\n        image: IMAGE_PLACEHOLDER\n        # Convenience pull policy for development\n        imagePullPolicy: Always\n        # Mount the tls cert/key in the default location\n        volumeMounts:\n        - name: certs\n          mountPath: /etc/cri-resmgr-webhook/certs.d/\n          readOnly: true\n        args:\n         - \"-cert-file=/etc/cri-resmgr-webhook/certs.d/svc.crt\"\n         - \"-key-file=/etc/cri-resmgr-webhook/certs.d/svc.key\"\n         - \"-port=8443\"\n        securityContext:\n          allowPrivilegeEscalation: false\n          capabilities:\n            drop: [\"ALL\"]\n          readOnlyRootFilesystem: true\n          runAsNonRoot: true\n        resources:\n          limits:\n            cpu: 1\n            memory: 256Mi\n        livenessProbe:\n          httpGet:\n            scheme: HTTPS\n            port: 8443\n            httpHeaders:\n            - name: \"Content-Type\"\n              value: \"application/json\"\n          initialDelaySeconds: 5\n          periodSeconds: 30\n\n      nodeSelector:\n        node-role.kubernetes.io/control-plane: \"\"\n      tolerations:\n        - key: \"node-role.kubernetes.io/control-plane\"\n          operator: \"Equal\"\n          value: \"\"\n          effect: \"NoSchedule\"\n      volumes:\n      # This example deployment uses k8s secrests to store TLS secrets\n      # You need to manually generate the cert/key pair, and, the accompanying secret\n      # Expected filenames are \"svc.crt\" and \"svc.key\"\n      - name: certs\n        secret:\n          secretName: cri-resmgr-webhook-secret\n---\napiVersion: v1\nkind: Service\nmetadata:\n  name: cri-resmgr-webhook\n  namespace: cri-resmgr\nspec:\n  selector:\n    app: cri-resmgr-webhook\n  ports:\n  - port: 443\n    targetPort: 8443\n    protocol: TCP\n"
  },
  {
    "path": "cmd/cri-resmgr-webhook/webhook.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"log\"\n\t\"net/http\"\n)\n\ntype args struct {\n\tport     int\n\tcertFile string\n\tkeyFile  string\n}\n\n// Load server certificate and private key\nfunc loadTLS(certFile, keyFile string) *tls.Config {\n\tcert, err := tls.LoadX509KeyPair(certFile, keyFile)\n\tif err != nil {\n\t\tlog.Fatalf(\"Failed to initialize TLS config: %v\", err)\n\t}\n\treturn &tls.Config{\n\t\tCertificates: []tls.Certificate{cert},\n\t}\n}\n\n// Run is the main entry point for the webhook server\nfunc Run(args args) error {\n\t// Attach handlers\n\thttp.HandleFunc(\"/\", handle)\n\n\t// Create and run HTTP server\n\tserver := &http.Server{\n\t\tAddr:      fmt.Sprintf(\":%d\", args.port),\n\t\tTLSConfig: loadTLS(args.certFile, args.keyFile),\n\t}\n\tlog.Printf(\"Listening on port %d\", args.port)\n\treturn server.ListenAndServeTLS(\"\", \"\")\n}\n"
  },
  {
    "path": "demo/blockio/bb-scanner.yaml",
    "content": "# bb-scanner continuously calculates checksums of files found\n# under /scan. Output reveals added, deleted, renamed and modified\n# files together with timestamps.\n#\n# bb-scanner is configured as a low-priority activity:\n# 1. CPU usage is limited to 10 %.\n# 2. Disk/SSD bandwidth is limited by SlowReader configuration.\n#\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: bb-scanner\n  labels:\n    app: bb-scanner\nspec:\n  selector:\n    matchLabels:\n      app: bb-scanner\n  template:\n    metadata:\n      name: bb-scanner\n      labels:\n        app: bb-scanner\n      annotations:\n        blockioclass.cri-resource-manager.intel.com/pod: SlowReader\n    spec:\n      terminationGracePeriodSeconds: 1\n      containers:\n      - image: busybox\n        command:\n          - sh\n          - -c\n          - while true; do\n              find /scan -type f -print0 | xargs -0 md5sum | sort > curr.md5;\n              date +%s >> /output/diffs.md5;\n              diff -U1 prev.md5 curr.md5 >> /output/diffs.md5;\n              cp curr.md5 /output/files.md5;\n              mv curr.md5 prev.md5;\n            done\n        imagePullPolicy: IfNotPresent\n        name: busybox\n        resources:\n          limits:\n            cpu: 100m\n        volumeMounts:\n          - mountPath: /scan/usr-bin\n            name: usr-bin\n            readOnly: true\n          - mountPath: /scan/usr-lib\n            name: usr-lib\n            readOnly: true\n          - mountPath: /output\n            name: output\n            readOnly: false\n      volumes:\n        - name: usr-bin\n          hostPath:\n            path: /usr/bin\n            type: DirectoryOrCreate\n        - name: usr-lib\n          hostPath:\n            path: /usr/lib\n            type: DirectoryOrCreate\n        - name: output\n          hostPath:\n            path: /var/cache/bb-scanner\n            type: DirectoryOrCreate\n      restartPolicy: Always\n"
  },
  {
    "path": "demo/blockio/cri-resmgr-config.default.yaml",
    "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.default\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: none\n  logger: |+\n    Debug: blockio,cgroupblkio\n  blockio: |+\n    Classes:\n      SlowReader:\n        - Devices:\n            - /dev/vda\n          ThrottleReadBps: 512k\n"
  },
  {
    "path": "demo/blockio/run.sh",
    "content": "#!/bin/bash\n\nDEMO_TITLE=\"CRI Resource Manager: Block I/O Demo\"\n\nPV='pv -qL'\n\nSCRIPT_DIR=\"$(dirname \"${BASH_SOURCE[0]}\")\"\nLIB_DIR=$SCRIPT_DIR/../lib\nBIN_DIR=${bindir-$(realpath \"$SCRIPT_DIR/../../bin\")}\nOUTPUT_DIR=${outdir-$SCRIPT_DIR/output}\nCOMMAND_OUTPUT_DIR=$OUTPUT_DIR/commands\n\n# shellcheck disable=SC1091\n# shellcheck source=../lib/command.bash\nsource \"$LIB_DIR/command.bash\"\n# shellcheck disable=SC1091\n# shellcheck source=../lib/host.bash\nsource \"$LIB_DIR/host.bash\"\n# shellcheck disable=SC1091\n# shellcheck source=../lib/vm.bash\nsource \"$LIB_DIR/vm.bash\"\n\nusage() {\n    echo \"$DEMO_TITLE\"\n    echo \"Usage: [VAR=VALUE] ./run.sh MODE\"\n    echo \"  MODE:     \\\"play\\\" plays the demo.\"\n    echo \"            \\\"record\\\" plays and records the demo.\"\n    echo \"            \\\"test\\\" runs fast, reports pass or fail.\"\n    echo \"  VARs:\"\n    echo \"    vm:      govm virtual machine name.\"\n    echo \"             The default is \\\"crirm-demo-blockio\\\".\"\n    echo \"    speed:   Demo play speed.\"\n    echo \"             The default is 10 (keypresses per second).\"\n    echo \"    cleanup: 0: leave VM running. (\\\"play\\\" mode default)\"\n    echo \"             1: delete VM (\\\"test\\\" mode default)\"\n    echo \"             2: stop VM, but do not delete it.\"\n    echo \"    outdir:  Save output under given directory.\"\n    echo \"             The default is \\\"${SCRIPT_DIR}/output\\\".\"\n    echo \"    binsrc:  Where to get cri-resmgr to the VM.\"\n    echo \"             \\\"github\\\": go get and build in VM (\\\"play\\\" mode default).\"\n    echo \"             \\\"local\\\": copy from source tree bin/ (\\\"test\\\" mode default)\"\n    echo \"                      (set bindir=/path/to/cri-resmgr* to override bin/)\"\n}\n\nerror() {\n    (echo \"\"; echo \"error: $1\" ) >&2\n    exit 1\n}\n\nout() {\n    if [ -n \"$PV\" ]; then\n        speed=${speed-10}\n        echo \"$1\" | $PV \"$speed\"\n    else\n        echo \"$1\"\n    fi\n    echo \"\"\n}\n\nrecord() {\n    clear\n    out \"Recording this screencast...\"\n    host-command \"asciinema rec -t \\\"$DEMO_TITLE\\\" crirm-demo-blockio.cast -c \\\"./run.sh play\\\"\"\n}\n\nscreen-create-vm() {\n    speed=60 out \"### Running the demo in VM \\\"$vm\\\".\"\n    host-create-vm \"$vm\"\n    vm-networking\n    if [ -z \"$VM_IP\" ]; then\n        error \"creating VM failed\"\n    fi\n}\n\nscreen-install-k8s() {\n    speed=60 out \"### Installing Kubernetes to the VM.\"\n    vm-install-cri\n    vm-install-k8s\n}\n\nscreen-install-cri-resmgr() {\n    speed=60 out \"### Installing CRI Resource Manager to VM.\"\n    vm-install-cri-resmgr\n}\n\nscreen-launch-cri-resmgr() {\n    policy=${policy-none}\n    speed=60 out \"### Launching cri-resmgr.\"\n    vm-command \"(echo \\\"policy:\\\"; echo \\\"  Active: $policy\\\") > cri-resmgr.fallback.cfg\"\n    vm-command \"cri-resmgr -relay-socket /var/run/cri-resmgr/cri-resmgr.sock -runtime-socket /var/run/containerd/containerd.sock -fallback-config cri-resmgr.fallback.cfg >cri-resmgr.output.txt 2>&1 &\"\n}\n\nscreen-create-singlenode-cluster() {\n    speed=60 out \"### Setting up single-node Kubernetes cluster.\"\n    speed=60 out \"### CRI Resource Manager + containerd will act as the container runtime.\"\n    vm-create-singlenode-cluster\n}\n\nscreen-launch-cri-resmgr-agent() {\n    speed=60 out \"### Launching cri-resmgr-agent.\"\n    speed=60 out \"### The agent will make cri-resmgr configurable with ConfigMaps.\"\n    vm-command \"NODE_NAME=\\$(hostname) cri-resmgr-agent -kubeconfig \\$HOME/.kube/config >cri-resmgr-agent.output.txt 2>&1 &\"\n}\n\nscreen-measure-io-speed() {\n    process=$1\n    measuretime=2\n    vm-command \"echo 3 > /proc/sys/vm/drop_caches\"\n    out \"### Measuring $process read speed -- twice.\"\n    cmd=\"pid=\\$(ps -A | awk \\\"/$process/{print \\\\\\$1}\\\"); [ -n \\\"\\$pid\\\" ] && { echo \\$(grep read_bytes /proc/\\$pid/io; sleep $measuretime; grep read_bytes /proc/\\$pid/io) | awk \\\"{print \\\\\\\"$process read speed: \\\\\\\"(\\\\\\$4-\\\\\\$2)/$measuretime/1024\\\\\\\" kBps\\\\\\\"}\\\"; }\"\n    speed=360 outcolor=10 vm-command \"$cmd\"\n    sleep 1\n    speed=360 outcolor=10 vm-command \"$cmd\"\n}\n\ndemo-blockio() {\n    out \"### Let the show begin!\"\n    out \"### Configuring cri-resmgr: introduce a SlowReader block I/O class.\"\n    host-command \"scp cri-resmgr-config.default.yaml $VM_SSH_USER@$VM_IP:\"\n    vm-command \"cat cri-resmgr-config.default.yaml\"\n    out \"### Note: SlowReaders can read from each of the listed devices up to $(vm-command-q \"awk '/ThrottleRead/{print \\$2}' < cri-resmgr-config.default.yaml\")Bps.\"\n    vm-command \"kubectl apply -f cri-resmgr-config.default.yaml\"\n    out \"### Our test workload, bb-scanner, is annotated as a SlowReader.\"\n    host-command \"scp bb-scanner.yaml $VM_SSH_USER@$VM_IP:\"\n    vm-command \"grep -A1 annotations: bb-scanner.yaml\"\n    out \"### Flushing caches and deploying bb-scanner.\"\n    vm-command \"echo 3 > /proc/sys/vm/drop_caches\"\n    vm-command \"kubectl create -f bb-scanner.yaml\"\n\n    out \"### Now bb-scanner is running md5sum to all mounted directories, non-stop.\"\n    vm-wait-process --timeout 60 md5sum\n\n    screen-measure-io-speed md5sum\n\n    out \"### Reconfiguring cri-resmgr: set SlowReader read speed to 2 MBps.\"\n    out \"### This applies to all pods and containers in this block I/O class,\"\n    out \"### both new and already running, like our bb-scanner.\"\n    vm-command \"sed -i 's/ThrottleReadBps:.*/ThrottleReadBps: 2Mi/' cri-resmgr-config.default.yaml\"\n    vm-command \"cat cri-resmgr-config.default.yaml\"\n    vm-command \"kubectl apply -f cri-resmgr-config.default.yaml\"\n\n    # Give some time for new config to become effective and process\n    # I/O to accelerate.\n    sleep 2;\n\n    screen-measure-io-speed md5sum\n\n    out \"### Thanks for watching!\"\n    out \"### Cleaning up: deleting bb-scanner.\"\n    vm-command \"kubectl delete daemonset bb-scanner\"\n}\n\n# Validate parameters\nmode=$1\ndistro=${distro:=\"ubuntu-20.04\"}\ncri=${cri:=\"containerd\"}\nvm=${vm:=\"blockio-$distro-$cri\"}\necho \"vm is here: \\\"$vm\\\"\"\nhost-set-vm-config \"$vm\" \"$distro\" \"$cri\"\n\nif [ \"$mode\" == \"play\" ]; then\n    speed=${speed-10}\n    cleanup=${cleanup-0}\n    binsrc=${binsrc-github}\nelif [ \"$mode\" == \"test\" ]; then\n    PV=\n    cleanup=${cleanup-1}\n    binsrc=${binsrc-local}\nelif [ \"$mode\" == \"record\" ]; then\n    record\nelse\n    usage\n    error \"missing valid MODE\"\n    exit 1\nfi\n\n# Prepare for test/demo\nmkdir -p \"$OUTPUT_DIR\"\nmkdir -p \"$COMMAND_OUTPUT_DIR\"\nrm -f \"$COMMAND_OUTPUT_DIR\"/0*\n( echo x > \"$OUTPUT_DIR\"/x && rm -f \"$OUTPUT_DIR\"/x ) || {\n    error \"output directory outdir=\\\"$OUTPUT_DIR\\\" is not writable\"\n}\n\nif [ \"$binsrc\" == \"local\" ]; then\n    [ -f \"${BIN_DIR}/cri-resmgr\" ] || error \"missing \\\"${BIN_DIR}/cri-resmgr\\\"\"\n    [ -f \"${BIN_DIR}/cri-resmgr-agent\" ] || error \"missing \\\"${BIN_DIR}/cri-resmgr-agent\\\"\"\nfi\n\nif [ -z \"$VM_IP\" ] || [ -z \"$VM_SSH_USER\" ] || [ -z \"$VM_NAME\" ]; then\n    screen-create-vm\nfi\n\nif ! vm-command-q \"dpkg -l | grep -q kubelet\"; then\n    screen-install-k8s\nfi\n\nif ! vm-command-q \"[ -f /usr/bin/cri-resmgr ] || [ -f /usr/local/bin/cri-resmgr ]\"; then\n    screen-install-cri-resmgr\nfi\n\n# start cri-resmgr if not already running\nif ! vm-command-q \"pidof cri-resmgr\" >/dev/null; then\n    screen-launch-cri-resmgr\nfi\n\n# create kubernetes cluster or wait that it is online\nif vm-command-q \"[ ! -f /var/lib/kubelet/config.yaml ]\"; then\n    screen-create-singlenode-cluster\nelse\n    # wait for kube-apiserver to launch (may be down if the VM was just booted)\n    vm-wait-process kube-apiserver\nfi\n\n# start cri-resmgr-agent if not already running\nif ! vm-command-q \"pidof cri-resmgr-agent >/dev/null\"; then\n    screen-launch-cri-resmgr-agent\nfi\n\n# Run test/demo\ndemo-blockio\n\n# Cleanup\nif [ \"$cleanup\" == \"0\" ]; then\n    echo \"The VM, Kubernetes and cri-resmgr are left running. Next steps:\"\n    vm-print-usage\nelif [ \"$cleanup\" == \"1\" ]; then\n    host-stop-vm $vm\n    host-delete-vm $vm\nelif [ \"$cleanup\" == \"2\" ]; then\n    host-stop-vm $vm\nfi\n\n# Summarize results\nSUMMARY_FILE=\"$OUTPUT_DIR/summary.txt\"\necho -n \"\" > \"$SUMMARY_FILE\" || error \"cannot write summary to \\\"$SUMMARY_FILE\\\"\"\nfirst_speed=\"$(grep \"^md5sum read speed:\" \"$COMMAND_OUTPUT_DIR\"/0* | head -n 1 | awk '{print $4}')\"\nlast_speed=\"$(grep \"^md5sum read speed:\" \"$COMMAND_OUTPUT_DIR\"/0* | tail -n 1 | awk '{print $4}')\"\necho \"First md5sum read speed (512 kBps throttling): $first_speed kBps\" >> \"$SUMMARY_FILE\"\necho \"Last  md5sum read speed (2 MBps throttling): $last_speed kBps\" >> \"$SUMMARY_FILE\"\n# Declare verdict in test mode\nexit_status=0\nif [ \"$mode\" == \"test\" ]; then\n    min_first=100 max_first=600 min_last=1500 max_last=2500\n    [[ \"$first_speed\" -gt \"$min_first\" ]] || exit_status=1\n    [[ \"$first_speed\" -lt \"$max_first\" ]] || exit_status=1\n    [[ \"$last_speed\" -gt \"$min_last\" ]] || exit_status=1\n    [[ \"$last_speed\" -lt \"$max_last\" ]] || exit_status=1\n    if [ \"$exit_status\" == \"1\" ]; then\n        echo \"Error: speeds outside acceptable ranges ($min_first..$max_first kBps and $min_last..$max_last kBps).\" >> \"$SUMMARY_FILE\"\n        echo \"Test verdict: FAIL\" >> \"$SUMMARY_FILE\"\n    else\n        echo \"Speeds within acceptable ranges ($min_first..$max_first kBps and $min_last..$max_last kBps).\" >> \"$SUMMARY_FILE\"\n        echo \"Test verdict: PASS\" >> \"$SUMMARY_FILE\"\n    fi\n    echo \"\"\n    cat \"$SUMMARY_FILE\"\nfi\nexit $exit_status\n"
  },
  {
    "path": "demo/lib/command.bash",
    "content": "# Hooks for displaying and logging how shell commands (local and\n# remote) are executed, and handling their output and exit status.\n#\n# Example in a Bash script, run-on-mytargethost function:\n#   command-start mytargethost \"ls -la\"\n#   ssh mytargethost $COMMAND 2>&1 | command-handle-output\n#   command-end ${PIPESTATUS[0]}\n#   [ \"$COMMAND_STATUS\" == \"0\" ] || command-error \"non-zero exit status\"\n#\n# command-start and command-end set environment variables:\n# COMMAND, COMMAND_STATUS, COMMAND_OUTPUT\n\nexport LC_NUMERIC=C\n\n# These exports force ssh-* to fail instead of prompting for a passphrase.\nexport DISPLAY=bogus-none\nexport SSH_ASKPASS=/bin/false\nSSH_KEY=\"${HOME}/.ssh/id_rsa\"\nSSH_OPTS=\"-o StrictHostKeyChecking=No\"\nSSH=\"ssh $SSH_OPTS\"\nSCP=\"scp $SSH_OPTS\"\n\nepochrealtime() {\n    [ -n \"$EPOCHREALTIME\" ] && echo \"$EPOCHREALTIME\" || echo \"$SECONDS\"\n}\n\nCOMMAND_COUNTER=0\ncommand_init_time=$(epochrealtime)\n\ncommand-start() {\n    # example: command-start vm prompt \"mkdir $MYDIR\"\n    COMMAND_TARGET=\"$1\"\n    COMMAND_PROMPT=\"$2\"\n    COMMAND=\"$3\"\n    COMMAND_STATUS=\"\"\n    COMMAND_OUTPUT=\"\"\n    COMMAND_COUNTER=$(( COMMAND_COUNTER + 1 ))\n    local command_start_time=$(epochrealtime)\n    local time_since_start=$(echo \"$command_start_time - $command_init_time\" | bc)\n    COMMAND_OUT_FILE=\"$COMMAND_OUTPUT_DIR/$(printf %04g $COMMAND_COUNTER)-$COMMAND_TARGET\"\n    echo \"# start time: $time_since_start\" > \"$COMMAND_OUT_FILE\" || {\n        echo \"cannot write command output to file \\\"$COMMAND_OUT_FILE\\\"\"\n        exit 1\n    }\n    echo \"# command: $COMMAND\" >> \"$COMMAND_OUT_FILE\"\n    echo -e -n \"${COMMAND_PROMPT}\"\n    if [ -n \"$PV\" ]; then\n        echo \"$COMMAND\" | $PV $speed\n    else\n        echo \"$COMMAND\"\n    fi\n    if [ -n \"$outcolor\" ]; then\n        COMMAND_OUTSTART=\"\\e[38;5;${outcolor}m\"\n        COMMAND_OUTEND=\"\\e[0m\"\n    else\n        COMMAND_OUTSTART=\"\"\n        COMMAND_OUTEND=\"\"\n    fi\n}\n\ncommand-handle-output() {\n    # example: sh -c $command | command-handle-output\n    tee \"$COMMAND_OUT_FILE.tmp\" | ( echo -e -n \"$COMMAND_OUTSTART\"; cat; echo -e -n \"$COMMAND_OUTEND\" )\n    cat \"$COMMAND_OUT_FILE.tmp\" >> \"$COMMAND_OUT_FILE\"\n    if [ -n \"$PV\" ]; then\n        echo | $PV $speed\n    fi\n}\n\ncommand-runs-in-bg() {\n    echo \"(runs in background)\"\n    echo \"\"\n}\n\ncommand-end() {\n    # example: command-end EXIT_STATUS\n    COMMAND_STATUS=$1\n    local command_end_time=$(epochrealtime)\n    local time_since_start=$(echo \"$command_end_time - $command_init_time\" | bc)\n    ( echo \"# exit status: $COMMAND_STATUS\"; echo \"# end time: $time_since_start\" ) >> \"$COMMAND_OUT_FILE\"\n    COMMAND_OUTPUT=$(<\"$COMMAND_OUT_FILE.tmp\")\n    rm -f \"$COMMAND_OUT_FILE.tmp\"\n}\n\ncommand-error() { # script API\n    # Usage: command-error MESSAGE\n    #\n    # Print executed command, observed output, exit status and MESSAGE.\n    # Stop script execution.\n    ( echo \"command:     $COMMAND\";\n      echo \"output:      $COMMAND_OUTPUT\";\n      echo \"exit status: $COMMAND_STATUS\";\n      echo \"error:       $1\" ) >&2\n    command-exit-if-not-interactive\n}\n\ncommand-exit-if-not-interactive() {\n    if [ -z \"$INTERACTIVE_MODE\" ] || [ \"$INTERACTIVE_MODE\" == \"0\" ]; then\n        exit ${1:-1}\n    fi\n}\n\ncommand-debug-log() {\n    if [ \"$(type -t -- debug-log)\" = \"function\" ]; then\n        debug-log \"$@\"\n        return 0\n    else\n        if [ -n \"$OUTPUT_DIR\" ] && [ -d \"$OUTPUT_DIR\" ]; then\n            touch \"$OUTPUT_DIR\"/debug-log\n            echo \"$@\" >> \"$OUTPUT_DIR\"/debug-log\n            return 0\n        fi\n    fi\n    echo \"$@\" 1>&2\n}\n"
  },
  {
    "path": "demo/lib/distro.bash",
    "content": "# shellcheck disable=SC2120\nGO_URLDIR=https://golang.org/dl\nGO_VERSION=1.24.1\nGOLANG_URL=$GO_URLDIR/go$GO_VERSION.linux-amd64.tar.gz\nCRICTL_VERSION=${CRICTL_VERSION:-\"v1.25.0\"}\nMINIKUBE_VERSION=${MINIKUBE_VERSION:-v1.27.0}\n\n###########################################################################\n\n#\n# distro-agnostic interface\n#\n# To add a new distro implement distro-specific versions of these\n# functions. You can omit implementing those which already resolve\n# to an existing function which works for the new distro.\n#\n# To add a new API function, add an new briding resolution entry below.\n#\n\ndistro-image-url()          { distro-resolve \"$@\"; }\ndistro-ssh-user()           { distro-resolve \"$@\"; }\ndistro-pkg-type()           { distro-resolve \"$@\"; }\ndistro-install-repo-key()   { distro-resolve \"$@\"; }\ndistro-install-repo()       { distro-resolve \"$@\"; }\ndistro-refresh-pkg-db()     { distro-resolve \"$@\"; }\ndistro-install-pkg()        { distro-resolve \"$@\"; }\ndistro-install-pkg-local()  { distro-resolve \"$@\"; }\ndistro-remove-pkg()         { distro-resolve \"$@\"; }\ndistro-setup-proxies()      { distro-resolve \"$@\"; }\ndistro-setup-oneshot()      { distro-resolve \"$@\"; }\ndistro-install-utils()      { distro-resolve \"$@\"; }\ndistro-install-golang()     { distro-resolve \"$@\"; }\ndistro-install-runc()       { distro-resolve \"$@\"; }\ndistro-install-containerd() { distro-resolve \"$@\"; }\ndistro-config-containerd()  { distro-resolve \"$@\"; }\ndistro-restart-containerd() { distro-resolve \"$@\"; }\ndistro-install-crio()       { distro-resolve \"$@\"; }\ndistro-config-crio()        { distro-resolve \"$@\"; }\ndistro-restart-crio()       { distro-resolve \"$@\"; }\ndistro-install-crictl()     { distro-resolve \"$@\"; }\ndistro-install-cri-dockerd(){ distro-resolve \"$@\"; }\ndistro-install-minikube()   { distro-resolve \"$@\"; }\ndistro-install-k8s()        { distro-resolve \"$@\"; }\ndistro-install-kernel-dev() { distro-resolve \"$@\"; }\ndistro-k8s-cni()            { distro-resolve \"$@\"; }\ndistro-k8s-cni-subnet()     { distro-resolve \"$@\"; }\ndistro-set-kernel-cmdline() { distro-resolve \"$@\"; }\ndistro-govm-env()           { distro-resolve \"$@\"; }\ndistro-bootstrap-commands() { distro-resolve \"$@\"; }\ndistro-env-file-dir()       { distro-resolve \"$@\"; }\n\n###########################################################################\n\n# distro-specific function resolution\ndistro-resolve() {\n    local apifn=\"${FUNCNAME[1]}\" fn prefn postfn\n    # shellcheck disable=SC2086\n    {\n        fn=\"$(distro-resolve-fn $apifn)\"\n        prefn=\"$(distro-resolve-fn $apifn-pre)\"\n        postfn=\"$(distro-resolve-fn $apifn-post)\"\n        command-debug-log \"$VM_DISTRO/${FUNCNAME[1]}: pre: ${prefn:--}, fn: ${fn:--}, post: ${postfn:--}\"\n    }\n    [ -n \"$prefn\" ] && { $prefn \"$@\" || return $?; }\n    $fn \"$@\" || return $?\n    [ -n \"$postfn\" ] && { $postfn \"$@\" || return $?; }\n    return 0\n}\n\ndistro-resolve-fn() {\n    # We try resolving distro-agnostic implementations by looping through\n    # a list of candidate function names in decreasing order of precedence\n    # and returning the first one found. The candidate list has version-\n    # exact and unversioned distro-specific functions and a set fallbacks\n    # based on known distro, derivative, and package type relations.\n    #\n    # For normal functions the last fallback is 'distro-unresolved' which\n    # prints and returns an error. For pre- and post-functions there is no\n    # similar setup. IOW, unresolved normal distro functions fail while\n    # unresolved pre- and post-functions get ignored (in distro-resolve).\n    local apifn=\"$1\" candidates fn\n\n    case $apifn in\n        distro-*) apifn=\"${apifn#distro-}\";;\n        *) error \"internal error: can't resolve non-API function $apifn\";;\n    esac\n    candidates=\"${VM_DISTRO/./_}-$apifn ${VM_DISTRO%%-*}-$apifn\"\n    case $VM_DISTRO in\n        ubuntu*) candidates=\"$candidates debian-$apifn\";;\n        fedora*) candidates=\"$candidates rpm-$apifn\";;\n        *suse*)  candidates=\"$candidates rpm-$apifn\";;\n        sles*)   candidates=\"$candidates opensuse-$apifn rpm-$apifn\";;\n    esac\n    case $apifn in\n        *-pre|*-post) ;;\n        *) candidates=\"$candidates default-$apifn distro-unresolved\";;\n    esac\n    for fn in $candidates; do\n        if [ \"$(type -t -- \"$fn\")\" = \"function\" ]; then\n            echo \"$fn\"\n            return 0\n        fi\n    done\n}\n\n# distro-unresolved terminates failed API function resolution with an error.\ndistro-unresolved() {\n    local apifn=\"${FUNCNAME[2]}\"\n    command-error \"internal error: can't resolve \\\"$apifn\\\" for \\\"$VM_DISTRO\\\"\"\n    return 1\n}\n\n###########################################################################\n\n#\n# Ubuntu, Debian\n#\n\nubuntu-18_04-image-url() {\n    echo \"https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img\"\n}\n\nubuntu-20_04-image-url() {\n    echo \"https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img\"\n}\n\nubuntu-22_04-image-url() {\n    echo \"https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.img\"\n}\n\nubuntu-24_04-image-url() {\n    echo \"https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.img\"\n}\n\ndebian-11-image-url() {\n    echo \"https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2\"\n}\n\ndebian-12-image-url() {\n    echo \"https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2\"\n}\n\ndebian-sid-image-url() {\n    echo \"https://cloud.debian.org/images/cloud/sid/daily/latest/debian-sid-generic-amd64-daily.qcow2\"\n}\n\nubuntu-download-kernel() {\n    # Usage:\n    #   ubuntu-download-kernel list\n    #   ubuntu-download-kernel VERSION\n    #\n    # List or download Ubuntu kernel team kernels.\n    #\n    # Example:\n    #   ubuntu-download-kernel list | grep 5.9\n    #   ubuntu-download-kernel 5.9-rc8\n    #   vm-command \"dpkg -i kernels/linux*rc8*deb\"\n    #   vm-reboot\n    #   vm-command \"uname -a\"\n    local version=$1\n    [ -n \"$version\" ] ||\n        error \"missing kernel version to install\"\n    if [ \"$version\" == \"list\" ]; then\n        wget -q -O- https://kernel.ubuntu.com/~kernel-ppa/mainline/  | grep -E '^<tr>.*href=\"v[5-9]' | sed 's|^.*href=\"v\\([0-9][^\"]*\\)/\".*$|\\1|g'\n        return 0\n    fi\n    vm-command \"mkdir -p kernels; rm -f kernels/linux*$version*deb; for deb in \\$(wget -q -O- https://kernel.ubuntu.com/~kernel-ppa/mainline/v$version/ | awk -F'\\\"' '/amd64.*deb/{print \\$2}' | grep -v -E 'headers|lowlatency'); do ( cd kernels; wget -q https://kernel.ubuntu.com/~kernel-ppa/mainline/v$version/\\$deb ); done; echo; echo 'Downloaded kernel packages:'; du -h kernels/*.deb\" ||\n        command-error \"downloading kernel $version failed\"\n}\n\nubuntu-ssh-user() {\n    echo ubuntu\n}\n\ndebian-ssh-user() {\n    echo debian\n}\n\nubuntu-apparmor-disable-runc() {\n    vm-command \"[ -f /etc/apparmor.d/runc ] && ln -s /etc/apparmor.d/runc /etc/apparmor.d/disable/ && apparmor_parser -R /etc/apparmor.d/runc\"\n}\n\nubuntu-config-containerd() {\n    ubuntu-apparmor-disable-runc\n    default-config-containerd\n}\n\nubuntu-config-crio() {\n    ubuntu-apparmor-disable-runc\n    default-config-crio\n}\n\n\ndebian-pkg-type() {\n    echo deb\n}\n\ndebian-install-repo-key() {\n    local key\n    # apt-key needs gnupg2, that might not be available by default\n    vm-command \"command -v gpg >/dev/null 2>&1\" || {\n        vm-command \"apt-get update && apt-get install -y gnupg2\"\n    }\n    for key in \"$@\"; do\n        vm-command \"curl -L -s $key | apt-key add -\" ||\n            command-error \"failed to install repo key $key\"\n    done\n}\n\ndebian-install-repo() {\n    if [ $# = 1 ]; then\n        # shellcheck disable=SC2086,SC2048\n        set -- $*\n    fi\n    vm-command \"echo $* > /etc/apt/sources.list.d/$3-$4.list && apt-get update\" ||\n        command-error \"failed to install apt repository $*\"\n}\n\ndebian-refresh-pkg-db() {\n    vm-command \"apt-get update\" ||\n        command-error \"failed to refresh apt package DB\"\n}\n\ndebian-install-pkg() {\n    # dpkg configure may ask \"The default action is to keep your\n    # current version\", for instance when a test has added\n    # /etc/containerd/config.toml and then apt-get installs\n    # containerd. 'yes \"\"' will continue with the default answer (N:\n    # keep existing) in this case. Without 'yes' installation fails.\n\n    # Add apt-get option \"--reinstall\" if any environment variable\n    # reinstall_<pkg>=1\n    local pkg\n    local opts=\"\"\n    for pkg in \"$@\"; do\n        if [ \"$(eval echo \\$reinstall_$pkg)\" == \"1\" ]; then\n            opts=\"$opts --reinstall\"\n            break\n        fi\n    done\n    vm-command \"yes \\\"\\\" | DEBIAN_FRONTEND=noninteractive apt-get install $opts -y --allow-downgrades $*\" ||\n        command-error \"failed to install $*\"\n}\n\ndebian-remove-pkg() {\n    vm-command \"for pkg in $*; do dpkg -l \\$pkg >& /dev/null && apt remove -y --purge \\$pkg || :; done\" ||\n        command-error \"failed to remove package(s) $*\"\n}\n\ndebian-install-pkg-local() {\n    local force=\"\"\n    if [ \"$1\" == \"--force\" ]; then\n        force=\"--force-all\"\n        shift\n    fi\n    vm-command \"dpkg -i $force $*\" ||\n        command-error \"failed to install local package(s)\"\n}\n\ndebian-install-golang() {\n    debian-refresh-pkg-db\n    debian-install-pkg golang git-core\n}\n\ndebian-install-kernel-dev() {\n    distro-refresh-pkg-db\n    distro-install-pkg git-core build-essential linux-source bc kmod cpio flex libncurses5-dev libelf-dev libssl-dev dwarves bison\n    vm-command \"[ -d linux ] || git clone https://github.com/torvalds/linux\"\n    vm-command '[ -f linux/.config ] || cp -v /boot/config-$(uname -r) linux/.config'\n    echo \"Kernel ready for patching and configuring.\"\n    echo \"build:   cd linux && make bindeb-pkg\"\n    echo \"install: dpkg -i linux-*.deb\"\n}\n\ndebian-11-install-containerd-pre() {\n    debian-install-repo-key https://download.docker.com/linux/debian/gpg\n    debian-install-repo \"deb https://download.docker.com/linux/debian bullseye stable\"\n}\n\ndebian-11-install-containerd() {\n    vm-command-q \"[ -f /usr/bin/containerd ]\" || {\n        distro-install-pkg containerd.io\n    }\n}\n\ndebian-sid-config-containerd-post() {\n    vm-command \"sed -e 's|bin_dir = \\\"/usr/lib/cni\\\"|bin_dir = \\\"/opt/cni/bin\\\"|g' -i /etc/containerd/config.toml\"\n}\n\ndebian-install-cri-dockerd-pre() {\n    debian-refresh-pkg-db\n    debian-install-pkg docker.io conntrack\n    vm-command \"addgroup $(vm-ssh-user) docker\"\n    distro-install-golang\n}\n\ndebian-install-crio-pre() {\n    debian-refresh-pkg-db\n    debian-install-pkg libgpgme11 conmon runc containernetworking-plugins conntrack || true\n}\n\ndebian-install-k8s() {\n    local _k8s=$k8s\n    debian-refresh-pkg-db\n    debian-install-pkg gpg apt-transport-https curl\n\n    if [[ -z \"$k8s\" ]] || [[ \"$k8s\" == \"latest\" ]]; then\n        vm-command \"curl -s https://api.github.com/repos/kubernetes/kubernetes/releases/latest | grep tag_name | sed -e 's/.*v\\([0-9]\\+\\.[0-9]\\+\\).*/\\1/g'\"\n        _k8s=$COMMAND_OUTPUT\n    fi\n    echo \"installing Kubernetes v${_k8s}\"\n    vm-command \"curl -fsSL https://pkgs.k8s.io/core:/stable:/v${_k8s}/deb/Release.key -o /tmp/Release.key\" || \\\n        command-error \"failed to download Kubernetes v${_k8s} key\"\n\n    if vm-command \"command -v apt-key >/dev/null\"; then\n        vm-command \"sudo apt-key add /tmp/Release.key\"\n        vm-command \"echo 'deb https://pkgs.k8s.io/core:/stable:/v${_k8s}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && apt update\" || \\\n            command-error \"failed to add Kubernetes v${_k8s} repo\"\n    else\n        vm-command \"sudo gpg --dearmor --batch --yes -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg /tmp/Release.key\"\n        vm-command \"echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${_k8s}/deb/ /' > /etc/apt/sources.list.d/kubernetes.list && apt update\" || \\\n            command-error \"failed to add Kubernetes v${_k8s} repo\"\n    fi\n    debian-install-pkg \"kubeadm\" \"kubelet\" \"kubectl\"\n}\n\ndebian-set-kernel-cmdline() {\n    local e2e_defaults=\"$*\"\n    vm-command \"echo 'GRUB_CMDLINE_LINUX_DEFAULT=\\\"\\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\\\"' > /etc/default/grub.d/60-e2e-defaults.cfg\" || {\n        command-error \"writing new command line parameters failed\"\n    }\n    vm-command \"update-grub\" || {\n        command-error \"updating grub failed\"\n    }\n}\n\ndebian-env-file-dir() {\n    echo \"/etc/default\"\n}\n\ndebian-sid-govm-env() {\n    echo \"DISABLE_VGA=N\"\n}\n\n###########################################################################\n\n#\n# Generic Fedora\n#\n\nYUM_INSTALL=\"yum install --disableplugin=fastestmirror -y\"\nYUM_REMOVE=\"yum remove --disableplugin=fastestmirror -y\"\n\nfedora-image-url() {\n    fedora-40-image-url\n}\n\nfedora-40-image-url() {\n    echo \"https://mirrors.xtom.de/fedora/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2\"\n}\n\nfedora-ssh-user() {\n    echo fedora\n}\n\nfedora-install-utils() {\n    distro-install-pkg /usr/bin/pidof\n}\n\nfedora-install-repo() {\n    distro-install-pkg dnf-plugins-core\n    vm-command \"dnf config-manager --add-repo $*\" ||\n        command-error \"failed to install DNF repository $*\"\n}\n\nfedora-install-pkg() {\n    local pkg\n    local do_reinstall=0\n    for pkg in \"$@\"; do\n        if [ \"$(eval echo \\$reinstall_$pkg)\" == \"1\" ]; then\n            do_reinstall=1\n            break\n        fi\n    done\n    vm-command \"dnf install -y $*\" ||\n        command-error \"failed to install $*\"\n    # When requesting reinstallation, detect which packages were\n    # already installed and reinstall those.\n    # (Unlike apt and zypper, dnf offers no option for reinstalling\n    # existing and installing new packages on the same run.)\n    if [ \"$do_reinstall\" == \"1\" ]; then\n        local reinstall_pkgs\n        reinstall_pkgs=$(awk -F '[ -]' -v ORS=\" \" '/Package .* already installed/{print $2}' <<< \"$COMMAND_OUTPUT\")\n        if [ -n \"$reinstall_pkgs\" ]; then\n            vm-command \"dnf reinstall -y $reinstall_pkgs\"\n        fi\n    fi\n}\n\nfedora-remove-pkg() {\n    vm-command \"dnf remove -y $*\" ||\n        command-error \"failed to remove package(s) $*\"\n}\n\nfedora-install-pkg-local() {\n    local force=\"\"\n    if [ \"$1\" == \"--force\" ]; then\n        force=\"--nodeps --force\"\n        shift\n    fi\n    vm-command \"rpm -Uvh $force $*\" ||\n        command-error \"failed to install local package(s)\"\n}\n\nfedora-install-kernel-dev() {\n    fedora-install-pkg fedpkg fedora-packager rpmdevtools ncurses-devel pesign grubby git-core\n    vm-command \"(set -x -e\n      echo root >> /etc/pesign/users\n      echo $(vm-ssh-user) >> /etc/pesign/users\n      /usr/libexec/pesign/pesign-authorize\n      fedpkg clone -a kernel\n      cd kernel\n      git fetch\n      git switch ${VM_DISTRO/edora-/} # example: git switch f40 in fedora-40\n      sed -i 's/# define buildid .local/%define buildid .e2e/g' kernel.spec\n    )\" || {\n        echo \"installing kernel development environment failed\"\n        return 1\n    }\n    echo \"Kernel ready for patching and configuring.\"\n    echo \"build:   cd kernel && dnf builddep -y kernel.spec && fedpkg local\"\n    echo \"install: cd kernel/x86_64 && dnf install -y --nogpgcheck kernel-{core-,modules-,}[5-9]*.e2e.fc*.x86_64.rpm\"\n}\n\nfedora-install-golang() {\n    fedora-install-pkg wget tar gzip git-core\n    from-tarball-install-golang\n}\n\nfedora-install-crio-version() {\n    distro-install-pkg runc conmon\n    vm-command \"ln -sf /usr/lib64/libdevmapper.so.1.02 /usr/lib64/libdevmapper.so.1.02.1\" || true\n\n    if [ -z \"$crio_src\" ]; then\n        vm-command \"dnf -y module enable cri-o:${crio_version:-$1}\"\n    fi\n}\n\nfedora-install-containernetworking-plugins() {\n    distro-install-pkg containernetworking-plugins\n    vm-command \"[ -x /opt/cni/bin/loopback ] || { mkdir -p /opt/cni/bin; mount --bind /usr/libexec/cni /opt/cni/bin; }\"\n    vm-command \"grep /usr/libexec/cni /etc/fstab || echo /usr/libexec/cni /opt/cni/bin none defaults,bind,nofail 0 0 >> /etc/fstab\"\n}\n\nfedora-install-cri-dockerd-pre() {\n    distro-install-pkg docker git-core conntrack\n    vm-command \"systemctl enable docker --now; usermod --append --groups docker $(vm-ssh-user)\"\n    distro-install-golang\n}\n\nfedora-install-crio-pre() {\n    fedora-install-crio-version 1.21\n    fedora-install-containernetworking-plugins\n}\n\nfedora-install-crio() {\n    if [ -n \"$crio_src\" ]; then\n        default-install-crio\n    else\n        distro-install-pkg cri-o\n        vm-command \"systemctl enable --now crio\" ||\n            command-error \"failed to enable cri-o\"\n    fi\n}\n\nfedora-install-containerd-pre() {\n    distro-install-repo https://download.docker.com/linux/fedora/docker-ce.repo\n    fedora-install-containernetworking-plugins\n}\n\nfedora-install-containerd-post() {\n    vm-command \"systemctl enable containerd\"\n}\n\nfedora-install-k8s() {\n    _k8s=$k8s\n    if [[ -z \"$_k8s\" ]] || [[ \"$_k8s\" == \"latest\" ]]; then\n        vm-command \"curl -s https://api.github.com/repos/kubernetes/kubernetes/releases/latest | grep tag_name | sed -e 's/.*v\\([0-9]\\+\\.[0-9]\\+\\).*/\\1/g'\"\n        _k8s=$COMMAND_OUTPUT\n    fi\n\n    local repo=\"/etc/yum.repos.d/kubernetes.repo\"\n\n    cat <<EOF |\n[kubernetes]\nname=Kubernetes\nbaseurl=https://pkgs.k8s.io/core:/stable:/v$_k8s/rpm/\nenabled=1\ngpgcheck=1\ngpgkey=https://pkgs.k8s.io/core:/stable:/v$_k8s/rpm/repodata/repomd.xml.key\nEOF\n      vm-pipe-to-file $repo\n\n    if [ -n \"$k8s\" ]; then\n        k8sverparam=\"-${k8s}-0\"\n    else\n        k8sverparam=\"\"\n    fi\n\n    distro-install-pkg iproute-tc kubelet$k8sverparam kubeadm$k8sverparam kubectl$k8sverparam\n    vm-command \"systemctl enable --now kubelet\" ||\n        command-error \"failed to enable kubelet\"\n}\n\nfedora-bootstrap-commands-post() {\n    cat <<EOF\nreboot_needed=0\nmkdir -p /etc/sudoers.d\necho 'Defaults !requiretty' > /etc/sudoers.d/10-norequiretty\n\nsetenforce 0\nsed -E -i 's/^SELINUX=.*$/SELINUX=permissive/' /etc/selinux/config\n\necho PATH='\\$PATH:/usr/local/bin:/usr/local/sbin' > /etc/profile.d/usr-local-path.sh\nEOF\n    if [[ \"${cgroups:-}\" != \"v2\" ]]; then\n        cat <<EOF\nif grep -q NAME=Fedora /etc/os-release; then\n    if ! grep -q systemd.unified_cgroup_hierarchy=0 /proc/cmdline; then\n        sudo grubby --update-kernel=ALL --args=\"systemd.unified_cgroup_hierarchy=0\"\n        reboot_needed=1\n    fi\nfi\nEOF\n    fi\n\n    # Using swapoff is not enough as we also need to disable the swap from systemd\n    # and then reboot the VM\n    cat <<EOF\nif swapon --show | grep -q partition; then\n    sed -E -i '/^\\\\/.*[[:space:]]swap[[:space:]].*\\$/d' /etc/fstab\n    systemctl --type swap\n    for swp in \\`systemctl --type swap | awk '/\\\\.swap/ { print \\$1 }'\\`; do systemctl stop \"\\$swp\"; systemctl mask \"\\$swp\"; done\n    swapoff --all\n    reboot_needed=1\nfi\nEOF\n\n    cat <<EOF\nif [ \"\\$reboot_needed\" == \"1\" ]; then\n   shutdown -r now\nfi\nEOF\n}\n\nfedora-set-kernel-cmdline() {\n    local e2e_defaults=\"$*\"\n    vm-command \"mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:fedora-set-kernel-cmdline/d' /etc/default/grub\"\n    vm-command \"echo 'GRUB_CMDLINE_LINUX_DEFAULT=\\\"\\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\\\" # by e2e:fedora-set-kernel-cmdline' >> /etc/default/grub\" || {\n        command-error \"writing new command line parameters failed\"\n    }\n    vm-command \"grub2-mkconfig -o /boot/grub2/grub.cfg\" || {\n        command-error \"updating grub failed\"\n    }\n}\n\n###########################################################################\n\n#\n# OpenSUSE and SLES\n#\n\nZYPPER=\"zypper --non-interactive --no-gpg-checks\"\n\nsles-image-url() {\n    echo \"/DOWNLOAD-MANUALLY-TO-HOME/vms/images/SLES15-SP3-JeOS.x86_64-15.3-OpenStack-Cloud-GM.qcow2\"\n}\n\nsles-ssh-user() {\n    echo \"sles\"\n}\n\nsles-install-utils() {\n    local sles_registered=0\n    local sles_version=\"\"\n    vm-command \"SUSEConnect -s\" || {\n        command-error \"cannot run SUSEConnect\"\n    }\n    # Parse registration status and SLES version.\n    if [ \"$(jq '.[] | select(.identifier == \"SLES\") | .status' <<< $COMMAND_OUTPUT)\" == '\"Registered\"' ]; then\n        sles_registered=1\n    fi\n    sles_version=\"$(jq -r '.[] | select(.identifier == \"SLES\") | .version' <<< $COMMAND_OUTPUT)\"\n    if [ -z \"$sles_version\" ]; then\n        command-error \"cannot read SLES version information from SUSEConnect -s output\"\n    fi\n    # Try automatic registration if registration code is provided.\n    if [ \"$sles_registered\" == 0 ] && [ -n \"$VM_SLES_REGCODE\" ]; then\n            vm-command \"SUSEConnect -r $VM_SLES_REGCODE\" || {\n                echo \"ERROR:\"\n                echo \"ERROR: Registering to SUSE Customer Center failed.\"\n                echo \"ERROR: - Verify VM_SLES_REGCODE and try again.\"\n                echo \"ERROR: - Unset VM_SLES_REGCODE to skip registration (use unsupported repos).\"\n                echo \"ERROR:\"\n                exit 1\n            }\n            sles_registered=1\n    fi\n    # Add correct repo, depending on registration status.\n    if [ \"$sles_registered\" == 0 ]; then\n        echo \"WARNING:\"\n        echo \"WARNING: Unregistered SUSE Linux Enterprise Server.\"\n        echo \"WARNING: VM_SLES_REGCODE is not set, automatic registration skipped.\"\n        echo \"WARNING: Fallback to use OpenSUSE OSS repository.\"\n        echo \"WARNING:\"\n        sleep \"${warning_delay:-0}\"\n        vm-command-q \"$ZYPPER lr openSUSE-Oss >/dev/null\" || {\n            distro-install-repo \"http://download.opensuse.org/distribution/leap/${sles_version}/repo/oss/\" openSUSE-Oss\n        }\n    else\n        vm-command-q \"$ZYPPER lr | grep -q SUSE-PackageHub\" || {\n            vm-command \"SUSEConnect -p PackageHub/${sles_version}/x86_64\"\n        }\n    fi\n    distro-install-pkg sysvinit-tools psmisc\n}\n\nopensuse-image-url() {\n    opensuse-15_6-image-url\n}\n\nopensuse-15_6-image-url() {\n    echo \"https://download.opensuse.org/pub/opensuse/distribution/leap/15.6/appliances/openSUSE-Leap-15.6-Minimal-VM.x86_64-Cloud.qcow2\"\n}\n\nopensuse-tumbleweed-image-url() {\n    echo \"https://ftp.uni-erlangen.de/opensuse/tumbleweed/appliances/openSUSE-MicroOS.x86_64-ContainerHost-OpenStack-Cloud.qcow2\"\n}\n\nopensuse-install-utils() {\n    distro-install-pkg psmisc sysvinit-tools\n}\n\nopensuse-ssh-user() {\n    echo \"opensuse\"\n}\n\nopensuse-pkg-type() {\n    echo \"rpm\"\n}\n\nopensuse-set-kernel-cmdline() {\n    local e2e_defaults=\"$*\"\n    vm-command \"mkdir -p /etc/default; touch /etc/default/grub; sed -i '/e2e:opensuse-set-kernel-cmdline/d' /etc/default/grub\"\n    vm-command \"echo 'GRUB_CMDLINE_LINUX_DEFAULT=\\\"\\${GRUB_CMDLINE_LINUX_DEFAULT} ${e2e_defaults}\\\" # by e2e:opensuse-set-kernel-cmdline' >> /etc/default/grub\" || {\n        command-error \"writing new command line parameters failed\"\n    }\n    vm-command \"grub2-mkconfig -o /boot/grub2/grub.cfg\" || {\n        command-error \"updating grub failed\"\n    }\n}\n\nopensuse-setup-oneshot() {\n    # Remove bad version of containerd if it is already installed,\n    # otherwise valid version of the package will not be installed.\n    vm-command \"rpm -q containerd && ( zypper info containerd | awk '/Repository/{print $3}' | grep -v Virtualization ) && echo Removing wrong containerd version && zypper --non-interactive rm containerd\"\n}\n\nopensuse-install-repo() {\n    opensuse-wait-for-zypper\n    vm-command \"$ZYPPER addrepo $* && $ZYPPER refresh\" ||\n        command-error \"failed to add zypper repository $*\"\n}\n\nopensuse-refresh-pkg-db() {\n    opensuse-wait-for-zypper\n    vm-command \"$ZYPPER refresh\" ||\n        command-error \"failed to refresh zypper package DB\"\n}\n\nopensuse-install-pkg() {\n    opensuse-wait-for-zypper\n    # Add zypper option \"--force\" if environment variable reinstall_<pkg>=1\n    local pkg\n    local opts=\"\"\n    for pkg in \"$@\"; do\n        if [ \"$(eval echo \\$reinstall_$pkg)\" == \"1\" ]; then\n            opts=\"$opts --force\"\n            break\n        fi\n    done\n    # In OpenSUSE 15.2 zypper exits with status 106 if already installed,\n    # in 15.3 the exit status is 0. Do not consider \"already installed\"\n    # as an error.\n    vm-command \"$ZYPPER install $opts $*\" || [ \"$COMMAND_STATUS\" == \"106\" ] ||\n        command-error \"failed to install $*\"\n}\n\nopensuse-install-pkg-local() {\n    opensuse-wait-for-zypper\n    local force=\"\"\n    if [ \"$1\" == \"--force\" ]; then\n        force=\"--nodeps --force\"\n        shift\n    fi\n    vm-command \"rpm -Uvh $force $*\" ||\n        command-error \"failed to install local package(s)\"\n}\n\nopensuse-remove-pkg() {\n    vm-command 'for i in $*; do rpm -q --quiet $i || continue; $ZYPPER remove $i || exit 1; done' ||\n        command-error \"failed to remove package(s) $*\"\n}\n\nopensuse-install-golang() {\n    distro-install-pkg wget tar gzip git-core\n    from-tarball-install-golang\n}\n\nopensuse-wait-for-zypper() {\n    vm-run-until --timeout 5 '( ! pgrep zypper >/dev/null ) || ( pkill -9 zypper; sleep 1; exit 1 )' ||\n        error \"Failed to stop zypper running in the background\"\n}\n\nopensuse-install-k8s() {\n    vm-command \"( lsmod | grep -q br_netfilter ) || { echo br_netfilter > /etc/modules-load.d/50-br_netfilter.conf; modprobe br_netfilter; }\"\n    vm-command \"echo 1 > /proc/sys/net/ipv4/ip_forward\"\n    vm-command \"zypper ls\"\n    if ! grep -q snappy <<< \"$COMMAND_OUTPUT\"; then\n        distro-install-repo \"http://download.opensuse.org/repositories/system:/snappy/openSUSE_Leap_15.6 snappy\"\n        distro-refresh-pkg-db\n    fi\n    distro-install-pkg \"snapd apparmor-profiles socat ebtables conntrackd iptables ethtool cni-plugins\"\n    distro-install-crictl\n    vm-command \"mkdir -p /opt/cni && ln -fs /usr/lib/cni/ -T /opt/cni/bin\"\n\n    vm-command \"systemctl enable --now snapd\"\n    vm-command \"snap wait system seed.loaded\"\n    for kubepart in kubelet kubectl kubeadm; do\n        local snapcmd=install\n        local k8sverparam\n        if vm-command-q \"snap info $kubepart | grep -q tracking\"; then\n            # $kubepart is already installed, either refresh or reinstall it.\n            if [ \"$(eval echo \\$reinstall_$kubepart)\" == \"1\" ]; then\n                # Reinstalling $kubepart requested.\n                # snap has no option for direct reinstalling,\n                # so the package needs to be removed first.\n                vm-command \"snap remove $kubepart\"\n                snapcmd=install\n            else\n                snapcmd=refresh\n            fi\n        fi\n        # Specify snap channel if user has requested a specific k8s version.\n        if [[ \"$k8s\" == *.*.* ]]; then\n            echo \"WARNING: cannot snap install k8s=X.Y.Z, installing latest X.Y\"\n            k8sverparam=\"--channel ${k8s%.*}/edge\"\n        elif [[ \"$k8s\" == *.* ]]; then\n            k8sverparam=\"--channel ${k8s}/edge\"\n        elif [[ -z \"$k8s\" ]]; then\n            k8sverparam=\"\"\n        else\n            error \"invalid k8s version ${k8s}, expected k8s=X.Y\"\n        fi\n        vm-command \"snap $snapcmd $k8sverparam $kubepart --classic\"\n    done\n    # Manage kubelet with systemd rather than snap\n    vm-command \"snap stop kubelet\"\ncat <<EOF |\n[Unit]\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/snap/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime-endpoint=${k8scri_sock} --pod-infra-container-image=k8s.gcr.io/pause:3.4.1\nRestart=always\nStartLimitInterval=0\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n    vm-pipe-to-file /etc/systemd/system/kubelet.service\n    vm-command \"systemctl enable --now kubelet\" ||\n        command-error \"failed to enable kubelet\"\n}\n\nopensuse-install-kernel-dev() {\n    vm-command-q \"zypper lr | grep -q openSUSE_Tools\" ||\n        distro-install-repo \"http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_Factory/openSUSE:Tools.repo\"\n    distro-install-pkg \"git-core make gcc flex bison bc ncurses-devel patch bzip2 osc build python quilt\"\n    vm-command \"cd /root; [ -d kernel ] || git clone --depth=100 https://github.com/SUSE/kernel\"\n    vm-command \"cd /root; [ -d kernel-source ] || git clone --depth=100 https://github.com/SUSE/kernel-source\"\n    vm-command \"[ -f /etc/profile.d/linux_git.sh ] || echo export LINUX_GIT=/root/kernel > /etc/profile.d/linux_git.sh\"\n}\n\nopensuse-bootstrap-commands-pre() {\n    cat <<EOF\nsed -e '/Signature checking/a gpgcheck = off' -i /etc/zypp/zypp.conf\nEOF\n}\n\nopensuse-govm-env() {\n    echo \"DISABLE_VGA=N\"\n}\n\n###########################################################################\n\n#\n# Generic rpm functions\n#\n\nrpm-pkg-type() {\n    echo rpm\n}\n\nrpm-install-repo-key() {\n    local key\n    for key in \"$@\"; do\n        vm-command \"rpm --import $key\" ||\n            command-error \"failed to import repo key $key\"\n    done\n}\n\nrpm-refresh-pkg-db() {\n    :\n}\n\n###########################################################################\n\n#\n# default implementations\n#\n\ndefault-bootstrap-commands() {\n    cat <<EOF\nrm -f /etc/modules-load.d/k8s.conf; touch /etc/modules-load.d/k8s.conf\nmodprobe bridge && echo bridge >> /etc/modules-load.d/k8s.conf || :\nmodprobe nf-tables-bridge && echo nf-tables-bridge >> /etc/modules-load.d/k8s.conf || :\nmodprobe br_netfilter && echo br_netfilter >> /etc/modules-load.d/k8s.conf || :\n\ntouch /etc/sysctl.d/k8s.conf\necho \"net.bridge.bridge-nf-call-ip6tables = 1\" >> /etc/sysctl.d/k8s.conf\necho \"net.bridge.bridge-nf-call-iptables = 1\" >> /etc/sysctl.d/k8s.conf\necho \"net.ipv4.ip_forward = 1\" >> /etc/sysctl.d/k8s.conf\n\n# rp_filter (partially) mitigates DDOS attacks with spoofed IP addresses\n# by dropping packages with non-routable (unanswerable) source addresses.\n# However, rp_filter > 0 breaks cilium networking. Make sure it's disabled.\necho \"net.ipv4.conf.*.rp_filter = 0\" >> /etc/sysctl.d/k8s.conf\n\n/sbin/sysctl -p /etc/sysctl.d/k8s.conf || :\nEOF\n}\n\ndefault-setup-proxies() {\n    # Notes:\n    #   We blindly assume that upper- vs. lower-case env vars are identical.\n    # shellcheck disable=SC2154\n    if [ -z \"$http_proxy$https_proxy$ftp_proxy$no_proxy\" ]; then\n        return 0\n    fi\n    if vm-command-q \"grep -q \\\"http_proxy=$http_proxy\\\" /etc/profile.d/proxy.sh && \\\n                     grep -q \\\"https_proxy=$https_proxy\\\" /etc/profile.d/proxy.sh && \\\n                     grep -q \\\"ftp_proxy=$ftp_proxy\\\" /etc/profile.d/proxy.sh && \\\n                     grep -q \\\"no_proxy=$no_proxy\\\" /etc/profile.d/proxy.sh\" 2>/dev/null; then\n        # No changes in proxy configuration\n        return 0\n    fi\n\n    local file scope=\"\" append=\"--append\" hn ext_no_proxy\n    hn=\"$(vm-command-q hostname)\"\n\n    local master_node_ip_comma=\"\"\n    if [ -n \"$k8smaster\" ]; then\n        local master_user_ip\n        master_user_ip=\"$(vm-ssh-user-ip $k8smaster)\"\n        master_node_ip_comma=${master_user_ip/*@},\n    fi\n\n    ext_no_proxy=\"$master_node_ip_comma$VM_IP,10.0.0.0/8,$CNI_SUBNET,$hn,.svc,.internal,192.168.0.0/16\"\n\n    for file in /etc/environment /etc/profile.d/proxy.sh; do\n        cat <<EOF |\n${scope}http_proxy=$http_proxy\n${scope}https_proxy=$https_proxy\n${scope}ftp_proxy=$ftp_proxy\n${scope}no_proxy=$no_proxy,$ext_no_proxy\n${scope}HTTP_PROXY=$http_proxy\n${scope}HTTPS_PROXY=$https_proxy\n${scope}FTP_PROXY=$ftp_proxy\n${scope}NO_PROXY=$no_proxy,$ext_no_proxy\nEOF\n      vm-pipe-to-file $append $file\n      scope=\"export \"\n      append=\"\"\n    done\n    # Setup proxies for systemd services that might be installed later\n    for file in /etc/systemd/system/{containerd,docker,crio}.service.d/proxy.conf; do\n        cat <<EOF |\n[Service]\nEnvironment=HTTP_PROXY=$http_proxy\nEnvironment=HTTPS_PROXY=$https_proxy\nEnvironment=NO_PROXY=$no_proxy,$ext_no_proxy\nEOF\n        vm-pipe-to-file $file\n    done\n    # Setup proxies inside docker containers\n    for file in /{root,home/$VM_SSH_USER}/.docker/config.json; do\n        cat <<EOF |\n{\n    \"proxies\": {\n        \"default\": {\n            \"httpProxy\": \"$http_proxy\",\n            \"httpsProxy\": \"$https_proxy\",\n            \"noProxy\": \"$no_proxy,$ext_no_proxy\"\n        }\n    }\n}\nEOF\n        vm-pipe-to-file $file\n    done\n}\n\ndefault-setup-oneshot() {\n    :\n}\n\ndefault-install-utils() {\n    # $distro-install-utils() is responsible for installing common\n    # utilities, such as pidof and killall, that the test framework\n    # and tests in general can expect to be found on VM.\n    :\n}\n\ndefault-k8s-cni() {\n    echo ${k8scni:-bridge}\n}\n\ndefault-k8s-cni-subnet() {\n    if [ \"$(distro-k8s-cni)\" == \"flannel\" ]; then\n        echo 10.244.0.0/16\n    else\n        echo 10.217.0.0/16\n    fi\n}\n\ndefault-install-runc() {\n    distro-install-pkg runc\n}\n\ndefault-install-containerd() {\n    vm-command-q \"[ -f /usr/bin/containerd ]\" || {\n        distro-install-pkg containerd\n    }\n}\n\ndefault-config-containerd() {\n    if vm-command-q \"[ ! -f /etc/containerd/config.toml ]\"; then\n        vm-command \"mkdir -p /etc/containerd && containerd config default > /etc/containerd/config.toml\"\n    fi\n\n    vm-sed-file /etc/containerd/config.toml 's/^.*disabled_plugins *= *.*$/disabled_plugins = []/'\n\n    if vm-command-q \"containerd config dump | grep -v -q SystemdCgroup\"; then\n        vm-command \"containerd config dump > /etc/containerd/config.toml\"\n    fi\n\n    vm-sed-file /etc/containerd/config.toml 's/SystemdCgroup = false/SystemdCgroup = true/g'\n}\n\ndefault-restart-containerd() {\n    vm-command \"systemctl daemon-reload && systemctl restart containerd\" ||\n        command-error \"failed to restart containerd systemd service\"\n}\n\ndefault-install-crio() {\n    [ -n \"$crio_src\" ] || error \"crio install error: crio_src is not set\"\n    [ -x \"$crio_src/bin/crio\" ] || error \"crio install error: file not found $crio_src/bin/crio\"\n    for f in crio crio-status pinns; do\n        vm-put-file \"$crio_src/bin/$f\" \"/usr/bin/$f\"\n    done\n    cat <<EOF |\n[Unit]\nDescription=cri-o container runtime\nDocumentation=https://cri-o.io\nAfter=network.target\n\n[Service]\nExecStart=/usr/bin/crio\n\nDelegate=yes\nKillMode=process\nRestart=always\nLimitNPROC=infinity\nLimitCORE=infinity\nLimitNOFILE=1048576\nTasksMax=infinity\n\n[Install]\nWantedBy=multi-user.target\nEOF\n    vm-pipe-to-file /etc/systemd/system/crio.service\n    vm-command \"mkdir -p /etc/systemd/system/crio.service.d\"\n    vm-command \"(echo \\\"[Service]\\\"; echo \\\"Environment=PATH=/sbin:/usr/sbin:$PATH:/usr/libexec/podman\\\") > /etc/systemd/system/crio.service.d/path.conf; systemctl daemon-reload\"\n}\n\ndefault-config-crio() {\n    vm-command \"mkdir -p /etc/containers\"\n    echo '{\"default\": [{\"type\":\"insecureAcceptAnything\"}]}' | vm-pipe-to-file /etc/containers/policy.json\n    cat <<EOF |\n[registries.search]\nregistries = ['docker.io']\nEOF\n    vm-pipe-to-file /etc/containers/registries.conf\n}\n\ndefault-restart-crio() {\n    vm-command \"systemctl daemon-reload && systemctl restart crio\" ||\n        command-error \"failed to restart crio systemd service\"\n}\n\ndefault-install-minikube() {\n    vm-command \"curl -Lo /usr/local/bin/minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 && chmod +x /usr/local/bin/minikube\"\n    distro-install-crictl\n}\n\ndefault-install-crictl() {\n    vm-command \"set -e -x\n    wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz\n    sudo tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz -C /usr/local/bin\n    rm -f crictl-${CRICTL_VERSION}-linux-amd64.tar.gz\n    \"\n}\n\ndefault-install-cri-dockerd() {\n    vm-command \"set -e -x\n    git clone --depth=1 https://github.com/Mirantis/cri-dockerd.git\n    cd cri-dockerd\n    mkdir bin\n    go build -o bin/cri-dockerd\n    mkdir -p /usr/local/bin\n    install -o root -g root -m 0755 bin/cri-dockerd /usr/local/bin/cri-dockerd\n    cp -a packaging/systemd/* /etc/systemd/system\n    sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service\n    systemctl daemon-reload\n    systemctl enable cri-docker.service\n    systemctl enable --now cri-docker.socket\n    \"\n}\n\ndefault-govm-env() {\n    echo \"DISABLE_VGA=Y\"\n}\n\ndefault-env-file-dir() {\n    echo \"/etc/sysconfig\"\n}\n\n###########################################################################\n\n#\n# generic supporting functions\n#\n\nfrom-tarball-install-golang() {\n    vm-command-q \"go version | grep -q go$GOLANG_VERSION\" || {\n        vm-command \"wget --progress=dot:giga $GOLANG_URL -O go.tgz\" && \\\n            vm-command \"tar -C /usr/local -xvzf go.tgz >/dev/null && rm go.tgz\" && \\\n            vm-command \"echo 'PATH=/usr/local/go/bin:\\$PATH' > /etc/profile.d/go.sh\" && \\\n            vm-command \"echo \\* installed \\$(go version)\"\n    }\n}\n\ncreate-ext4-var-lib-containerd() {\n    local dir=\"/var/lib/containerd\" file=\"/loop-ext4.dsk\" dev\n\n    echo \"Creating loopback-mounted ext4 $dir...\"\n\n    if ! dev=\"$(vm-command-q \"losetup -f\")\" || [ -z \"$dev\" ]; then\n        command-error \"failed to find unused loopback device\"\n    fi\n    vm-command \"dd if=/dev/zero of=$file bs=$((1024*1000)) count=$((1000*5))\" ||\n        command-error \"failed to create file for ext4 loopback mount\"\n    vm-command \"losetup $dev $file\" ||\n        command-error \"failed to attach $file to $dev\"\n    vm-command \"mkfs.ext4 $dev\" ||\n        command-error \"failed to create ext4 filesystem on $dev ($file)\"\n    if vm-command \"[ -d $dir ]\"; then\n        vm-command \"mv $dir $dir.orig\" ||\n            command-error \"failed to rename original $dir to $dir.orig\"\n    fi\n    vm-command \"mkdir -p $dir\" ||\n        command-error \"failed to create $dir\"\n\n    cat <<EOF |\n[Unit]\nDescription=Activate loop device\nDefaultDependencies=no\nAfter=systemd-udev-settle.service\nWants=systemd-udev-settle.service\n\n[Service]\nExecStart=/sbin/losetup $dev $file\nType=oneshot\n\n[Install]\nWantedBy=local-fs.target\nEOF\n    vm-pipe-to-file /etc/systemd/system/attach-loop-devices.service\n    vm-command \"systemctl enable attach-loop-devices.service\"\n\n    cat <<EOF |\n$dev    $dir    ext4    defaults    1 2\nEOF\n    vm-pipe-to-file --append /etc/fstab\n\n    vm-command \"mount $dir\" ||\n        command-error \"failed to mount new ext4 $dir\"\n    if vm-command \"[ -d $dir.orig ]\"; then\n        vm-command \"tar -C $dir.orig -cf - . | tar -C $dir -xf -\" ||\n            command-error \"failed to copy $dir.orig to $dir\"\n    fi\n}\n\nCNI_SUBNET=$(distro-k8s-cni-subnet)\n"
  },
  {
    "path": "demo/lib/host.bash",
    "content": "source \"$(dirname \"${BASH_SOURCE[0]}\")/command.bash\"\n\nHOST_PROMPT=${HOST_PROMPT-\"\\e[38;5;11mhost>\\e[0m \"}\nHOST_LIB_DIR=\"$(dirname \"${BASH_SOURCE[0]}\")\"\nHOST_PROJECT_DIR=\"$(dirname \"$(dirname \"$(realpath \"$HOST_LIB_DIR\")\")\")\"\nHOST_VM_IMAGE_DIR=~/vms/images\nHOST_VM_DATA_DIR_TEMPLATE=\"~/vms/data/\\${VM_NAME}\"\nif [ -z \"$HOST_GORESCTRL_DIR\" ]; then\n    HOST_GORESCTRL_DIR=\"$(realpath \"$HOST_PROJECT_DIR/../goresctrl\")\"\nfi\nGOVM=${GOVM-govm}\n\nhost-command() {\n    command-start \"host\" \"$HOST_PROMPT\" \"$1\"\n    bash -c \"$COMMAND\" 2>&1 | command-handle-output\n    command-end ${PIPESTATUS[0]}\n    return $COMMAND_STATUS\n}\n\nhost-require-govm() {\n    command -v \"$GOVM\" >/dev/null || error \"cannot run govm \\\"$GOVM\\\". Check PATH or set GOVM=/path/to/govm.\"\n}\n\nhost-require-cmd() {\n    command -v \"$1\" >/dev/null || error \"cannot run \\\"$1\\\". Check dependencies.\"\n}\n\nhost-get-vm-config() {\n    if [ -z \"$1\" ]; then\n        error \"can't get VM configuration, name not set\"\n    fi\n    VM_NAME=\"$1\"\n    HOST_VM_DATA_DIR=\"$(eval \"echo $HOST_VM_DATA_DIR_TEMPLATE\")\"\n    VM_DATA_CONFIG=\"$HOST_VM_DATA_DIR/vm-config\"\n    if ! [ -f \"$VM_DATA_CONFIG\" ]; then\n        return 1\n    fi\n    source \"$VM_DATA_CONFIG\"\n    if [ -z \"$VM_NAME\" ] || [ -z \"$VM_DISTRO\" ] || [ -z \"$VM_CRI\" ] || [ -z \"$VM_SSH_USER\" ]; then\n        return 1\n    fi\n    VM_COMPOSE_YAML=\"$HOST_VM_DATA_DIR/govm-compose.yaml\"\n}\n\nhost-set-vm-config() {\n    if [ -z \"$1\" ]; then\n        error \"can't configure VM, name not set\"\n    fi\n    if [ -z \"$2\" ]; then\n        error \"can't configure VM, distro not set\"\n    fi\n    if [ -z \"$3\" ]; then\n        error \"can't configure VM, CRI runtime not set\"\n    fi\n    VM_NAME=\"$1\"\n    VM_DISTRO=\"$2\"\n    VM_CRI=\"$3\"\n    VM_SSH_USER=\"$(vm-ssh-user)\"\n    HOST_VM_DATA_DIR=\"$(eval \"echo $HOST_VM_DATA_DIR_TEMPLATE\")\"\n    mkdir -p \"$HOST_VM_DATA_DIR\"\n    VM_COMPOSE_YAML=\"$HOST_VM_DATA_DIR/govm-compose.yaml\"\n    VM_DATA_CONFIG=\"$HOST_VM_DATA_DIR/vm-config\"\n    cat > \"$VM_DATA_CONFIG\" <<EOF\nVM_NAME=\"$VM_NAME\"\nVM_DISTRO=\"$VM_DISTRO\"\nVM_CRI=\"$VM_CRI\"\nVM_SSH_USER=\"$VM_SSH_USER\"\nEOF\n}\n\nhost-fetch-vm-image() {\n    local url=$(vm-image-url)\n    local file=$(basename $url)\n    local image decompress\n    [ -d \"$HOST_VM_IMAGE_DIR\" ] || mkdir -p \"$HOST_VM_IMAGE_DIR\" ||\n        error \"cannot create directory for VM images: $HOST_VM_IMAGE_DIR\"\n    case $file in\n        *.xz)\n            image=${file%.xz}\n            decompress=\"xz -d\"\n            ;;\n        *.bz2)\n            image=${file%.bz2}\n            decompress=\"bzip -d\"\n            ;;\n        *.gz)\n            image=${file%.gz}\n            decompress=\"gzip -d\"\n            ;;\n        *)\n            image=\"$file\"\n            decompress=\":\"\n            ;;\n    esac\n    [ -f \"$HOST_VM_IMAGE_DIR/$image\" ] || {\n        echo \"VM image $HOST_VM_IMAGE_DIR/$image not found...\"\n        [ -f \"$HOST_VM_IMAGE_DIR/$file\" ] || {\n            echo \"downloading VM image $image...\"\n            host-command \"wget --progress=dot:giga -O \\\"$HOST_VM_IMAGE_DIR/$file\\\" \\\"$url\\\"\" ||\n                error \"failed to download VM image ($url)\"\n        }\n        if [ -n \"$decompress\" ]; then\n            echo \"decompressing VM image $file...\"\n            ( cd \"$HOST_VM_IMAGE_DIR\" && $decompress $file ) ||\n                error \"failed to decompress $file to $image using $decompress\"\n        fi\n        if [ ! -f \"$HOST_VM_IMAGE_DIR/$image\" ]; then\n            error \"internal error, fetching+decompressing $url did not produce $HOST_VM_IMAGE_DIR/$image\"\n        fi\n    }\n    VM_IMAGE=\"$HOST_VM_IMAGE_DIR/$image\"\n}\n\nhost-create-vm() {\n    # Usage: host-create-vm NAME [NUMANODELIST_JSON]\n    #\n    # If successful, VM_IP variable contains the IP address of the govm guest.\n    #\n    # If NUMANODELIST_JSON is given, Qemu CPU and memory parameters are\n    # generated from it. Example, create VM with four identical NUMA nodes:\n    #     host-create-vm myvm '[{\"cpu\": 2, \"mem\": \"2G\", \"nodes\": 4}]'\n    #\n    # If NUMANODELIST_JSON is not given, Qemu CPU and memory parameters\n    # can be defined directly in VM_QEMU_CPUMEM environment variable.\n    # VM_QEMU_CPUMEM is expected to contain at least parameters\n    #     -m MEMORY -smp CPUCORES\n    #\n    # Example: four numa nodes, 2 cores each\n    #     VM_QEMU_CPUMEM=\"-m 8G,slots=4,maxmem=32G \\\n    #         -smp cpus=8 \\\n    #         -numa node,cpus=0-1,nodeid=0 \\\n    #         -numa node,cpus=2-3,nodeid=1 \\\n    #         -numa node,cpus=4-5,nodeid=2 \\\n    #         -numa node,cpus=6-7,nodeid=3 \\\n    #         -cpu host\"\n    #     host-create-vm my-four-numa-node-pc\n    #\n    # If NUMANODELIST_JSON parameter or VM_QEMU_CPUMEM environment\n    # variable defined, the VM will be created with \"govm compose\" and\n    # VM_GOVM_COMPOSE_TEMPLATE yaml. In both cases parameters in\n    # VM_QEMU_EXTRA environment variable are passed through to Qemu.\n    #\n    # Debug Qemu parameters and output with\n    #     $ docker logs $(docker ps | awk '/govm/{print $1; exit}')\n    #\n    local TOPOLOGY=\"$2\"\n\n    if [ -z \"$VM_NAME\" ]; then\n        error \"cannot create VM: missing name\"\n    fi\n    if [ -n \"$TOPOLOGY\" ]; then\n        if [ -n \"$VM_QEMU_CPUMEM\" ]; then\n            error \"cannot take both VM_QEMU_CPUMEM and numa node JSON\"\n        fi\n        VM_QEMU_CPUMEM=$(echo \"$TOPOLOGY\" | \"$HOST_LIB_DIR/topology2qemuopts.py\")\n        if [ \"$?\" -ne  \"0\" ]; then\n            error \"error in topology\"\n        fi\n    fi\n    host-require-govm\n    # If VM does not exist, create it from scrach\n    ${GOVM} ls | grep -q \"$VM_NAME\" || {\n        host-fetch-vm-image\n        mkdir -p \"$(dirname \"$VM_COMPOSE_YAML\")\"\n        vm-compose-govm-template > \"$VM_COMPOSE_YAML\"\n        host-command \"${GOVM} compose -f \\\"$VM_COMPOSE_YAML\\\"\"\n        echo \"# VM base image  : $VM_IMAGE\"\n        echo \"# VM govm yaml   : $VM_COMPOSE_YAML\"\n    }\n\n    sleep 1\n    VM_CONTAINER_ID=$(${GOVM} ls | awk \"/$VM_NAME/{print \\$1}\")\n    # Verify Qemu version. Refuse to run if Qemu < 5.0.\n    # Use \"docker run IMAGE\" instead of \"docker exec CONTAINER\",\n    # because the container may have already failed.\n    VM_CONTAINER_IMAGE=$(docker inspect $VM_CONTAINER_ID | jq '.[0].Image' -r | awk -F: '{print $2}')\n    echo \"# VM name        : $VM_NAME\"\n    echo \"# VM Linux distro: $VM_DISTRO\"\n    echo \"# VM CRI         : $VM_CRI\"\n    echo \"# VM Docker image: $VM_CONTAINER_IMAGE\"\n    echo \"# VM Docker cntnr: $VM_CONTAINER_ID\"\n    if [ -n \"$VM_CONTAINER_IMAGE\" ]; then\n        VM_CONTAINER_QEMU_VERSION=$(docker run --rm --entrypoint=/usr/bin/qemu-system-x86_64 $VM_CONTAINER_IMAGE -version | awk '/QEMU emulator version/{print $4}')\n    fi\n    if [ -n \"$VM_CONTAINER_QEMU_VERSION\" ]; then\n        if [[ \"$VM_CONTAINER_QEMU_VERSION\" > \"5\" ]]; then\n            echo \"# VM Qemu version: $VM_CONTAINER_QEMU_VERSION\"\n        else\n            if [[ \"$QEMU_CPUMEM\" =~ \",dies=\" ]]; then\n                error \"Too old Qemu version \\\"$VM_CONTAINER_QEMU_VERSION\\\". Topology with dies > 1 requires Qemu >= 5.0\"\n            else\n                echo \"# (Your Qemu does not support dies > 1, consider updating for full topology support)\"\n            fi\n        fi\n    else\n        echo \"Warning: cannot verify Qemu version on govm image. In case of failure, check it is >= 5.0\" >&2\n    fi\n    echo \"# VM Qemu output : docker logs $VM_CONTAINER_ID\"\n    echo \"# VM Qemu monitor: docker exec -it $VM_CONTAINER_ID nc local:/data/monitor\"\n    VM_MONITOR=\"docker exec -i $VM_CONTAINER_ID nc local:/data/monitor\"\n    host-wait-vm-ssh-server\n    host-wait-cloud-init\n}\n\nget-ssh-timeout() {\n    echo $((`date +%s` + $1))\n}\n\nhost-wait-vm-ssh-server() {\n    timeout=`get-ssh-timeout 120`\n\n    while [ \"${1#-}\" != \"$1\" ] && [ -n \"$1\" ]; do\n        case \"$1\" in\n            --timeout)\n                timeout=`get-ssh-timeout $2`\n                shift; shift\n                ;;\n            *)\n                invalid=\"${invalid}${invalid:+,}\\\"$1\\\"\"\n                shift\n                ;;\n        esac\n    done\n    if [ -n \"$invalid\" ]; then\n        error \"invalid options: $invalid\"\n        return 1\n    fi\n\n    if [ -z \"$VM_IP\" ]; then\n        VM_IP=$(${GOVM} ls | awk \"/$VM_NAME/{print \\$4}\")\n        while [ \"x$VM_IP\" == \"x\" ]; do\n            host-command \"${GOVM} start \\\"$VM_NAME\\\"\"\n            sleep 5\n            VM_IP=$(${GOVM} ls | awk \"/$VM_NAME/{print \\$4}\")\n        done\n    fi\n    echo \"# VM SSH server  : ssh $VM_SSH_USER@$VM_IP\"\n\n    if [ -d \"$HOME/vms/data/$VM_NAME\" ]; then\n        SSH_OPTS=\"$SSH_OPTS -o ControlMaster=auto -o ControlPath=$HOME/vms/data/$VM_NAME/ssh -o ControlPersist=30\"\n        SSH=\"${SSH%% *} $SSH_OPTS\"\n        SCP=\"${SCP%% *} $SSH_OPTS\"\n        export SSH SSH_OPTS SCP\n    fi\n\n    ssh-keygen -f \"$HOME/.ssh/known_hosts\" -R \"$VM_IP\" >/dev/null 2>&1\n\n    print_info=1\n\n    while ! $SSH ${VM_SSH_USER}@${VM_IP} -o ConnectTimeout=2 true 2>/dev/null; do\n\tCURR_TIME=`date +%s`\n\tif [ $CURR_TIME -gt $timeout ]; then\n            error \"timeout\"\n\tfi\n\n\tif [ \"$print_info\" == 1 ]; then\n            echo -n \"Waiting for VM SSH server to respond...\"\n\t    print_info=0\n\tfi\n        sleep 2\n        echo -n \".\"\n    done\n    echo \"\"\n}\n\nhost-wait-cloud-init() {\n    retries=60\n    retries_left=$retries\n    while true; do\n        $SSH -o ConnectTimeout=2 ${VM_SSH_USER}@${VM_IP} sudo cloud-init status --wait 2>/dev/null\n        [ \"$?\" -eq 0 -o \"$?\" -eq 2 ] && break\n\n        if [ \"$retries\" == \"$retries_left\" ]; then\n            echo -n \"Waiting for VM cloud-init to finish...\"\n        fi\n        sleep 2\n        echo -n \".\"\n        retries_left=$(( $retries_left - 1 ))\n        if [ \"$retries_left\" == \"0\" ]; then\n            error \"timeout\"\n        fi\n    done\n    [ \"$retries\" == \"$retries_left\" ] || echo \"\"\n}\n\nhost-stop-vm() {\n    #VM_NAME=$1\n    host-require-govm\n    host-command \"${GOVM} stop $VM_NAME\" || {\n        command-error \"stopping govm \\\"$VM_NAME\\\" failed\"\n    }\n}\n\nhost-delete-vm() {\n    #VM_NAME=$1\n    host-require-govm\n    host-command \"${GOVM} delete $VM_NAME\" || {\n        command-error \"deleting govm \\\"$VM_NAME\\\" failed\"\n    }\n}\n\nhost-is-encrypted-ssh-key() {\n    ssh-keygen -y -f \"$1\" < /dev/null >& /dev/null\n    if [ $? != 0 ]; then\n        return 0\n    else\n        return 1\n    fi\n}\n\nhost-mount-vm() {\n    # Usage: host-mount-vm\n    #\n    # Mount VM / to VM data directory on host.\n    # host-get-vm-config NAME must be run first.\n    local mountpoint=\"${HOST_VM_DATA_DIR}/sshfs\"\n    local vm_sftp_server=\"\"\n    local vm_sftp_server_candidates=(/usr/lib/openssh/sftp-server /usr/libexec/sftp-server)\n    local vm_sftp_server_candidate\n    command -v sshfs >/dev/null || {\n        error \"host-mount-vm: missing sshfs\"\n    }\n    if mount | grep \"${mountpoint}\"; then\n        echo \"host-mount-vm: already mounted\"\n        return 0\n    fi\n    for vm_sftp_server_candidate in \"${vm_sftp_server_candidates[@]}\"; do\n        if vm-command-q \"command -v ${vm_sftp_server_candidate} >/dev/null\"; then\n            vm_sftp_server=\"${vm_sftp_server_candidate}\"\n            break\n        fi\n    done\n    if [ -z \"${vm_sftp_server}\" ]; then\n        error \"cannot find sftp-server from vm\"\n    fi\n    mkdir -p \"${mountpoint}\"\n    sshfs \"${VM_SSH_USER}@${VM_IP}:/\" \"${mountpoint}\" -o sftp_server=\"/usr/bin/sudo ${vm_sftp_server}\" $SSH_OPTS || {\n        error \"sshfs mount failed\"\n    }\n    echo \"host-mount-vm: mounted ${VM_NAME}:/ to ${mountpoint}\"\n}\n"
  },
  {
    "path": "demo/lib/numactlH2numajson.py",
    "content": "#!/usr/bin/env python3\n\n\"\"\"numactlH2numajson - convert numactl -H output to numajson\n\nExample:\n  numactl -H | numactlH2numajson\n\"\"\"\n\nimport json\nimport math\nimport re\nimport sys\n\nQEMU_DEFAULT_DIST_OTHER = 20\nQEMU_DEFAULT_DIST_SELF = 10\n\ndef error(msg, exit_status=1):\n    sys.stderr.write(\"numactlH2numajson: %s\\n\" % (msg,))\n    if not exit_status is None:\n        sys.exit(1)\n\ndef round_size(size, size_unit, non_zero_numbers=3):\n    if size_unit == \"kB\":\n        size_mb = size / 1024\n    elif size_unit == \"MB\":\n        size_mb = size\n    elif size_unit == \"GB\":\n        size_mb = size * 1024\n    elif size_unit == \"TB\":\n        size_mb = size * 1024 * 1024\n    else:\n        raise Exception(\"unsupported size unit: %r\" % (size_unit,))\n    if size_mb == 0:\n        return \"0G\"\n    size_mul = 10**int(math.log10(size_mb))\n    rounded = round(size_mb * 10**(non_zero_numbers-1) / size_mul) * size_mul / (10**(non_zero_numbers-1))\n    if size_mul < 1000:\n        return \"%.0fM\" % (rounded,)\n    else:\n        return \"%.0fG\" % (rounded/1000)\n\ndef add_dists_to_numalist(numalist, dists):\n    \"\"\"Add/replace distance information in numalist with node distances in dists.\n       dists[i][j] = distance from node i to node j.\n       dists can be a matrix or a dict: {sourcenode: {destnode: dist}}\"\"\"\n    dist_matrix = []\n    node = -1\n    node_group = {} # {node: group_index_in_numalist}\n    group_nodes = {} # {group_index_in_numalist: set_of_nodes}\n    for groupindex, numaspec in enumerate(numalist):\n        group_nodes[groupindex] = set()\n        nodecount = int(numaspec.get(\"nodes\", 1))\n        for _ in range(nodecount):\n            node += 1\n            group_nodes[groupindex].add(node)\n            node_group[node] = groupindex\n    lastnode = node\n    if isinstance(dists, list):\n        # dists is a dist matrix.\n        dist_matrix = dists\n    else:\n        # dists is a dict. create dist_matrix from it.\n        for sourcenode in range(lastnode + 1):\n            dist_matrix.append([])\n            for destnode in range(lastnode + 1):\n                if sourcenode in dists and destnode in dists[sourcenode]:\n                    d = dists[sourcenode][destnode]\n                elif sourcenode != destnode:\n                    d = QEMU_DEFAULT_DIST_OTHER\n                else:\n                    d = QEMU_DEFAULT_DIST_SELF\n                dist_matrix[-1].append(d)\n    dist_freq = {} # {distance: number-of-appearances}\n    try:\n        for sourcenode in range(lastnode + 1):\n            for destnode in range(lastnode + 1):\n                if sourcenode != destnode:\n                    d = dist_matrix[sourcenode][destnode]\n                    dist_freq[d] = dist_freq.get(d, 0) + 1\n    except IndexError:\n        raise ValueError(\"invalid dists matrix dimensions, %sx%s expected\" % (lastnode + 1, lastnode + 1))\n    # Read the most common distance from the matrix, ignore distance-to-self.\n    if len(dist_freq) > 0:\n        default_dist = max([(v, k) for k, v in dist_freq.items()])[1]\n    else:\n        default_dist = QEMU_DEFAULT_DIST_SELF # don't care: there's only one node\n    # Try filling symmetric distances with the default dist.\n    # There may be asymmetry or node grouping that making this impossible.\n    # In those cases sym_dist_errors > 0.\n    sym_dist_errors = 0\n    group_node_dist = {} # {group_index: {othernode: dist}}\n    for sourcenode in range(lastnode + 1):\n        sourcegroup = node_group[sourcenode]\n        if not sourcegroup in group_node_dist:\n            group_node_dist[sourcegroup] = {}\n        for destnode in range(lastnode + 1):\n            destgroup = node_group[destnode]\n            if sourcenode == destnode:\n                continue\n            elif dist_matrix[sourcenode][destnode] == default_dist:\n                continue\n            elif dist_matrix[sourcenode][destnode] != dist_matrix[destnode][sourcenode]:\n                # There is asymmetry.\n                sym_dist_errors += 1\n                continue\n            for othernode in [n for n in group_nodes[sourcegroup] if n != sourcenode and n != destnode]:\n                if (dist_matrix[othernode][destnode] != dist_matrix[sourcenode][destnode] or\n                    dist_matrix[othernode][destnode] != dist_matrix[destnode][sourcenode]):\n                    # Different nodes in the same group have different distances.\n                    sym_dist_errors += 1\n            group_node_dist[sourcegroup][destnode] = dist_matrix[sourcenode][destnode]\n    # Clear existing distance definitions from numalist.\n    for numaspec in numalist:\n        if \"dist\" in numaspec:\n            del numaspec[\"dist\"]\n        if \"dist-all\" in numaspec:\n            del numaspec[\"dist-all\"]\n        if \"node-dist\" in numaspec:\n            del numaspec[\"node-dist\"]\n    # Now we are ready to add distance information.\n    if sym_dist_errors == 0 and len(str(group_node_dist)) < len(str(dist_matrix)):\n        # Add info using \"dist\" and \"node-dist\", that is symmetrical distances.\n        # This time it is more compact representation than a matrix.\n        for groupindex, numaspec in enumerate(numalist):\n            if group_node_dist[groupindex] != {}:\n                # if all nodes mentioned in node-dist are in earlier groups,\n                # there is no need to inject this definition, because it has been\n                # covered by distance symmetry.\n                nodes_with_dists = set(group_node_dist[groupindex].keys())\n                for earlier_group in range(groupindex):\n                    nodes_with_dists -= group_nodes[earlier_group]\n                # there are new distance definitions, include all\n                if len(nodes_with_dists) > 0:\n                    numaspec[\"node-dist\"] = group_node_dist[groupindex]\n        if default_dist != QEMU_DEFAULT_DIST_OTHER:\n            numalist[0][\"dist\"] = default_dist\n    elif len(numalist) > 1:\n        # Add distances as a matrix.\n        numalist[-1][\"dist-all\"] = dist_matrix\n    else:\n        # There is no need for distance information in the numalist,\n        # as there is only one node.\n        pass\n\ndef numactlH2numajson(input_line_iter):\n    numalist = []\n    dist_matrix = []\n    re_node_cpus = re.compile('^node (?P<node>[0-9]+) cpus:( (?P<cpus>([0-9]+\\s?)*))?')\n    re_node_size = re.compile('^node (?P<node>[0-9]+) size:( (?P<size>[0-9]+) (?P<size_unit>[a-zA-Z]+))?')\n    re_node_distances = re.compile('^\\s*(?P<sourcenode>[0-9]+):(?P<dists>(\\s*[0-9]+)*)')\n    for line in input_line_iter:\n        m = re_node_cpus.match(line)\n        if m:\n            m_dict = m.groupdict()\n            node = int(m_dict[\"node\"])\n            if m_dict[\"cpus\"] is None:\n                cpus = []\n            else:\n                cpus = [int(cpu) for cpu in m.groupdict()[\"cpus\"].strip().split()]\n            continue\n        m = re_node_size.match(line)\n        if m:\n            m_dict = m.groupdict()\n            if int(m_dict[\"node\"]) != node:\n                raise Exception(\"expected node %s size, got %r\" % (node, line))\n            size_unit = m_dict[\"size_unit\"]\n            mem = round_size(int(m_dict[\"size\"]), size_unit)\n            if (len(numalist) == 0\n                or numalist[-1][\"cpu\"] != len(cpus)\n                or numalist[-1][\"mem\"] != mem):\n                # found a node that is different from the previous\n                numalist.append({\"cpu\": len(cpus),\n                                      \"mem\": mem,\n                                      \"nodes\": 1})\n            else:\n                # found a node that looks the same as the previous\n                numalist[-1][\"nodes\"] += 1\n            nodecount = node + 1\n            continue\n        m = re_node_distances.match(line)\n        if m:\n            m_dict = m.groupdict()\n            dist_matrix.append([int(d) for d in m_dict['dists'].strip().split()])\n\n    # filter out unnecessary \"nodes\": 1 from the list:\n    for d in numalist:\n        if d[\"nodes\"] == 1:\n            del d[\"nodes\"]\n    # parse distances\n    add_dists_to_numalist(numalist, dist_matrix)\n    return numalist\n\ndef self_test():\n    input_output = {\n        \"\"\"available: 5 nodes (0-4)\nnode 0 cpus: 0\nnode 0 size: 1007 MB\nnode 0 free: 784 MB\nnode 1 cpus: 1\nnode 1 size: 1007 MB\nnode 1 free: 262 MB\nnode 2 cpus: 2 3\nnode 2 size: 1951 MB\nnode 2 free: 1081 MB\nnode 3 cpus: 4 5 6 7\nnode 3 size: 4030 MB\nnode 3 free: 693 MB\nnode 4 cpus:\nnode 4 size: 8039 MB\nnode 4 free: 8029 MB\nnode distances:\nnode   0   1   2   3   4\n  0:  10  22  22  22  88\n  1:  22  10  22  22  88\n  2:  22  22  10  22  88\n  3:  22  22  22  10  88\n  4:  88  88  88  88  10\n\"\"\": [{'cpu': 1, 'mem': '1G', 'nodes': 2, 'node-dist': {4: 88}, 'dist': 22}, {'cpu': 2, 'mem': '2G', 'node-dist': {4: 88}}, {'cpu': 4, 'mem': '4G', 'node-dist': {4: 88}}, {'cpu': 0, 'mem': '8G'}],\n        \"\"\"available: 2 nodes (0-1)\nnode 0 cpus: 0 1 2 3\nnode 0 size: 3966 MB\nnode 0 free: 1649 MB\nnode 1 cpus: 4 5 6 7\nnode 1 size: 4006 MB\nnode 1 free: 983 MB\nnode distances:\nnode   0   1\n  0:  10  20\n  1:  20  10\n\"\"\": [{'cpu': 4, 'mem': '4G', 'nodes': 2}],\n\"\"\"available: 4 nodes (0-3)\nnode 0 cpus: 0 1 2 3\nnode 0 size: 3966 MB\nnode 0 free: 1649 MB\nnode 1 cpus: 4 5 6 7\nnode 1 size: 4006 MB\nnode 1 free: 983 MB\nnode 1 cpus: 8 9 10 11\nnode 1 size: 4006 MB\nnode 1 free: 983 MB\nnode 1 cpus: 12 13 14 15\nnode 1 size: 4006 MB\nnode 1 free: 983 MB\nnode distances:\nnode   0   1   2   3\n  0:  10  55  55  55\n  1:  55  10  55  55\n  2:  55  55  10  55\n  3:  55  55  55  10\n\"\"\": [{'cpu': 4, 'mem': '4G', 'nodes': 4, 'dist': 55}],\n    \"\"\"available: 1 nodes (0)\nnode 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19\nnode 0 size: 128000 MB\nnode 0 free: 80000 MB\nnode distances:\nnode   0\n  0:  10\n\"\"\": [{'cpu': 20, 'mem': '128G'}],\n        \"\"\"available: 5 nodes (0-4)\nnode 0 cpus: 0\nnode 0 size: 4007 MB\nnode 0 free: 784 MB\nnode 1 cpus: 1\nnode 1 size: 1007 MB\nnode 1 free: 262 MB\nnode 2 cpus: 2 3\nnode 2 size: 1951 MB\nnode 2 free: 1081 MB\nnode 3 cpus: 4 5 6 7\nnode 3 size: 4030 MB\nnode 3 free: 693 MB\nnode 4 cpus:\nnode 4 size: 8039 MB\nnode 4 free: 8029 MB\nnode distances:\nnode   0   1   2   3   4\n  0:  10  22  33  44  55\n  1:  22  10  22  22  22\n  2:  33  22  10  22  22\n  3:  44  22  22  10  22\n  4:  55  22  22  22  10\n\"\"\": [{'cpu': 1, 'mem': '4G', 'node-dist': {2: 33, 3: 44, 4: 55}, 'dist': 22}, {'cpu': 1, 'mem': '1G'}, {'cpu': 2, 'mem': '2G'}, {'cpu': 4, 'mem': '4G'}, {'cpu': 0, 'mem': '8G'}]\n    }\n\n    for input_string in input_output.keys():\n        observed = numactlH2numajson(input_string.splitlines())\n        expected = input_output[input_string]\n        if observed != expected:\n            raise Exception(\"self-test: observed/expected mismatch on numanodes\\n%s\\n\\nobserved: %r\\nexpected: %r\" % (input_string, observed, expected))\n    add_dists_to_numalist([], [])\n    return 0\n\nif __name__ == \"__main__\":\n    if len(sys.argv) > 1 and sys.argv[1] == \"test\":\n        sys.exit(self_test())\n    try:\n        numalist = numactlH2numajson(sys.stdin)\n    except Exception as e:\n        raise\n        error(str(e))\n    print(json.dumps(numalist))\n"
  },
  {
    "path": "demo/lib/topology.py",
    "content": "#!/usr/bin/env python3\n\n\"\"\"topology.py - topology utility\n\nUsage: topology.py [options] command\n\nOptions:\n  -t TOPOLOGY_DUMP    load topology_dump from TOPOLOGY_DUMP file instead of\n                      the \"topology_dump\" environment variable or local host.\n  -r RES_ALLOWED      load res_allowed from RES_ALLOWED file instead of\n                      the \"res_allowed\" environment variable or local host.\n  -o OUTPUT_FORMAT    \"json\" or \"text\". The default is \"text\".\n\nCommands:\n  help                print help\n  cpus                view CPU topology from topology_dump.\n  cpus_allowed [PROCESS...]\n                      view how matching PROCESSes are allowed to use CPUs.\n                      (Uses RES_ALLOWED like res_allowed below.)\n  res                 view CPU and memory topology from topology_dump.\n  res_allowed [PROCESS...]\n                      view how matching PROCESSes are allowed to use CPUs\n                      and memory in CPU/mem topology tree.\n                      If the RES_ALLOWED file or the res_allowed\n                      environment variable are not defined, \"pgrep -f PROCESS\"\n                      is used to match processes.\n  bash_topology_dump  print a Bash command that creates topology_dump.\n  bash_res_allowed PROCESS [PROCESS...]\n                      print a Bash command that creates res_allowed\n                      dump that contains Cpus_allowed and Mems_allowed\n                      masks of processes matching \"pgrep -f PROCESS\".\n\nExamples:\n  Print local host CPU topology\n  $ topology.py cpus\n\n  Print how processes with pod0..2 in their names are allowed to use CPUs\n  $ topology.py res_allowed pod0 pod1 pod2\n\n  Print remote host CPU topology\n  $ topology_dump=\"$(ssh remotehost \"$(topology.py bash_topology_dump)\")\" topology.py cpus\n\n  Watch how pod0..2 are allowed to CPUS on remote host, read topology only once\n  $ export topology_dump=\"$(ssh remotehost \"$(topology.py bash_topology_dump)\")\"\n  $ watch 'res_allowed=$(ssh remotehost \"$(topology.py bash_res_allowed pod0 pod1 pod2)\") topology.py res_allowed'\n\"\"\"\n\nimport getopt\nimport json\nimport os\nimport re\nimport subprocess\nimport sys\n\n_bash_topology_dump = \"\"\"for cpu in /sys/devices/system/cpu/cpu[0-9]*; do cpu_id=${cpu#/sys/devices/system/cpu/cpu}; echo \"cpu p:$(< ${cpu}/topology/physical_package_id) d:$(< ${cpu}/topology/die_id) n:$(basename  ${cpu}/node* | sed 's:node::g') c:$(< ${cpu}/topology/core_id) t:$(< ${cpu}/topology/thread_siblings) cpu:${cpu_id}\" ; done;  for node in /sys/devices/system/node/node[0-9]*; do node_id=${node#/sys/devices/system/node/node}; echo \"dist n:$node_id d:$(< $node/distance)\"; echo \"mem n:$node_id s:$(awk '/MemTotal/{print $4/1024}' < $node/meminfo)\"; done\"\"\"\n\n_bash_res_allowed = r\"\"\"for process in '%s'; do for pid in $(pgrep -f \"$process\"); do proc_pid_cmdline=$(< /proc/$pid/cmdline) || continue; proc_pid_status=$(< /proc/$pid/status) || continue; name=$(echo \"$proc_pid_cmdline\" | tr '\\0 ' '\\n' | grep -E \"^$process\" | head -n 1); [ -n \"$name\" ] && [ \"$pid\" != \"$$\" ] && [ \"$pid\" != \"$PPID\" ] && echo \"${name}/${pid} $(awk '/Cpus_allowed:/{c=$2}/Mems_allowed:/{m=$2}END{print \"c:\"c\" m:\"m}' <<< \"$proc_pid_status\")\"; done 2>/dev/null; done\"\"\"\n\ndef error(msg, exit_status=1):\n    \"\"\"Print error message and exit.\"\"\"\n    if not msg is None:\n        sys.stderr.write('topology.py: %s\\n' % (msg,))\n    if not exit_status is None:\n        sys.exit(exit_status)\n\ndef warning(msg):\n    \"\"\"Print warning.\"\"\"\n    sys.stderr.write('topology.py warning: %s\\n' % (msg,))\n\ndef output_tree(tree):\n    \"\"\"Print tree to output in OUTPUT_FORMAT\"\"\"\n    if opt_output_format == \"json\":\n        sys.stdout.write(json.dumps(tree))\n    else:\n        sys.stdout.write(str_tree(tree) + \"\\n\")\n    sys.stdout.flush()\n\ndef add_tree(root, branch, value_dict):\n    \"\"\"Add key-value pairs in value_dict to given branch in the tree starting from root.\n\n    If the branch does not exist in the tree, it will be created.\n\n    Example:\n      add_tree(tree, (\"package0\", \"die1\", \"node3\", \"core7\", \"thread0\", \"cpu15\"), {\"GHz\", 4.2})\n    \"\"\"\n    node = root\n    for b in branch:\n        if b in node:\n            node = node[b]\n        else:\n            node[b] = {}\n            node = node[b]\n    node.update(value_dict)\n\ndef _str_node(root, lines, branch):\n    \"\"\"Format node names in tree to lines ([[line1col1, line1col2], ...]).\"\"\"\n    for key in sorted(root.keys()):\n        branch.append(key)\n        if root[key]:\n            _str_node(root[key], lines, branch)\n        else:\n            # Add those column texts to the new line which does not have the same value\n            # as previous non-empty text in the same column.\n            new_line = []\n            new_col_txt_added = False\n            for col, txt in enumerate(branch):\n                if new_col_txt_added:\n                    prev_col_txt = \"\"\n                else:\n                    for prev_line in lines[::-1]:\n                        if len(prev_line) > col and prev_line[col] != \"\":\n                            prev_col_txt = prev_line[col]\n                            break\n                    else:\n                        prev_col_txt = \"\"\n                if txt != prev_col_txt:\n                    new_line.append(txt)\n                    new_col_txt_added = True\n                else:\n                    new_line.append(\"\")\n            lines.append(new_line)\n        branch.pop()\n\ndef str_tree(root):\n    \"\"\"Format tree to string.\"\"\"\n    lines = []\n    _str_node(root, lines, [])\n    col_max_len = {} # {column-index: max-string-length}\n    max_col = -1\n    for line in lines:\n        for col, txt in enumerate(line):\n            if col > max_col:\n                max_col = col\n            if len(txt) > col_max_len.get(col, -1):\n                col_max_len[col] = len(txt)\n    str_lines = []\n    for line in lines:\n        line_cols = len(line)\n        new_str_fmt = \"\"\n        for col, txt in enumerate(line):\n            new_str_fmt += \"%-\" + str(col_max_len[col] + 1) + \"s\"\n        str_lines.append(new_str_fmt % tuple(line))\n    return \"\\n\".join(str_lines)\n\ndef bash_output(cmd):\n    \"\"\"Return standard output of executing cmd in Bash.\"\"\"\n    p = subprocess.Popen([\"bash\", \"-c\", cmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n    out, err = p.communicate()\n    return out.decode(\"utf-8\")\n\ndef get_local_topology_dump():\n    \"\"\"Return topology_dump from local system.\"\"\"\n    return bash_output(_bash_topology_dump)\n\ndef get_local_res_allowed_dump(processes):\n    \"\"\"Return res_allowed from local system.\"\"\"\n    return bash_output(_bash_res_allowed % (\"' '\".join(processes),))\n\ndef dump_to_topology(dump, show_mem=True):\n    \"\"\"Parse topology_dump, return topology data structures.\"\"\"\n    # Output data structures:\n    tree = {} # {\"package0\": {\"die1\": {\"node1\": ...}}}\n    cpu_branch = {} # {cpu_id: (package_name, die_name, node_name, core_name, thread_name, cpu_name)}\n    node_branch = {} # {node_id: (package_name, die_name, node_name)}\n    mem_branch = {} # {node_id: (package_name, ...)}\n    # Example input line to be parsed:\n    # cpu line:\n    # \"cpu p:0 d:1 n:3 c:2 t:00003000 cpu:13\"\n    # mem line:\n    # \"mem n:4: s:8063.83\"\n    re_cpu_line = re.compile('cpu p:(?P<package>[0-9]+) d:(?P<die>[0-9]*) n:(?P<node>[0-9]+) c:(?P<core>[0-9]+) t:(?P<thread_siblings>[0-9a-f,]+) cpu:(?P<cpu_id>[0-9]+)')\n    re_mem_line = re.compile('mem n:(?P<node>[0-9]+) s:(?P<size>[0-9.]+)')\n    re_dist_line = re.compile('dist n:(?P<node>[0-9]+) d:(?P<dist>([0-9 ]+))')\n    numeric_cpu_lines = []\n    numeric_mem_lines = []\n    numeric_dist_lines = []\n    for line in dump.splitlines():\n        m = re_cpu_line.match(line)\n        if m:\n            mdict = m.groupdict()\n            package = int(mdict[\"package\"])\n            try:\n                die = int(mdict[\"die\"])\n            except ValueError:\n                die = 0 # handle kernels that do not provide topology/die_id\n            node = int(mdict[\"node\"])\n            core = int(mdict[\"core\"])\n            thread_siblings = eval(\"0x\" + mdict[\"thread_siblings\"].replace(\",\", \"\"))\n            cpu_id = int(mdict[\"cpu_id\"])\n            # Calculate thread id.\n            # Let the lowest CPU bit owner in thread_siblings be thread 0, next thread 1 and so on.\n            thread = -1\n            bit = 1 << cpu_id\n            while bit:\n                if thread_siblings & bit:\n                    thread += 1\n                bit >>= 1\n            numeric_cpu_lines.append((package, die, node, core, thread, cpu_id))\n            continue\n        m = re_mem_line.match(line)\n        if m:\n            mdict = m.groupdict()\n            numeric_mem_lines.append((int(mdict[\"node\"]), float(mdict[\"size\"])))\n            continue\n        m = re_dist_line.match(line)\n        if m:\n            mdict = m.groupdict()\n            numeric_dist_lines.append((int(mdict[\"node\"]),\n                                      tuple([int(n) for n in mdict[\"dist\"].strip().split()])))\n    numeric_mem_lines.sort() # make sure memory sizes are from node 0, 1, ...\n    numeric_dist_lines.sort()\n\n    # Build tree on CPUs\n    max_package_len = max(len(str(nl[0])) for nl in numeric_cpu_lines)\n    max_die_len = max(len(str(nl[1])) for nl in numeric_cpu_lines)\n    max_node_len = max(len(str(nl[2])) for nl in numeric_cpu_lines)\n    max_core_len = max(len(str(nl[3])) for nl in numeric_cpu_lines)\n    max_thread_len = max(len(str(nl[4])) for nl in numeric_cpu_lines)\n    max_cpu_id_len = max(len(str(nl[5])) for nl in numeric_cpu_lines)\n    for (package, die, node, core, thread, cpu_id) in numeric_cpu_lines:\n        branch = (\"package\" + str(package).zfill(max_package_len),\n                  \"die\" + str(die).zfill(max_die_len),\n                  \"node\" + str(node).zfill(max_node_len),\n                  \"core\" + str(core).zfill(max_core_len),\n                  \"thread\" + str(thread).zfill(max_thread_len),\n                  \"cpu\" + str(cpu_id).zfill(max_cpu_id_len))\n        add_tree(tree, branch, {})\n        cpu_branch[cpu_id] = branch\n        node_branch[node] = branch[:3]\n    if show_mem:\n        # Add node memory information to the tree\n        for node, distvec in numeric_dist_lines:\n            mem_node_name = \"node\" + str(node).zfill(max_node_len)\n            node_mem_size = str(int(round((numeric_mem_lines[node][1]/1024)))) + \"G\"\n            dists = sorted(distvec)\n            if node in node_branch:\n                # This node has CPU(s) as it has been added to the tree already in CPU lines.\n                # Add memory branch to the tree under the existing node branch.\n                branch = node_branch[node] + (\n                    \"mem\", mem_node_name, node_mem_size)\n            elif (dists[0] == 10 # sane distance-to-self\n                  and (len(dists) < 3 or dists[1] < dists[2])  # there is a node closer than others\n                  and distvec.index(dists[1]) in node_branch): # that node is already in the tree\n                # This means that the node has the same memory controller as this node.\n                # Add memory branch from this node under the existing node.\n                node_same_ctrl = distvec.index(dists[1])\n                branch = node_branch[node_same_ctrl] + (\n                    \"mem\", mem_node_name, node_mem_size)\n                node_branch[node] = branch[:3]\n            else:\n                # Suitable memory controller not found, create completely separate branch.\n                branch = (\"packagex\", \"mem\", \"node\" + str(node).zfill(max_node_len),\n                    \"mem\", mem_node_name, node_mem_size)\n                node_branch[node] = branch[:3]\n            add_tree(tree, branch, {})\n            mem_branch[node] = branch\n    return {\"tree\": tree,\n            \"cpu_branch\": cpu_branch,\n            \"node_branch\": node_branch,\n            \"mem_branch\": mem_branch}\n\ndef dump_to_res_allowed(res_allowed_dump):\n    \"\"\"Parse res_allowed data, return allowed cpu and mem bitmasks in a data structure.\"\"\"\n    # Output data structure:\n    owner_mask = {} # {owner_string: {\"cpu\": bitmask_int, \"mem\": bitmask_int}}\n    # Example input line to be parsed:\n    # \"pod2  c:040c0000,00000000 m:00000000,00000300\"\n    re_owner_mask = re.compile(r'(?P<owner>[^ ]+)\\s+c:(?P<cpumask>[0-9a-f,]+)\\s+m:(?P<memmask>[0-9a-f,]+)')\n    for line in res_allowed_dump.splitlines():\n        if not line:\n            continue\n        try:\n            mdict = re_owner_mask.match(line).groupdict()\n        except:\n            warning(\"cannot parse res_allowed line %r\" % (line,))\n            continue\n        owner_mask[mdict[\"owner\"]] = {\n            \"cpu\": eval(\"0x\" + mdict[\"cpumask\"].replace(\",\", \"\")),\n            \"mem\": eval(\"0x\" + mdict[\"memmask\"].replace(\",\", \"\"))\n        }\n    return owner_mask\n\ndef get_topology(show_mem=True):\n    \"\"\"Return topology data structure.\"\"\"\n    # Priority: use file, environment variable or read from local system\n    if opt_topology_dump:\n        topology_dump = opt_topology_dump\n    else:\n        topology_dump = os.getenv(\"topology_dump\", None)\n    if topology_dump is None:\n        topology_dump = get_local_topology_dump()\n    return dump_to_topology(topology_dump, show_mem=show_mem)\n\ndef get_res_allowed(processes):\n    \"\"\"Return res_allowed data structure.\"\"\"\n    # Priority: use file, environment variable or read from local system\n    if opt_res_allowed_dump:\n        res_allowed_dump = opt_res_allowed_dump\n    else:\n        res_allowed_dump = os.getenv(\"res_allowed\", None)\n    if res_allowed_dump is None:\n        res_allowed_dump = get_local_res_allowed_dump(processes)\n    return dump_to_res_allowed(res_allowed_dump)\n\ndef report_res(show_mem=True):\n    \"\"\"Print topology tree.\"\"\"\n    topology = get_topology(show_mem=show_mem)\n    output_tree(topology[\"tree\"])\n\ndef report_res_allowed(processes, show_mem=True):\n    \"\"\"Print topology tree with allowed processes as leaf nodes.\"\"\"\n    topology = get_topology(show_mem=show_mem)\n    tree = topology[\"tree\"]\n    cpu_branch = topology[\"cpu_branch\"]\n    mem_branch = topology[\"mem_branch\"]\n    node_branch = topology[\"node_branch\"]\n    max_cpu = max(cpu_branch.keys())\n    max_node = max(node_branch.keys())\n    res_allowed = get_res_allowed(processes)\n    # add found owners to tree as children of cpus\n    for owner, masks in sorted(res_allowed.items()):\n        cpumask = masks[\"cpu\"]\n        memmask = masks[\"mem\"]\n        for cpu in range(max_cpu + 1):\n            if cpumask & (1 << cpu):\n                add_tree(tree, cpu_branch[cpu], {owner: {}})\n        if show_mem:\n            for node in range(max_node + 1):\n                if memmask & (1 << node):\n                    add_tree(tree, mem_branch[node], {owner: {}})\n    output_tree(tree)\n\nif __name__ == \"__main__\":\n    opt_topology_dump = None\n    opt_res_allowed_dump = None\n    opt_output_format = \"text\"\n    try:\n        options, commands = getopt.gnu_getopt(\n            sys.argv[1:], 'ht:r:o:',\n            ['help', '--topology-dump-file=', '--res-allowed-file='])\n    except getopt.GetoptError as e:\n        error(str(e))\n    for opt, arg in options:\n        if opt in [\"-h\", \"--help\"]:\n            print(__doc__)\n            error(None, exit_status=0)\n        elif opt in [\"-t\", \"--topology-file\"]:\n            try:\n                opt_topology_dump = open(arg).read()\n            except IOError as e:\n                error(\"cannot read topology dump from file %r: %s\" % (arg, e))\n        elif opt in [\"-r\", \"--res-allowed-file\"]:\n            try:\n                opt_res_allowed_dump = open(arg).read()\n            except IOError as e:\n                error(\"cannot read res_allowed dump from file %r: %s\" % (arg, e))\n        elif opt in [\"-o\"]:\n            if arg in [\"json\", \"text\"]:\n                opt_output_format = arg\n            else:\n                error(\"invalid output format %r\")\n    if not commands:\n        error(\"missing command, see --help\")\n    elif commands[0] == \"help\":\n        print(__doc__)\n        error(None, exit_status=0)\n    elif commands[0] == \"cpus\":\n        report_res(show_mem=False)\n    elif commands[0] == \"cpus_allowed\":\n        report_res_allowed(commands[1:], show_mem=False)\n    elif commands[0] == \"res\":\n        report_res(show_mem=True)\n    elif commands[0] == \"res_allowed\":\n        report_res_allowed(commands[1:])\n    elif commands[0] == \"bash_topology_dump\":\n        print(_bash_topology_dump)\n    elif commands[0] == \"bash_res_allowed\":\n        print(_bash_res_allowed % (\"' '\".join(commands[1:]),))\n    else:\n        error('invalid command %r' % (commands[0],))\n"
  },
  {
    "path": "demo/lib/topology2qemuopts.py",
    "content": "#!/usr/bin/env python3\n\n\"\"\"topology2qemuopts - convert NUMA node list from JSON to Qemu options\n\nNUMA node group definitions:\n\"mem\"                 mem (RAM) size on each NUMA node in this group.\n                      The default is \"0G\".\n\"nvmem\"               nvmem (non-volatile RAM) size on each NUMA node\n                      in this group. The default is \"0G\".\n\"dimm\"                \"\": the default, memory is there without pc-dimm defined.\n                      \"plugged\": start with cold plugged pc-dimm.\n                      \"unplugged\": start with free slot for hot plug.\n                        Add the dimm in Qemu monitor at runtime:\n                          device_add pc-dimm,id=dimmX,memdev=memX,node=X\n                        or\n                          device_add nvdimm,id=nvdimmX,memdev=nvmemX,node=X\n\"cores\"               number of CPU cores on each NUMA node in this group.\n                      The default is 0.\n\"threads\"             number of threads on each CPU core.\n                      The default is 2.\n\"nodes\"               number of NUMA nodes on each die.\n                      The default is 1.\n\"dies\"                number of dies on each package.\n                      The default is 1.\n\"packages\"            number of packages.\n                      The default is 1.\n\nNUMA node distances are defined with following keys:\n\"dist-all\": [[from0to0, from0to1, ...], [from1to0, from1to1, ...], ...]\n                      distances from every node to all nodes.\n                      The order is the same as in to numactl -H\n                      \"node distances:\" output.\n\"node-dist\": {\"node\": dist, ...}\n                      symmetrical distances from nodes in this group to other\n                      nodes.\n\nDistances that apply to all NUMA groups if defined in any:\n\"dist-same-die\": N    the default distance between NUMA nodes on the same die.\n\"dist-same-package\": N the default distance between NUMA nodes on the same package.\n\"dist-other-package\": N  the default distance between NUMA nodes in other packages.\n\nNote that the distance from a node to itself is always 10. The default\ndistance to a node on the same die is 11, and to other nodes on the\nsame and different packages is 21.\n\nExample: Each of the first two NUMA groups in the list contains two\nNUMA nodes. Each node in the first group includes two CPU cores and 2G\nRAM, while nodes in the second group two CPU cores and 1G RAM. The\nonly NUMA node defined in the third group has 8G of NVRAM, and no CPU.\n\nEvery NUMA group with CPU cores adds a package (a socket) to the\nconfiguration, or many identical packages if \"packages\" > 1.  This\nexample creates a two-socket system, four CPU cores per package. Note\nthat CPU cores are divided symmetrically to packages, meaning that\nevery NUMA group with CPU cores should contain the same number of\ncores.\n\n$ ( cat << EOF\n[\n    {\n        \"mem\": \"2G\",\n        \"cores\": 2,\n        \"nodes\": 2\n    },\n    {\n        \"mem\": \"1G\",\n        \"cores\": 2,\n        \"nodes\": 2\n    },\n    {\n        \"nvmem\": \"8G\",\n        \"node-dist\": {\"0\": 88, \"1\": 88, \"2\": 88, \"3\": 88,\n                      \"4\": 66, \"5\": 66, \"7\": 66, \"8\": 66}\n    }\n]\nEOF\n) | python3 topology2qemuopts.py\n\"\"\"\n\nimport sys\nimport json\n\nDEFAULT_DIST = 21\nDEFAULT_DIST_SAME_PACKAGE = 21\nDEFAULT_DIST_SAME_DIE = 11\nDEFAULT_DIST_SAME_NODE = 10\n\ndef error(msg, exitstatus=1):\n    sys.stderr.write(\"topology2qemuopts: %s\\n\" % (msg,))\n    if exitstatus is not None:\n        sys.exit(exitstatus)\n\ndef siadd(s1, s2):\n    if s1.lower().endswith(\"g\") and s2.lower().endswith(\"g\"):\n        return str(int(s1[:-1]) + int(s2[:-1])) + \"G\"\n    raise ValueError('supports only sizes in gigabytes, example: 2G')\n\ndef sisub(s1, s2):\n    if s1.lower().endswith(\"g\") and s2.lower().endswith(\"g\"):\n        return str(int(s1[:-1]) - int(s2[:-1])) + \"G\"\n    raise ValueError('supports only sizes in gigabytes, example: 2G')\n\ndef validate(numalist):\n    if not isinstance(numalist, list):\n        raise ValueError('expected list containing dicts, got %s' % (type(numalist,).__name__))\n    valid_keys = set((\"mem\", \"nvmem\", \"dimm\",\n                      \"cores\", \"threads\", \"nodes\", \"dies\", \"packages\",\n                      \"node-dist\", \"dist-all\",\n                      \"dist-other-package\", \"dist-same-package\", \"dist-same-die\"))\n    int_range_keys = {'cores': ('>= 0', lambda v: v >= 0),\n                      'threads': ('> 0', lambda v: v > 0),\n                      'nodes': ('> 0', lambda v: v > 0),\n                      'dies': ('> 0', lambda v: v > 0),\n                      'packages': ('> 0', lambda v: v > 0)}\n    for numalistindex, numaspec in enumerate(numalist):\n        for key in numaspec:\n            if not key in valid_keys:\n                raise ValueError('invalid name %r in node %r' % (key, numaspec))\n            if key in [\"mem\", \"nvmem\"]:\n                val = numaspec.get(key)\n                if val == \"0\":\n                    continue\n                errmsg = 'invalid %s in node %r, expected string like \"2G\"' % (key, numaspec)\n                if not isinstance(val, str):\n                    raise ValueError(errmsg)\n                try:\n                    siadd(val, \"0G\")\n                except ValueError:\n                    raise ValueError(errmsg)\n            if key in int_range_keys:\n                try:\n                    val = int(numaspec[key])\n                    if not int_range_keys[key][1](val):\n                        raise Exception()\n                except:\n                    raise ValueError('invalid %s in node %r, expected integer %s' % (key, numaspec, int_range_keys[key][0]))\n        if 'threads' in numaspec and int(numaspec.get('cores', 0)) == 0:\n            raise ValueError('threads set to %s but \"cores\" is 0 in node %r' % (numaspec[\"threads\"], numaspec))\n\ndef dists(numalist):\n    dist_dict = {} # Return value: {sourcenode: {destnode: dist}}, fully defined for all nodes\n    sourcenode = -1\n    lastsocket = -1\n    dist_same_die = DEFAULT_DIST_SAME_DIE\n    dist_same_package = DEFAULT_DIST_SAME_PACKAGE\n    dist_other_package = DEFAULT_DIST # numalist \"dist\", if defined\n    node_package_die = {} # topology {node: (package, die)}\n    dist_matrix = None # numalist \"dist_matrix\", if defined\n    node_node_dist = {} # numalist {sourcenode: {destnode: dist}}, if defined for sourcenode\n    lastnode_in_group = -1\n    for groupindex, numaspec in enumerate(numalist):\n        nodecount = int(numaspec.get(\"nodes\", 1))\n        corecount = int(numaspec.get(\"cores\", 0))\n        diecount = int(numaspec.get(\"dies\", 1))\n        packagecount = int(numaspec.get(\"packages\", 1))\n        first_node_in_group = sourcenode + 1\n        for package in range(packagecount):\n            if nodecount > 0:\n                lastsocket += 1\n            for die in range(diecount):\n                for node in range(nodecount):\n                    sourcenode += 1\n                    dist_dict[sourcenode] = {}\n                    node_package_die[sourcenode] = (lastsocket, die)\n        lastnode_in_group = sourcenode + 1\n        if \"dist\" in numaspec:\n            dist = numaspec[\"dist\"]\n        if \"dist-same-die\" in numaspec:\n            dist_same_die = numaspec[\"dist-same-die\"]\n        if \"dist-same-package\" in numaspec:\n            dist_same_package = numaspec[\"dist-same-package\"]\n        if \"dist-all\" in numaspec:\n            dist_matrix = numaspec[\"dist-all\"]\n        if \"node-dist\" in numaspec:\n            for n in range(first_node_in_group, lastnode_in_group):\n                node_node_dist[n] = {int(nodename): value for nodename, value in numaspec[\"node-dist\"].items()}\n    if lastnode_in_group < 0:\n        raise ValueError('no NUMA nodes found')\n    lastnode = lastnode_in_group - 1\n    if dist_matrix is not None:\n        # Fill the dist_dict directly from dist_matrix.\n        # It must cover all distances.\n        if len(dist_matrix) != lastnode + 1:\n            raise ValueError(\"wrong dimensions in dist-all %s rows seen, %s expected\" % (len(dist_matrix), lastnode))\n        for sourcenode, row in enumerate(dist_matrix):\n            if len(row) != lastnode + 1:\n                raise ValueError(\"wrong dimensions in dist-all on row %s: %s distances seen, %s expected\" % (sourcenode + 1, len(row), lastnode + 1))\n            for destnode, source_dest_dist in enumerate(row):\n                dist_dict[sourcenode][destnode] = source_dest_dist\n    else:\n        for sourcenode in range(lastnode + 1):\n            for destnode in range(lastnode + 1):\n                if sourcenode == destnode:\n                    dist_dict[sourcenode][destnode] = DEFAULT_DIST_SAME_NODE\n                elif sourcenode in node_node_dist and destnode in node_node_dist[sourcenode]:\n                    # User specified explicit node-to-node distance\n                    dist_dict[sourcenode][destnode] = node_node_dist[sourcenode][destnode]\n                    dist_dict[destnode][sourcenode] = node_node_dist[sourcenode][destnode]\n                elif not destnode in dist_dict[sourcenode]:\n                    # Set distance based on topology\n                    if node_package_die[sourcenode] == node_package_die[destnode]:\n                        dist_dict[sourcenode][destnode] = dist_same_die\n                    elif node_package_die[sourcenode][0] == node_package_die[destnode][0]:\n                        dist_dict[sourcenode][destnode] = dist_same_package\n                    else:\n                        dist_dict[sourcenode][destnode] = dist_other_package\n    return dist_dict\n\ndef qemuopts(numalist):\n    machineparam = \"-machine pc\"\n    numaparams = []\n    objectparams = []\n    deviceparams = []\n    lastnode = -1\n    lastcpu = -1\n    lastdie = -1\n    lastsocket = -1\n    lastmem = -1\n    lastnvmem = -1\n    totalmem = \"0G\"\n    totalnvmem = \"0G\"\n    unpluggedmem = \"0G\"\n    pluggedmem = \"0G\"\n    memslots = 0\n    groupnodes = {} # groupnodes[NUMALISTINDEX] = (NODEID, ...)\n    validate(numalist)\n\n    # Read cpu counts, and \"mem\" and \"nvmem\" sizes for all nodes.\n    threadcount = -1\n    for numalistindex, numaspec in enumerate(numalist):\n        nodecount = int(numaspec.get(\"nodes\", 1))\n        groupnodes[numalistindex] = tuple(range(lastnode + 1, lastnode + 1 + nodecount))\n        corecount = int(numaspec.get(\"cores\", 0))\n        if corecount > 0:\n            if threadcount < 0:\n                # threads per cpu, set only once based on the first cpu-ful numa node\n                threadcount = int(numaspec.get(\"threads\", 2))\n                threads_set_node = numaspec\n            else:\n                # threadcount already set, only check that there is no mismatch\n                if (numaspec.get(\"threads\", None) is not None and\n                    threadcount != int(numaspec.get(\"threads\"))):\n                    raise ValueError('all CPUs must have the same number of threads, '\n                                     'but %r had %s threads (the default) which contradicts %r' %\n                                     (threads_set_node, threadcount, numaspec))\n        cpucount = int(numaspec.get(\"cores\", 0)) * threadcount # logical cpus per numa node (cores * threads)\n        diecount = int(numaspec.get(\"dies\", 1))\n        packagecount = int(numaspec.get(\"packages\", 1))\n        memsize = numaspec.get(\"mem\", \"0\")\n        memdimm = numaspec.get(\"dimm\", \"\")\n        if memsize != \"0\":\n            memcount = 1\n        else:\n            memcount = 0\n        nvmemsize = numaspec.get(\"nvmem\", \"0\")\n        if nvmemsize != \"0\":\n            nvmemcount = 1\n        else:\n            nvmemcount = 0\n        for package in range(packagecount):\n            if nodecount > 0 and cpucount > 0:\n                lastsocket += 1\n            for die in range(diecount):\n                if nodecount > 0 and cpucount > 0:\n                    lastdie += 1\n                for node in range(nodecount):\n                    lastnode += 1\n                    currentnumaparams = []\n                    for mem in range(memcount):\n                        lastmem += 1\n                        if memdimm == \"\":\n                            objectparams.append(\"-object memory-backend-ram,size=%s,id=membuiltin_%s_node_%s\" % (memsize, lastmem, lastnode))\n                            currentnumaparams.append(\"-numa node,nodeid=%s,memdev=membuiltin_%s_node_%s\" % (lastnode, lastmem, lastnode))\n                        elif memdimm == \"plugged\":\n                            objectparams.append(\"-object memory-backend-ram,size=%s,id=memdimm_%s_node_%s\" % (memsize, lastmem, lastnode))\n                            currentnumaparams.append(\"-numa node,nodeid=%s\" % (lastnode,))\n                            deviceparams.append(\"-device pc-dimm,node=%s,id=dimm%s,memdev=memdimm_%s_node_%s\" % (lastnode, lastmem, lastmem, lastnode))\n                            pluggedmem = siadd(pluggedmem, memsize)\n                            memslots += 1\n                        elif memdimm == \"unplugged\":\n                            objectparams.append(\"-object memory-backend-ram,size=%s,id=memdimm_%s_node_%s\" % (memsize, lastmem, lastnode))\n                            currentnumaparams.append(\"-numa node,nodeid=%s\" % (lastnode,))\n                            unpluggedmem = siadd(unpluggedmem, memsize)\n                            memslots += 1\n                        else:\n                            raise ValueError(\"unsupported dimm %r, expected 'plugged' or 'unplugged'\" % (memdimm,))\n                        totalmem = siadd(totalmem, memsize)\n                    for nvmem in range(nvmemcount):\n                        lastnvmem += 1\n                        lastmem += 1\n                        if lastnvmem == 0:\n                            machineparam += \",nvdimm=on\"\n                        # Don't use file-backed nvdimms because the file would\n                        # need to be accessible from the govm VM\n                        # container. Everything is ram-backed on host for now.\n                        if memdimm == \"\":\n                            objectparams.append(\"-object memory-backend-ram,size=%s,id=memnvbuiltin_%s_node_%s\" % (nvmemsize, lastmem, lastnode))\n                            currentnumaparams.append(\"-numa node,nodeid=%s,memdev=memnvbuiltin_%s_node_%s\" % (lastnode, lastmem, lastnode))\n                        elif memdimm == \"plugged\":\n                            objectparams.append(\"-object memory-backend-ram,size=%s,id=memnvdimm_%s_node_%s\" % (nvmemsize, lastmem, lastnode))\n                            currentnumaparams.append(\"-numa node,nodeid=%s\" % (lastnode,))\n                            deviceparams.append(\"-device nvdimm,node=%s,id=nvdimm%s,memdev=memnvdimm_%s_node_%s\" % (lastnode, lastmem, lastmem, lastnode))\n                            pluggedmem = siadd(pluggedmem, nvmemsize)\n                            memslots += 1\n                        elif memdimm == \"unplugged\":\n                            objectparams.append(\"-object memory-backend-ram,size=%s,id=memnvdimm_%s_node_%s\" % (nvmemsize, lastmem, lastnode))\n                            currentnumaparams.append(\"-numa node,nodeid=%s\" % (lastnode,))\n                            unpluggedmem = siadd(unpluggedmem, nvmemsize)\n                            memslots += 1\n                        else:\n                            raise ValueError(\"unsupported dimm %r, expected 'plugged' or 'unplugged'\" % (memdimm,))\n                        totalnvmem = siadd(totalnvmem, nvmemsize)\n                    if cpucount > 0:\n                        if not currentnumaparams:\n                            currentnumaparams.append(\"-numa node,nodeid=%s\" % (lastnode,))\n                        currentnumaparams[-1] = currentnumaparams[-1] + (\",cpus=%s-%s\" % (lastcpu + 1, lastcpu + cpucount))\n                        lastcpu += cpucount\n                    numaparams.extend(currentnumaparams)\n    node_node_dist = dists(numalist)\n    for sourcenode in sorted(node_node_dist.keys()):\n        for destnode in sorted(node_node_dist[sourcenode].keys()):\n            if sourcenode == destnode:\n                continue\n            numaparams.append(\"-numa dist,src=%s,dst=%s,val=%s\" % (\n                sourcenode, destnode, node_node_dist[sourcenode][destnode]))\n    if lastcpu == -1:\n        raise ValueError('no CPUs found, make sure at least one NUMA node has \"cores\" > 0')\n    if (lastdie + 1) // (lastsocket + 1) > 1:\n        diesparam = \",dies=%s\" % ((lastdie + 1) // (lastsocket + 1),)\n    else:\n        # Don't give dies parameter unless it is absolutely necessary\n        # because it requires Qemu >= 5.0.\n        diesparam = \"\"\n    cpuparam = \"-smp cpus=%s,threads=%s%s,sockets=%s\" % (lastcpu + 1, threadcount, diesparam, lastsocket + 1)\n    maxmem = siadd(totalmem, totalnvmem)\n    startmem = sisub(sisub(maxmem, unpluggedmem), pluggedmem)\n    memparam = \"-m size=%s,slots=%s,maxmem=%s\" % (startmem, memslots, maxmem)\n    if startmem.startswith(\"0\"):\n        if pluggedmem.startswith(\"0\"):\n            raise ValueError('no memory in any NUMA node')\n        raise ValueError(\"no initial memory in any NUMA node - cannot boot with hotpluggable memory\")\n    return (machineparam + \" \" +\n            cpuparam + \" \" +\n            memparam + \" \" +\n            \" \".join(numaparams) +\n            \" \" +\n            \" \".join(deviceparams) +\n            \" \" +\n            \" \".join(objectparams)\n            )\n\ndef main(input_file):\n    try:\n        numalist = json.loads(input_file.read())\n    except Exception as e:\n        error(\"error reading JSON: %s\" % (e,))\n    try:\n        print(qemuopts(numalist))\n    except Exception as e:\n        error(\"error converting JSON to Qemu opts: %s\" % (e,))\n\nif __name__ == \"__main__\":\n    if len(sys.argv) > 1:\n        if sys.argv[1] in [\"-h\", \"--help\"]:\n            print(__doc__)\n            sys.exit(0)\n        else:\n            input_file = open(sys.argv[1])\n    else:\n        input_file = sys.stdin\n    main(input_file)\n"
  },
  {
    "path": "demo/lib/vm.bash",
    "content": "# shellcheck disable=SC1091\n# shellcheck source=command.bash\nsource \"$(dirname \"${BASH_SOURCE[0]}\")/command.bash\"\n# shellcheck disable=SC1091\n# shellcheck source=distro.bash\nsource \"$(dirname \"${BASH_SOURCE[0]}\")/distro.bash\"\n\nVM_PROMPT=${VM_PROMPT-\"\\e[38;5;11mroot@vm>\\e[0m \"}\n\nvm-compose-govm-template() {\n    (echo \"\nvms:\n  - name: ${VM_NAME}\n    image: ${VM_IMAGE}\n    cloud: true\n    ContainerEnvVars:\n      - KVM_CPU_OPTS=${VM_QEMU_CPUMEM:=-machine pc -smp cpus=4 -m 8G}\n      - EXTRA_QEMU_OPTS=-monitor unix:/data/monitor,server,nowait ${VM_QEMU_EXTRA}\n      - USE_NET_BRIDGES=${USE_NET_BRIDGES:-0}\n$(for govm_env in $(distro-govm-env); do echo \"\n      - ${govm_env}\"; done)\n    user-data: |\n      #!/bin/bash\n      set -e\n\"\n     (if [ -n \"$VM_EXTRA_BOOTSTRAP_COMMANDS\" ]; then\n          # shellcheck disable=SC2001\n          sed 's/^/      /g' <<< \"${VM_EXTRA_BOOTSTRAP_COMMANDS}\"\n     fi\n      # shellcheck disable=SC2001\n      sed 's/^/      /g' <<< \"$(distro-bootstrap-commands)\")) |\n        grep -E -v '^ *$'\n}\n\nvm-bootstrap() {\n    distro-bootstrap-commands | vm-pipe-to-file \"./e2e-bootstrap.sh\"\n    vm-command \"sh ./e2e-bootstrap.sh\"\n    host-wait-vm-ssh-server --timeout 600\n}\n\nvm-image-url() {\n    distro-image-url\n}\n\nvm-ssh-user() {\n    if [ -n \"$VM_SSH_USER\" ]; then\n        echo \"$VM_SSH_USER\"\n    else\n        distro-ssh-user\n    fi\n}\n\n\nvm-is-govm() { # script API\n    local name=\"${1:-$VM_NAME}\"\n    # Usage: vm-is-govm [name]\n    #\n    # Check if the given name (or $VM_NAME if omitted) corresponds to\n    # a govm-managed virtual machine. Returns 0 if it does. Returns 1\n    # if it does not. Returns 2 if govm is not installed.\n\n    if ! type -f govm >& /dev/null; then\n        return 2\n    fi\n    if [ -z \"$name\" ]; then\n        return 1\n    fi\n\n    if govm ls | cut -d ' ' -f 2 | grep -q \"^$name$\"; then\n       return 0\n    fi\n\n    return 1\n}\n\nvm-check-env() {\n    # If VM IP address is already defined, govm is not needed.\n    if [ -n \"$VM_IP\" ]; then\n        if [ \"x$(vm-command-q \"whoami\")\" != \"xroot\" ]; then\n            echo \"ERROR:\"\n            echo \"ERROR: environment check failed:\"\n            echo \"ERROR:   cannot run commands (with sudo) when connecting\"\n            echo \"ERROR:   $SSH $VM_SSH_USER@$VM_IP\"\n            echo \"ERROR:\"\n            return 1\n        fi\n        return 0\n    fi\n    # Check that VM created/managed with govm in this environment.\n    type -p govm >& /dev/null || {\n        echo \"ERROR:\"\n        echo \"ERROR: environment check failed:\"\n        echo \"ERROR:   govm binary not found.\"\n        echo \"ERROR:\"\n        echo \"ERROR: You can install it using the following commands:\"\n        echo \"ERROR:\"\n        echo \"ERROR:     git clone https://github.com/govm-project/govm\"\n        echo \"ERROR:     cd govm\"\n        echo \"ERROR:     go build -o govm\"\n        echo \"ERROR:     cp -v govm \\$GOPATH/bin\"\n        echo \"ERROR:     docker build . -t govm/govm:latest\"\n        echo \"ERROR:     cd ..\"\n        echo \"ERROR:\"\n        return 1\n    }\n    docker inspect govm/govm >& /dev/null || {\n        echo \"ERROR:\"\n        echo \"ERROR: environment check failed:\"\n        echo \"ERROR:   govm/govm docker image not present (but govm needs it).\"\n        echo \"ERROR:\"\n        echo \"ERROR: You can install it using the following commands:\"\n        echo \"ERROR:\"\n        echo \"ERROR:     git clone https://github.com/govm-project/govm\"\n        echo \"ERROR:     cd govm\"\n        echo \"ERROR:     docker build . -t govm/govm:latest\"\n        echo \"ERROR:     cd ..\"\n        echo \"ERROR:\"\n        return 1\n    }\n    if [ ! -e \"$SSH_KEY\".pub ]; then\n        echo \"ERROR:\"\n        echo \"ERROR: environment check failed:\"\n        echo \"ERROR:   $SSH_KEY.pub SSH public key not found (but govm needs it).\"\n        echo \"ERROR:\"\n        echo \"ERROR: You can generate it using the following command:\"\n        echo \"ERROR:\"\n        echo \"ERROR:     ssh-keygen\"\n        echo \"ERROR:\"\n        return 1\n    fi\n    if [ -n \"$SSH_AUTH_SOCK\" ] && [ -e \"$SSH_AUTH_SOCK\" ]; then\n        if ! ssh-add -l | grep -q \"$(ssh-keygen -l -f \"$SSH_KEY\" < /dev/null 2>/dev/null | awk '{print $2}')\"; then\n            if ! ssh-add \"$SSH_KEY\" < /dev/null; then\n                echo \"ERROR:\"\n                echo \"ERROR: environment setup failed:\"\n                echo \"ERROR:   Failed to load $SSH_KEY SSH key to agent.\"\n                echo \"ERROR:\"\n                echo \"ERROR: Please make sure an SSH agent is running, then\"\n                echo \"ERROR: try loading the key using the following command:\"\n                echo \"ERROR:\"\n                echo \"ERROR:     ssh-add $SSH_KEY\"\n                echo \"ERROR:\"\n                return 1\n            fi\n        fi\n    else\n        if host-is-encrypted-ssh-key \"$SSH_KEY\"; then\n            echo \"ERROR:\"\n            echo \"ERROR: environment setup failed:\"\n            echo \"ERROR:   $SSH_KEY SSH key is encrypted, but agent is not running.\"\n            echo \"ERROR:\"\n            echo \"ERROR: Please make sure an SSH agent is running, then\"\n            echo \"ERROR: try loading the key using the following command:\"\n            echo \"ERROR:\"\n            echo \"ERROR:     ssh-add $SSH_KEY\"\n            echo \"ERROR:\"\n            return 1\n        fi\n    fi\n}\n\nvm-check-running-binary() {\n    local bin_file=\"$1\"\n    local bin_name\n    bin_name=\"$(basename \"$bin_file\")\"\n    pid_of_bin=\"$(vm-command-q \"pidof $bin_name\")\"\n    if [ -f \"$bin_file\" ] && [ -n \"$pid_of_bin\" ] && [ \"$(vm-command-q \"md5sum < /proc/$pid_of_bin/exe\")\" != \"$(md5sum < \"$bin_file\")\" ]; then\n        echo \"WARNING:\"\n        echo \"WARNING: Running $bin_name binary is different from\"\n        echo \"WARNING: $bin_file\"\n        echo \"WARNING: Consider restarting with reinstall_${bin_name//-/_}=1.\"\n        echo \"WARNING:\"\n        sleep \"${warning_delay:-0}\"\n        return 1\n    fi\n    return 0\n}\n\nvm-check-source-files-changed() {\n    local bin_change\n    local src_change\n    local src_dir=\"$1\"\n    local bin_file=\"$2\"\n    bin_change=$(stat --format \"%Z\" \"$bin_file\")\n    src_change=$(find \"$src_dir\" -name '*.go' -type f -print0 | xargs -0 stat --format \"%Z\" | sort -n | tail -n 1)\n    if [[ \"$src_change\" > \"$bin_change\" ]]; then\n        echo \"WARNING:\"\n        echo \"WARNING: Source files changed, outdated binaries in\"\n        echo \"WARNING: $(dirname \"$bin_file\")/\"\n        echo \"WARNING:\"\n        sleep \"${warning_delay:-0}\"\n    fi\n}\n\nvm-command() { # script API\n    # Usage: vm-command COMMAND\n    #\n    # Execute COMMAND on virtual machine as root.\n    # Returns the exit status of the execution.\n    # Environment variable COMMAND_OUTPUT contains what COMMAND printed\n    # in standard output and error.\n    #\n    # Examples:\n    #   vm-command \"kubectl get pods\"\n    #   vm-command \"whoami | grep myuser\" || command-error \"user is not myuser\"\n    command-start \"vm\" \"$VM_PROMPT\" \"$1\"\n    if [ \"$2\" == \"bg\" ]; then\n        ( $SSH \"${VM_SSH_USER}@${VM_IP}\" sudo bash -l <<<\"$COMMAND\" 2>&1 | command-handle-output ;\n          command-end \"${PIPESTATUS[0]}\"\n        ) &\n        command-runs-in-bg\n    else\n        $SSH \"${VM_SSH_USER}@${VM_IP}\" sudo bash -l <<<\"$COMMAND\" 2>&1 | command-handle-output ;\n        command-end \"${PIPESTATUS[0]}\"\n    fi\n    return \"$COMMAND_STATUS\"\n}\n\nvm-command-q() {\n    $SSH \"${VM_SSH_USER}@${VM_IP}\" sudo bash -l <<<\"$1\"\n}\n\nvm-ssh-user-ip() {\n    # Usage: vm-ssh-user-ip NODE\n    #\n    # Print canonical USER@HOST for NODE. NODE can be a govm vm name\n    # or already of the form: USER@HOST.\n    local NODE=\"$1\"\n    local node_ssh_user=\"\"\n    local node_ssh_ip=\"\"\n    if [[ \"$NODE\" == *\"@\"* ]]; then\n        node_ssh_ip=${NODE/*@}\n        node_ssh_user=${NODE%@*}\n    else\n        node_ssh_ip=$(${GOVM} ls | awk \"/$NODE/{print \\$4}\")\n        node_ssh_user=$( host-get-vm-config $NODE && echo $VM_SSH_USER )\n    fi\n    if [ -z \"$node_ssh_ip\" ]; then\n        error \"cannot find IP address for NODE=$NODE\"\n    fi\n    if [ -z \"$node_ssh_user\" ]; then\n        error \"cannot find ssh user for NODE=$NODE\"\n    fi\n    echo \"${node_ssh_user}@${node_ssh_ip}\"\n}\n\nvm-join() {\n    # Usage: vm-join MASTER_NODE\n    #\n    # Join vm to the cluster whose master node is MASTER_NODE.\"\n    # MASTER_NODE is a name of a govm virtual machine, or\n    # \"USER@HOST\" that can be logged into using ssh.\n    local MASTER_NODE=\"$1\"\n    local master_user_ip\n    local k8s_join_cmd\n    k8s_join_cmd=\"$(vm-join-cmd \"$MASTER_NODE\")\"\n    vm-command \"$k8s_join_cmd\" || {\n        command-error \"joining to the cluster master ($MASTER_NODE) failed\"\n    }\n    # Enable using kubectl on the worker vm by\n    # copying k8s admin configuration on it.\n    master_user_ip=\"$(vm-ssh-user-ip $MASTER_NODE)\"\n    ssh \"$master_user_ip\" \"sudo cat /etc/kubernetes/admin.conf\" | vm-pipe-to-file \"/root/.kube/config\"\n}\n\nvm-join-cmd() {\n    # Usage: vm-join-cmd MASTER_NODE\n    #\n    # Print a join command to join VM to existing cluster MASTER_NODE.\n    # MASTER_NODE is a name of a govm virtual machine (exists in \"govm ls\")\n    # or USERNAME@IP.\n    local MASTER_NODE=\"$1\"\n    local master_user_ip\n    local k8s_join_cmd=\"\"\n    master_user_ip=\"$(vm-ssh-user-ip $MASTER_NODE)\"\n    local ssh_get_join_cmd=\"ssh $master_user_ip sudo kubeadm token create --print-join-command\"\n    k8s_join_cmd=\"$( $ssh_get_join_cmd )\"\n    if [[ \"$k8s_join_cmd\" != *\" join \"* ]]; then\n        error \"failed to get kubeadm join command: $k8s_join_cmd\"\n    fi\n    echo $k8s_join_cmd\n}\n\nvm-mem-hotplug() { # script API\n    # Usage: vm-mem-hotplug MEMORY\n    #\n    # Hotplug currently unplugged MEMORY to VM.\n    # Find unplugged memory with \"vm-mem-hw | grep unplugged\".\n    #\n    # Examples:\n    #   vm-mem-hotplug mem2\n    local memmatch memline memid memdimm memnode memdriver\n    memmatch=$1\n    if [ -z \"$memmatch\" ]; then\n        error \"missing MEMORY\"\n        return 1\n    fi\n    memline=\"$(vm-mem-hw | grep unplugged | grep \"$memmatch\")\"\n    if [ -z \"$memline\" ]; then\n        error \"unplugged memory matching '$memmatch' not found\"\n        return 1\n    fi\n    memid=\"$(awk '{print $1}' <<< \"$memline\")\"\n    memid=${memid#mem}\n    memid=${memid%[: ]*}\n    memdimm=\"$(awk '{print $2}' <<< \"$memline\")\"\n    memnode=\"$(awk '{print $4}' <<< \"$memline\")\"\n    memnode=${memnode#node}\n    if [ \"$memdimm\" == \"nvdimm\" ]; then\n        memdriver=\"nvdimm\"\n    else\n        memdriver=\"pc-dimm\"\n    fi\n    vm-monitor \"device_add ${memdriver},id=${memdimm}${memid},memdev=mem${memdimm}_${memid}_node_${memnode},node=${memnode}\"\n}\n\nvm-mem-hotremove() { # script API\n    # Usage: vm-mem-hotremove MEMORY\n    #\n    # Hotremove currently plugged MEMORY from VM.\n    # Find plugged memory with \"vm-mem-hw | grep ' plugged'\".\n    #\n    # Examples:\n    #   vm-mem-hotremove mem2\n    local memmatch memline memid memdimm memnode memdriver\n    memmatch=$1\n    if [ -z \"$memmatch\" ]; then\n        error \"missing MEMORY\"\n        return 1\n    fi\n    memline=\"$(vm-mem-hw | grep \\ plugged | grep \"$memmatch\")\"\n    if [ -z \"$memline\" ]; then\n        error \"plugged memory matching '$memmatch' not found\"\n        return 1\n    fi\n    memid=\"$(awk '{print $1}' <<< \"$memline\")\"\n    memid=${memid#mem}\n    memid=${memid%[: ]*}\n    memdimm=\"$(awk '{print $2}' <<< \"$memline\")\"\n    vm-monitor \"device_del ${memdimm}${memid}\"\n}\n\nvm-mem-hw() { # script API\n    # Usage: vm-mem-hw\n    #\n    # List VM memory hardware with current status.\n    # See also: vm-mem-hotplug, vm-mem-hotremove\n    vm-monitor \"$(echo info memdev; echo info memory-devices)\" | awk '\n      /memdev: /{\n          split($2,a,\"_\");\n          state[a[2]]=\"plugged  \";\n      }\n      /memory backend: membuiltin/{\n          split($3,a,\"_\"); backend=1;\n          type[a[2]]=\"ram    \"; state[a[2]]=\"builtin  \"; node[a[2]]=a[4];\n      }\n      /memory backend: memnvbuiltin/{\n          split($3,a,\"_\"); backend=1;\n          type[a[2]]=\"nvram  \"; state[a[2]]=\"builtin  \"; node[a[2]]=a[4];\n      }\n      /memory backend: memnvdimm/{\n          split($3,a,\"_\"); backend=1;\n          type[a[2]]=\"nvdimm \"; state[a[2]]=\"unplugged\"; node[a[2]]=a[4];\n      }\n      /memory backend: memdimm/{\n          split($3,a,\"_\"); backend=1;\n          type[a[2]]=\"dimm   \"; state[a[2]]=\"unplugged\"; node[a[2]]=a[4];\n      }\n      /size: /{sz=$2/1024/1024; if (backend==1) {size[a[2]]=sz;backend=0;}}\n      END{\n          for (m in node) print \"mem\"m\": \"type[m]\" \"state[m]\" node\"node[m]\" size=\"size[m]\"M\";\n      }'\n}\n\nvm-monitor() { # script API\n    # Usage: vm-monitor COMMAND\n    #\n    # Execute COMMAND on Qemu monitor.\n    #\n    # Example: VM monitor help:\n    #  vm-monitor \"help\" | less\n    #\n    # Example: print memdev objects and plugged in memory devices:\n    #  vm-monitor \"info memdev\"\n    #  vm-monitor \"info memory-devices\"\n    #\n    # Example: hot plug a NVDIMM to NUMA node 1 when launched with topology\n    # topology='[{\"cores\":2,\"mem\":\"2G\"},{\"nvmem\":\"4G\",\"dimm\":\"unplugged\"}]':\n    #   vm-monitor \"device_add pc-dimm,id=nvdimm0,memdev=nvmem0,node=1\"\n    [ -n \"$VM_MONITOR\" ] ||\n        error \"VM is not running\"\n    eval \"$VM_MONITOR\" <<< \"$1\" | sed 's/\\r//g'\n    if [ \"${PIPESTATUS[0]}\" != \"0\" ]; then\n        error \"sending command to Qemu monitor failed\"\n    fi\n    echo \"\"\n}\n\nvm-wait-process() { # script API\n    # Usage: vm-wait-process [--timeout TIMEOUT] [--pidfile PIDFILE] PROCESS\n    #\n    # Wait for a PROCESS (string) to appear in process list (pidof output).\n    # If pidfile parameter is given, we also check that the process has that file open.\n    # The default TIMEOUT is 30 seconds.\n    local process timeout pidfile invalid\n    timeout=30\n    while [ \"${1#-}\" != \"$1\" ] && [ -n \"$1\" ]; do\n        case \"$1\" in\n            --timeout)\n                timeout=\"$2\"\n                shift 2\n                ;;\n            --pidfile)\n                pidfile=\"$2\"\n                shift 2\n                ;;\n            *)\n                invalid=\"${invalid}${invalid:+,}\\\"$1\\\"\"\n                shift\n                ;;\n        esac\n    done\n    if [ -n \"$invalid\" ]; then\n        error \"invalid options: $invalid\"\n        return 1\n    fi\n    process=\"$1\"\n    vm-run-until --timeout \"$timeout\" \"pidof \\\"$process\\\" > /dev/null\" || error \"timeout while waiting $process\"\n\n    # As we first wait for the process, and then wait for the pidfile (if enabled)\n    # we might wait longer than expected. Accept that anomaly atm.\n    if [ ! -z \"$pidfile\" ]; then\n\tvm-run-until --timeout $timeout \"[ ! -z \\\"\\$(fuser $pidfile 2>/dev/null)\\\" ]\" || error \"timeout while waiting $pidfile\"\n\tvm-run-until --timeout $timeout \"[ \\$(fuser $pidfile 2>/dev/null) -eq \\$(pidof $process) ]\" || error \"timeout while waiting $process and $pidfile\"\n    fi\n}\n\nvm-run-until() { # script API\n    # Usage: vm-run-until [--timeout TIMEOUT] CMD\n    #\n    # Keep running CMD (string) until it exits successfully.\n    # The default TIMEOUT is 30 seconds.\n    local cmd timeout invalid\n    timeout=30\n    while [ \"${1#-}\" != \"$1\" ] && [ -n \"$1\" ]; do\n        case \"$1\" in\n            --timeout)\n                timeout=\"$2\"\n                shift; shift\n                ;;\n            *)\n                invalid=\"${invalid}${invalid:+,}\\\"$1\\\"\"\n                shift\n                ;;\n        esac\n    done\n    if [ -n \"$invalid\" ]; then\n        error \"invalid options: $invalid\"\n        return 1\n    fi\n    cmd=\"$1\"\n    if ! vm-command-q \"retry=$timeout; until $cmd; do retry=\\$(( \\$retry - 1 )); [ \\\"\\$retry\\\" == \\\"0\\\" ] && exit 1; sleep 1; done\"; then\n        error \"waiting for command \\\"$cmd\\\" to exit successfully timed out after $timeout s\"\n    fi\n}\n\nvm-write-file() {\n    local vm_path_file=\"$1\"\n    local file_content_b64\n    file_content_b64=\"$(base64 <<<\"$2\")\"\n    vm-command-q \"mkdir -p $(dirname \"$vm_path_file\"); echo -n \\\"$file_content_b64\\\" | base64 -d > \\\"$vm_path_file\\\"\"\n}\n\nvm-put-file() { # script API\n    # Usage: vm-put-file [--cleanup] [--append] SRC-HOST-FILE DST-VM-FILE\n    #\n    # Copy SRC-HOST-FILE to DST-VM-FILE on the VM, removing\n    # SRC-HOST-FILE if called with the --cleanup flag, and\n    # appending instead of copying if the --append flag is\n    # specified.\n    #\n    # Example:\n    #   src=$(mktemp) && \\\n    #       echo 'Ahoy, Matey...' > $src && \\\n    #       vm-put-file --cleanup $src /etc/motd\n    local cleanup append invalid\n    while [ \"${1#-}\" != \"$1\" ] && [ -n \"$1\" ]; do\n        case \"$1\" in\n            --cleanup)\n                cleanup=1\n                shift\n                ;;\n            --append)\n                append=1\n                shift\n                ;;\n            *)\n                invalid=\"${invalid}${invalid:+,}\\\"$1\\\"\"\n                shift\n                ;;\n        esac\n    done\n    if [ -n \"$cleanup\" ] && [ -n \"$1\" ]; then\n        # shellcheck disable=SC2064\n        trap \"rm -f \\\"$1\\\"\" RETURN EXIT\n    fi\n    if [ -n \"$invalid\" ]; then\n        error \"invalid options: $invalid\"\n        return 1\n    fi\n    [ \"$(dirname \"$2\")\" == \".\" ] || vm-command-q \"[ -d \\\"$(dirname \"$2\")\\\" ]\" || vm-command \"mkdir -p \\\"$(dirname \"$2\")\\\"\" ||\n        command-error \"cannot create vm-put-file destination directory to VM\"\n    host-command \"$SCP \\\"$1\\\" ${VM_SSH_USER}@${VM_IP}:\\\"vm-put-file.${1##*/}\\\"\" ||\n        command-error \"failed to copy file to VM\"\n    if [ -z \"$append\" ]; then\n        vm-command \"mv \\\"vm-put-file.${1##*/}\\\" \\\"$2\\\"\" ||\n            command-error \"failed to rename file\"\n    else\n        vm-command \"touch \\\"$2\\\" && cat \\\"vm-put-file.${1##*/}\\\" >> \\\"$2\\\" && rm -f \\\"vm-put-file.${1##*/}\\\"\" ||\n            command-error \"failed to append file\"\n    fi\n}\n\nvm-put-pkg() { # script API\n    # Usage: vm-put-pkg [--force] HOST-FILE...\n    #\n    # Copies HOST-FILEs from host to vm and installs them.\n    #\n    # Examples:\n    #   vm-put-pkg /tmp/kernel.rpm /tmp/myutil.rpm\n    local host_pkg\n    local vm_pkgs=\"\"\n    local force=\"\"\n    if [ \"$1\" == \"--force\" ]; then\n        force=\"--force \"\n        shift\n    fi\n    for host_pkg in \"$@\"; do\n        local vm_pkg=\"pkgs/$(basename \"$host_pkg\")\"\n        vm-command-q \"mkdir -p $(dirname \"$vm_pkg\")\"\n        vm-put-file \"$host_pkg\" \"$vm_pkg\"\n        vm_pkgs=\"$vm_pkgs $vm_pkg\"\n    done\n    distro-install-pkg-local $force \"$vm_pkgs\"\n}\n\nvm-put-docker-image() { # script API\n    # Usage: vm-put-docker-image IMAGE\n    #\n    # Exports IMAGE from docker images on the host, and\n    # imports it in the \"k8s.io\" namespace (visible\n    # for kubernetes containers) on the vm.\n    #\n    # Works with containerd only.\n    #\n    # Examples:\n    #   vm-put-docker-image busybox:latest\n    local image_name=\"$1\"\n    local image_file_on_vm=\"images/${image_name//:/__}\"\n    vm-command-q \"mkdir -p $(dirname \"$image_file_on_vm\")\"\n    docker save \"$image_name\" | vm-pipe-to-file \"$image_file_on_vm\" ||\n        error \"failed to save and pipe image '$image_name'\"\n    vm-cri-import-image \"$image_name\" \"$image_file_on_vm\"\n}\n\nvm-pipe-to-file() { # script API\n    # Usage: vm-pipe-to-file [--append] DST-VM-FILE\n    #\n    # Reads stdin and writes the content to DST-VM-FILE, creating any\n    # intermediate directories necessary.\n    #\n    # Example:\n    #   echo 'Ahoy, Matey...' | vm-pipe-to-file /etc/motd\n    local tmp append\n    tmp=\"$(mktemp vm-pipe-to-file.XXXXXX)\"\n    if [ \"$1\" = \"--append\" ]; then\n        append=\"--append\"\n        shift\n    fi\n    cat > \"$tmp\"\n    vm-put-file --cleanup $append \"$tmp\" \"$1\"\n}\n\nvm-sed-file() { # script API\n    # Usage: vm-sed-file PATH-IN-VM SED-EXTENDED-REGEXP-COMMANDS\n    #\n    # Edits the given file in place with the given extended regexp\n    # sed commands.\n    #\n    # Example:\n    #   vm-sed-file /etc/motd 's/Matey/Guybrush Threepwood/'\n    local file=\"$1\" cmd\n    shift\n    for cmd in \"$@\"; do\n        vm-command \"sed -E -i \\\"$cmd\\\" $file\" ||\n            command-error \"failed to edit $file with sed\"\n    done\n}\n\nvm-set-kernel-cmdline() { # script API\n    # Usage: vm-set-kernel-cmdline E2E-DEFAULTS\n    #\n    # Adds/replaces E2E-DEFAULTS to kernel command line\"\n    #\n    # Example:\n    #   vm-set-kernel-cmdline nr_cpus=4\n    #   vm-reboot\n    #   vm-command \"cat /proc/cmdline\"\n    #   launch cri-resmgr\n    distro-set-kernel-cmdline \"$@\"\n}\n\nvm-reboot() { # script API\n    # Usage: vm-reboot\n    #\n    # Reboots the virtual machine and waits that the ssh server starts\n    # responding again.\n    vm-command \"reboot\"\n    sleep 10\n    if ! host-wait-vm-ssh-server; then\n        vm-monitor system_reset\n        host-wait-vm-ssh-server\n    fi\n}\n\nvm-setup-proxies() {\n    distro-setup-proxies\n}\n\nvm-networking() {\n    vm-command-q \"touch /etc/hosts; grep -q \\$(hostname) /etc/hosts\" || {\n        vm-command \"echo \\\"$VM_IP \\$(hostname)\\\" >>/etc/hosts\"\n    }\n\n    vm-setup-proxies\n}\n\nvm-install-cri-resmgr() {\n    prefix=/usr/local\n    # shellcheck disable=SC2154\n    if [ \"$binsrc\" == \"github\" ]; then\n        vm-install-golang\n        vm-install-pkg make\n        vm-command \"go get -d -v github.com/intel/cri-resource-manager\"\n        CRI_RESMGR_SOURCE_DIR=$(awk '/package.*cri-resource-manager/{print $NF}' <<< \"$COMMAND_OUTPUT\")\n        vm-command \"cd $CRI_RESMGR_SOURCE_DIR && make install && cd -\"\n    elif [ \"${binsrc#packages/}\" != \"$binsrc\" ]; then\n        suf=$(vm-pkg-type)\n        vm-command \"rm -f *.$suf\"\n        local pkg_count\n        # shellcheck disable=SC2010,SC2126\n        pkg_count=\"$(ls \"$HOST_PROJECT_DIR/$binsrc\"/cri-resource-manager*.\"$suf\" | grep -v dbg | wc -l)\"\n        if [ \"$pkg_count\" == \"0\" ]; then\n            error \"installing from $binsrc failed: cannot find cri-resource-manager_*.$suf from $HOST_PROJECT_DIR/$binsrc\"\n        elif [[ \"$pkg_count\" -gt 1 ]]; then\n            error \"installing from $binsrc failed: expected exactly one cri-resource-manager*.$suf in $HOST_PROJECT_DIR/$binsrc, found $pkg_count alternatives.\"\n        fi\n        vm-command \"mkdir -p /etc/cri-resmgr && touch /etc/cri-resmgr/fallback.cfg\"\n        host-command \"$SCP $HOST_PROJECT_DIR/$binsrc/*.$suf $VM_SSH_USER@$VM_IP:/tmp\" || {\n            command-error \"copying *.$suf to vm failed, run \\\"make cross-$suf\\\" first\"\n        }\n        vm-install-pkg \"/tmp/cri-resource-manager*.$suf\" || {\n            command-error \"installing packages failed\"\n        }\n        vm-command \"systemctl daemon-reload\"\n    elif [ -z \"$binsrc\" ] || [ \"$binsrc\" == \"local\" ]; then\n        vm-put-file \"$BIN_DIR/cri-resmgr\" \"$prefix/bin/cri-resmgr\"\n        vm-put-file \"$BIN_DIR/cri-resmgr-agent\" \"$prefix/bin/cri-resmgr-agent\"\n        sed -E -e \"s:__DEFAULTDIR__:$(distro-env-file-dir):g\" \\\n            -E -e \"s:__BINDIR__:$prefix/bin:g\" < \"$HOST_PROJECT_DIR/cmd/cri-resmgr/cri-resource-manager.service.in\" |\n            vm-pipe-to-file /usr/lib/systemd/system/cri-resource-manager.service\n        cat <<EOF |\nCONFIG_OPTIONS=\"--fallback-config /etc/cri-resmgr/fallback.cfg -relay-socket ${cri_resmgr_sock} -runtime-socket ${cri_sock} -image-socket ${cri_sock}\"\nEOF\n        vm-pipe-to-file \"$(distro-env-file-dir)/cri-resource-manager\"\n        vm-put-file \"$HOST_PROJECT_DIR/cmd/cri-resmgr/fallback.cfg.sample\" \"/etc/cri-resmgr/fallback.cfg\"\n    else\n        error \"vm-install-cri-resmgr: unknown binsrc=\\\"$binsrc\\\"\"\n    fi\n}\n\nvm-install-cri-resmgr-agent() {\n    prefix=/usr/local\n    local bin_change\n    local src_change\n    bin_change=$(stat --format \"%Z\" \"$BIN_DIR/cri-resmgr-agent\")\n    src_change=$(find \"$HOST_PROJECT_DIR\" -name '*.go' -type f -print0 | xargs -0 stat --format \"%Z\" | sort -n | tail -n 1)\n    if [[ \"$src_change\" > \"$bin_change\" ]]; then\n        echo \"WARNING:\"\n        echo \"WARNING: Source files changed - installing possibly outdated binaries from\"\n        echo \"WARNING: $BIN_DIR/\"\n        echo \"WARNING:\"\n        sleep \"${warning_delay:-0}\"\n    fi\n    vm-put-file \"$BIN_DIR/cri-resmgr-agent\" \"$prefix/bin/cri-resmgr-agent\"\n}\n\nvm-cri-import-image() {\n    local image_name=\"$1\"\n    local image_tar=\"$2\"\n    case \"$VM_CRI\" in\n        containerd)\n            vm-command \"ctr -n k8s.io images import '$image_tar'\" ||\n                command-error \"failed to import \\\"$image_tar\\\" on VM\"\n            ;;\n        *)\n            error \"vm-cri-import-image unsupported container runtime: \\\"$VM_CRI\\\"\"\n    esac\n}\n\nvm-install-cri-resmgr-webhook() {\n    local service=cri-resmgr-webhook\n    local namespace=cri-resmgr\n    vm-command-q \"\\\n        kubectl delete secret -n ${namespace} cri-resmgr-webhook-secret 2>/dev/null; \\\n        kubectl delete csr ${service}.${namespace} 2>/dev/null; \\\n        kubectl delete -f webhook/mutating-webhook-config.yaml 2>/dev/null; \\\n        kubectl delete -f webhook/webhook-deployment.yaml 2>/dev/null; \\\n        \"\n    local webhook_image_info webhook_image_id webhook_image_repotag webhook_image_tar\n    webhook_image_info=\"$(docker images --filter=reference=cri-resmgr-webhook --format '{{.ID}} {{.Repository}}:{{.Tag}} (created {{.CreatedSince}}, {{.CreatedAt}})' | head -n 1)\"\n    if [ -z \"$webhook_image_info\" ]; then\n        error \"cannot find cri-resmgr-webhook image on host, run \\\"make images\\\" and check \\\"docker images --filter=reference=cri-resmgr-webhook\\\"\"\n    fi\n    echo \"installing webhook to VM from image: $webhook_image_info\"\n    sleep 2\n    webhook_image_id=\"$(awk '{print $1}' <<< \"$webhook_image_info\")\"\n    webhook_image_repotag=\"$(awk '{print $2}' <<< \"$webhook_image_info\")\"\n    webhook_image_tar=\"$(realpath \"$OUTPUT_DIR/webhook-image-$webhook_image_id.tar\")\"\n    # It is better to export (save) the image with image_repotag rather than image_id\n    # because otherwise manifest.json RepoTags will be null and containerd will\n    # remove the image immediately after impoting it as part of garbage collection.\n    docker image save \"$webhook_image_repotag\" > \"$webhook_image_tar\"\n    vm-put-file \"$webhook_image_tar\" \"webhook/$(basename \"$webhook_image_tar\")\" || {\n        command-error \"copying webhook image to VM failed\"\n    }\n    vm-cri-import-image cri-resmgr-webhook \"webhook/$(basename \"$webhook_image_tar\")\"\n    # Create a self-signed certificate with SANs\n    vm-command \"openssl req -x509 -newkey rsa:2048 -sha256 -days 365 -nodes -keyout webhook/server-key.pem -out webhook/server-crt.pem -subj '/CN=${service}.${namespace}.svc' -addext 'subjectAltName=DNS:${service},DNS:${service}.${namespace},DNS:${service}.${namespace}.svc'\" ||\n        command-error \"creating self-signed certificate failed, requires openssl >= 1.1.1\"\n    # Allow webhook to run on node tainted by cmk=true\n    sed -e \"s|IMAGE_PLACEHOLDER|$webhook_image_repotag|\" \\\n        -e 's|^\\(\\s*\\)tolerations:$|\\1tolerations:\\n\\1  - {\"key\": \"cmk\", \"operator\": \"Equal\", \"value\": \"true\", \"effect\": \"NoSchedule\"}|g' \\\n        -e 's/imagePullPolicy: Always/imagePullPolicy: Never/' \\\n        < \"${HOST_PROJECT_DIR}/cmd/cri-resmgr-webhook/webhook-deployment.yaml\" \\\n        | vm-pipe-to-file webhook/webhook-deployment.yaml\n    # Create secret that contains svc.crt and svc.key for webhook deployment\n    local server_crt_b64 server_key_b64\n    server_crt_b64=\"$(vm-command-q \"cat webhook/server-crt.pem\" | base64 -w 0)\"\n    server_key_b64=\"$(vm-command-q \"cat webhook/server-key.pem\" | base64 -w 0)\"\n    cat <<EOF | vm-pipe-to-file --append webhook/webhook-deployment.yaml\n---\napiVersion: v1\nkind: Secret\nmetadata:\n  name: cri-resmgr-webhook-secret\n  namespace: cri-resmgr\ndata:\n  svc.crt: ${server_crt_b64}\n  svc.key: ${server_key_b64}\ntype: Opaque\nEOF\n    local cabundle_b64\n    cabundle_b64=\"$server_crt_b64\"\n    sed -e \"s/CA_BUNDLE_PLACEHOLDER/${cabundle_b64}/\" \\\n        < \"${HOST_PROJECT_DIR}/cmd/cri-resmgr-webhook/mutating-webhook-config.yaml\" \\\n        | vm-pipe-to-file webhook/mutating-webhook-config.yaml\n}\n\nvm-pkg-type() {\n    distro-pkg-type\n}\n\nvm-install-pkg() {\n    distro-install-pkg \"$@\"\n}\n\nvm-setup-oneshot() {\n    local util\n    ( distro-refresh-pkg-db ) || true\n    distro-setup-oneshot\n    distro-install-utils\n    # Verify that all required utilities exit on the VM.\n    for util in pidof killall; do\n        vm-command-q \"command -v $util >/dev/null\" || {\n            error \"required command '$util' missing on VM, fix/implement $distro-install-utils()\"\n        }\n    done\n}\n\nvm-install-golang() {\n    distro-install-golang\n}\n\nvm-install-runc() {\n    local host_runc=\"$runc_src/runc\"\n    local vm_runc=\"/usr/sbin/runc\"\n    if [ -n \"$runc_src\" ]; then\n        # Check if runc is already installed on VM.\n        # If it is, replace existing binary with local build.\"\n        vm-command 'command -v runc'\n        if [ -n \"$COMMAND_OUTPUT\" ] && [ \"x$COMMAND_STATUS\" == \"x0\" ]; then\n            vm_runc=\"$COMMAND_OUTPUT\"\n        fi\n        vm-put-file \"$host_runc\" \"$vm_runc\"\n    else\n        distro-install-runc\n    fi\n}\n\nvm-install-cri() {\n    local vm_cri_dir=\"/usr/bin\"\n    distro-install-\"$VM_CRI\"\n    distro-config-\"$VM_CRI\"\n    if [ \"$VM_CRI\" == \"containerd\" ]; then\n        if [ -n \"$containerd_src\" ]; then\n            vm-command \"systemctl stop containerd\"\n            vm-command 'command -v containerd'\n            if [ -n \"$COMMAND_OUTPUT\" ] && [ \"x$COMMAND_STATUS\" == \"x0\" ]; then\n                vm_cri_dir=\"${COMMAND_OUTPUT%/*}\"\n            fi\n            for f in ctr containerd containerd-stress containerd-shim containerd-shim-runc-v1 containerd-shim-runc-v2; do\n                vm-put-file \"$containerd_src/bin/$f\" \"$vm_cri_dir/$f\"\n            done\n            vm-command \"mkdir -p /etc/containerd; containerd config default | sed -e 's/SystemdCgroup = false/SystemdCgroup = true/g' > /etc/containerd/config.toml\"\n            vm-command \"systemctl enable --now containerd\"\n        fi\n    elif [ \"$VM_CRI\" == \"crio\" ]; then\n        if [ -n \"$crio_src\" ]; then\n            vm-command \"systemctl stop crio\"\n            vm-command 'command -v crio'\n            if [ -n \"$COMMAND_OUTPUT\" ] && [ \"x$COMMAND_STATUS\" == \"x0\" ]; then\n                vm_cri_dir=\"${COMMAND_OUTPUT%/*}\"\n            fi\n            for f in crio crio-status pinns; do\n                vm-put-file \"$crio_src/bin/$f\" \"$vm_cri_dir/$f\"\n            done\n            vm-command \"systemctl enable --now crio\"\n        fi\n    fi\n}\n\nvm-install-containernetworking() {\n    vm-install-golang\n    vm-command \"GO111MODULE=off go get -d github.com/containernetworking/plugins\"\n    CNI_PLUGINS_SOURCE_DIR=\"$(awk '/package.*plugins/{print $NF}' <<< \"$COMMAND_OUTPUT\")\"\n    [ -n \"$CNI_PLUGINS_SOURCE_DIR\" ] || {\n        command-error \"downloading containernetworking plugins failed\"\n    }\n    vm-command \"pushd \\\"$CNI_PLUGINS_SOURCE_DIR\\\" && ./build_linux.sh && mkdir -p /opt/cni && cp -rv bin /opt/cni && popd\" || {\n        command-error \"building and installing cri-tools failed\"\n    }\n    vm-command \"rm -rf /etc/cni/net.d && mkdir -p /etc/cni/net.d && cat > /etc/cni/net.d/10-bridge.conf <<EOF\n{\n  \\\"cniVersion\\\": \\\"0.4.0\\\",\n  \\\"name\\\": \\\"mynet\\\",\n  \\\"type\\\": \\\"bridge\\\",\n  \\\"bridge\\\": \\\"cni0\\\",\n  \\\"isGateway\\\": true,\n  \\\"ipMasq\\\": true,\n  \\\"ipam\\\": {\n    \\\"type\\\": \\\"host-local\\\",\n    \\\"subnet\\\": \\\"$CNI_SUBNET\\\",\n    \\\"routes\\\": [\n      { \\\"dst\\\": \\\"0.0.0.0/0\\\" }\n    ]\n  }\n}\nEOF\"\n    vm-command 'cat > /etc/cni/net.d/20-portmap.conf <<EOF\n{\n    \"cniVersion\": \"0.4.0\",\n    \"type\": \"portmap\",\n    \"capabilities\": {\"portMappings\": true},\n    \"snat\": true\n}\nEOF'\n    vm-command 'cat > /etc/cni/net.d/99-loopback.conf <<EOF\n{\n  \"cniVersion\": \"0.4.0\",\n  \"name\": \"lo\",\n  \"type\": \"loopback\"\n}\nEOF'\n}\n\nvm-install-dlv() {\n    vm-install-golang\n    vm-install-pkg rsync\n    vm-command \"go install github.com/go-delve/delve/cmd/dlv@latest\" || {\n        command-error \"installing delve failed\"\n    }\n    echo '[ \"`id -u`\" -eq 0 ] && PATH=$PATH:/root/go/bin' | vm-pipe-to-file /etc/profile.d/root-path-go.sh\n    vm-command \"mkdir -p \\\"\\$HOME/.config/dlv/config.yml.d\\\"\"\n    vm-command \"echo 'substitute-path:' > \\\"\\$HOME/.config/dlv/config.yml.d/00-substitute-path\\\"\"\n}\n\nvm-install-glibc() { # script API\n    # Usage: vm-install-glibc [VERSION]\n    #\n    # If glibc_src=/host/path/to/glibc is set, install a glibc that is\n    # built and installed on host using configure --prefix $glibc_src.\n    # If glibc_src is not set, download, build and install a glibc on vm.\n    # In both cases glibc is installed to /opt/glibc/VERSION on vm.\n    #\n    # vm-set-glibc wraps selected binaries to use an installed glibc.\n    #\n    # Example: install a glibc from host and use it with two binaries.\n    #   glibc_src=/host/glibc/install/prefix vm-install-glibc host-2.34\n    #   vm-set-glibc host-2.34 /usr/bin/containerd /usr/local/bin/cri-resmgr\n    #\n    # Example: download, build and install glibc 2.32 on vm:\n    #   vm-install-glibc 2.32\n    #   vm-set-glibc 2.32 /usr/bin/containerd /usr/local/bin/cri-resmgr\n    local glibc_ver=\"${1:-host}\"\n    local vm_glibc_dir=\"/opt/glibc/${glibc_ver}\"\n    if [ -n \"$glibc_src\" ] && [ -d \"$glibc_src\" ]; then\n        vm-command \"mkdir -p $vm_glibc_dir\"\n        ( cd \"$glibc_src\" && tar cz . ) | vm-pipe-to-file \"$vm_glibc_dir/glibc-$glibc_ver.tar.gz\" ||\n            error \"failed to package glibc from '$glibc_src'\"\n        vm-command \"cd $vm_glibc_dir && tar xf glibc-$glibc_ver.tar.gz && rm -f glibc-$glibc_ver.tar.gz\" ||\n            command-error \"failed to extract glibc-$glibc_ver.tar.gz\"\n        return 0\n    fi\n    if [[ \"$glibc_ver\" == \"host\"* ]]; then\n        error \"vm-install-glibc: invalid glibc_src='$glibc_src' when installing glibc from host\"\n    fi\n    local vm_glibc_src=\"$vm_glibc_dir/src/glibc-${glibc_ver}\"\n    local vm_glibc_build=\"$vm_glibc_dir/src/build\"\n    local vm_glibc_install=\"$vm_glibc_dir\"\n    vm-install-pkg make bison flex gcc\n    vm-command \"mkdir -p $vm_glibc_src; cd $vm_glibc_src; curl -L --remote-name-all https://ftp.gnu.org/gnu/glibc/glibc-${glibc_ver}.tar.gz\" ||\n        command-error \"failed to download glibc\"\n    vm-command \"mkdir -p $vm_glibc_src; cd $vm_glibc_src/..; tar xzf $vm_glibc_src/glibc-${glibc_ver}.tar.gz\" ||\n        command-error \"failed to extract glibc\"\n    vm-command \"mkdir -p $vm_glibc_build; cd $vm_glibc_build && $vm_glibc_src/configure --prefix=$vm_glibc_install\" ||\n        command-error \"failed to configure glibc\"\n    vm-command \"cd $vm_glibc_build && make -j 4 >make.output.txt 2>&1 || ( tail make.output.txt; exit 1 )\" ||\n        command-error \"failed to build glibc, see $vm_glibc_build/make.output.txt\"\n    vm-command \"cd $vm_glibc_build && make install\" ||\n        command-error \"failed to install glibc\"\n}\n\nvm-set-glibc() { # script API\n    # Usage: vm-set-glibc VERSION BIN [BIN...]\n    #\n    # Wrap binaries to use glibc VERSION.\n    #\n    # Note glibc VERSION must be installed first.\n    # See vm-install-glibc.\n    local glibc_ver=\"$1\"\n    local vm_glibc_dir=\"/opt/glibc/${glibc_ver}\"\n    local vm_glibc_install=\"$vm_glibc_dir\"\n    local vm_glibc_ld=\"$vm_glibc_install/lib/ld-linux-x86-64.so.2\"\n    shift\n    if [ -z \"$glibc_ver\" ]; then\n        error \"vm-switch-glibc: missing glibc version to switch to\"\n    fi\n    vm-command \"[ -x $vm_glibc_ld ]\" ||\n        command-error \"cannot find loader $vm_glibc_ld\"\n    local vm_bin\n    for vm_bin in \"$@\"; do\n        vm-command \"[ -x $vm_bin ]\" ||\n            command-error \"cannot find binary to be wrapped: $vm_bin\"\n        vm-command \"( [ \\\"\\$(dd bs=1 count=3 skip=1 if=$vm_bin)\\\" == \\\"ELF\\\" ] && mv $vm_bin ${vm_bin}.bin ) || [ -f $vm_bin.bin ]\" ||\n            command-error \"failed to rename binary\"\n        vm-pipe-to-file \"$vm_bin\" <<EOF\n#!/bin/bash\nLD_LIBRARY_PATH=$vm_glibc_install/lib:\\$LD_LIBRARY_PATH exec $vm_glibc_ld ${vm_bin}.bin \"\\$@\"\nEOF\n        vm-command \"chmod a+rx $vm_bin\"\n    done\n}\n\nvm-dlv-add-src() {\n    local host_src_dir=\"$1\"\n    [ -d \"$host_src_dir\" ] || error \"vm-dlv-add-src: invalid source directory \\\"$host_src_dir\\\", existing go project directory expected\"\n    vm-command \"mkdir -p /home/$VM_SSH_USER/src; chmod a+rwX /home/$VM_SSH_USER/src; mkdir -p \\$HOME/.config/dlv/config.yml.d\"\n    host-command \"cd \\\"$host_src_dir/..\\\" && rsync -avz --include \\\"*/\\\" --include \\\"**/*.go\\\" --exclude \\\"*\\\" \\\"$(basename \"$host_src_dir\")\\\" $VM_SSH_USER@$VM_IP:src/\"\n    vm-command \"echo ' - {from: \\\"$host_src_dir\\\", to: \\\"/home/$VM_SSH_USER/src/$(basename \"$host_src_dir\")\\\"}' > \\\"\\$HOME/.config/dlv/config.yml.d/01-$(basename \"$host_src_dir\")\\\"\"\n    vm-dlv-update-config\n}\n\nvm-dlv-update-config() {\n    vm-command \"( echo 'substitute-path:'; cat \\$HOME/.config/dlv/config.yml.d/* ) > \\$HOME/.config/dlv/config.yml\"\n}\n\nvm-install-k8s() {\n    distro-install-k8s\n    distro-restart-$VM_CRI\n}\n\nvm-install-minikube() {\n    vm-install-containernetworking\n    distro-install-cri-dockerd\n    distro-install-minikube\n}\n\nvm-create-minikube-cluster() {\n    vm-command \"sysctl fs.protected_regular=0; minikube start --driver=none --alsologtostderr=true\"\n}\n\nvm-create-singlenode-cluster() {\n    if ! [ \"$(type -t vm-install-cni-$(distro-k8s-cni))\" == \"function\" ]; then\n        error \"invalid CNI: $(distro-k8s-cni)\"\n    fi\n    vm-create-cluster\n    vm-command \"kubectl taint nodes --all node-role.kubernetes.io/control-plane-\"\n    vm-command \"kubectl taint nodes --all node-role.kubernetes.io/master-\"\n    vm-install-cni-\"$(distro-k8s-cni)\"\n    if ! vm-command \"kubectl wait --for=condition=Ready node/\\$(hostname) --timeout=240s\"; then\n        command-error \"kubectl waiting for node readiness timed out\"\n    fi\n    vm-run-until --timeout 30 \"kubectl get sa default > /dev/null\" || error \"serviceaccount 'default' not found\"\n}\n\nvm-create-cluster() {\n    vm-command \"kubeadm init --pod-network-cidr=$CNI_SUBNET --cri-socket ${k8scri_sock}\"\n    if ! grep -q \"initialized successfully\" <<< \"$COMMAND_OUTPUT\"; then\n        command-error \"kubeadm init failed\"\n    fi\n\n    user=\"$(vm-ssh-user)\"\n\n    vm-command \"mkdir -p ~$user/.kube\"\n    vm-command \"cp /etc/kubernetes/admin.conf ~$user/.kube/config\"\n    vm-command \"chown -R $user:$user ~$user/.kube\"\n    vm-command \"mkdir -p ~root/.kube\"\n    vm-command \"cp /etc/kubernetes/admin.conf ~root/.kube/config\"\n}\n\nvm-destroy-cluster() {\n    user=\"$(vm-ssh-user)\"\n    vm-command \"yes | kubeadm reset; rm -f ~$user/.kube/config ~root/.kube/config /etc/kubernetes\"\n}\n\nvm-install-cni-bridge() {\n    vm-command \"rm -rf /etc/cni/net.d/* && mkdir -p /etc/cni/net.d && cat > /etc/cni/net.d/10-bridge.conf <<EOF\n{\n  \\\"cniVersion\\\": \\\"0.4.0\\\",\n  \\\"name\\\": \\\"demonet\\\",\n  \\\"type\\\": \\\"bridge\\\",\n  \\\"isGateway\\\": true,\n  \\\"ipMasq\\\": true,\n  \\\"ipam\\\": {\n    \\\"type\\\": \\\"host-local\\\",\n    \\\"subnet\\\": \\\"$CNI_SUBNET\\\",\n    \\\"routes\\\": [\n      { \\\"dst\\\": \\\"0.0.0.0/0\\\" }\n    ]\n  }\n}\nEOF\"\n}\n\nvm-install-cni-cilium() {\n    if ! vm-command \"curl -L --remote-name-all https://github.com/cilium/cilium-cli/releases/latest/download/cilium-linux-amd64.tar.gz && tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin && cilium install && rm -f cilium-linux-amd64.tar.gz\"; then\n        command-error \"installing cilium CNI to Kubernetes failed\"\n    fi\n}\n\nvm-install-cni-weavenet() {\n    vm-command \"kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml\"\n    if ! vm-command \"kubectl rollout status --timeout=360s -n kube-system daemonsets/weave-net\"; then\n        command-error \"installing weavenet CNI to Kubernetes failed/timed out\"\n    fi\n}\n\nvm-install-cni-flannel() {\n    vm-command \"kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml\"\n    if ! vm-command \"kubectl rollout status --timeout=360s -n kube-system daemonsets/kube-flannel-ds\"; then\n        command-error \"installing flannel CNI to Kubernetes failed/timed out\"\n    fi\n}\n\nvm-install-kernel-dev() { # script API\n    # Usage: vm-install-kernel-dev\n    #\n    # Install dependencies and kernel sources ready for patching,\n    # configuring and building packages.\n    distro-install-kernel-dev\n}\n\nvm-print-usage() {\n    echo \"- Login VM:     ssh $VM_SSH_USER@$VM_IP\"\n    echo \"- Stop VM:      govm stop $VM_NAME\"\n    echo \"- Delete VM:    govm delete $VM_NAME\"\n}\n\nvm-check-env || exit 1\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.debian-11",
    "content": "# pull in base + a minimal set of useful packages\nFROM debian:bullseye as debian-11-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\n\n# pull in stuff for cgo\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.debian-12",
    "content": "# pull in base + a minimal set of useful packages\nFROM debian:bookworm as debian-11-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\n\n# pull in stuff for cgo\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.debian-sid",
    "content": "# pull in base + a minimal set of useful packages\nFROM debian:sid as debian-sid-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\n\n# pull in stuff for cgo\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.fedora",
    "content": "# pull in base + a minimal set of useful packages\nFROM fedora:latest as fedora-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"build\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\n\nRUN dnf install -y rpm-build systemd-rpm-macros \\\n    kernel-devel gcc \\\n    git-core make\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.opensuse-leap-15.6",
    "content": "# pull in base + a minimal set of useful packages\nFROM opensuse/leap:15.6 as suse-15.6-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"build\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\n\nRUN zypper install -y rpm-build \\\n    kernel-devel gcc \\\n    git make\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.ubuntu-18.04",
    "content": "# pull in base + a minimal set of useful packages\nFROM ubuntu:18.04 as ubuntu-18.04-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\n\n# pull in stuff for cgo\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.ubuntu-20.04",
    "content": "# pull in base + a minimal set of useful packages\nFROM ubuntu:20.04 as ubuntu-20.04-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\nENV DEBIAN_FRONTEND noninteractive\n\n# pull in stuff for building\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        tzdata build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.ubuntu-22.04",
    "content": "# pull in base + a minimal set of useful packages\nFROM ubuntu:22.04 as ubuntu-22.04-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\nENV DEBIAN_FRONTEND noninteractive\n\n# pull in stuff for building\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        tzdata build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ] && \\\n    useradd -m -s /bin/bash $CREATE_USER -u $USER_UID\n"
  },
  {
    "path": "dockerfiles/cross-build/Dockerfile.ubuntu-24.04",
    "content": "# pull in base + a minimal set of useful packages\nFROM ubuntu:24.04 as ubuntu-24.04-build\n\nARG GO_VERSION=x.yz\nARG GOLICENSES_VERSION\nARG CREATE_USER=\"test\"\nARG USER_UID=\"\"\nENV PATH /go/bin:/usr/local/go/bin:$PATH\nENV DEBIAN_FRONTEND noninteractive\n\n# pull in stuff for building\nRUN apt-get update && \\\n    apt-get install -y --no-install-recommends \\\n        tzdata build-essential fakeroot devscripts \\\n        bash git make sed debhelper ca-certificates && \\\n    rm -rf /var/lib/apt/lists/*\n\nADD http://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz /\n\nRUN tar xf /go${GO_VERSION}.linux-amd64.tar.gz -C \"/usr/local\" && \\\n    rm /go${GO_VERSION}.linux-amd64.tar.gz\n\nRUN GOBIN=/go/bin go install github.com/google/go-licenses@${GOLICENSES_VERSION}\n\nRUN if [ -n \"$CREATE_USER\" -a \"$CREATE_USER\" != \"root\" ]; then \\\n        if getent passwd $USER_UID; then \\\n             userdel `id -un $USER_UID`; \\\n        fi; \\\n        useradd -m -s /bin/bash $CREATE_USER -u $USER_UID; \\\n    fi\n"
  },
  {
    "path": "docs/Dockerfile",
    "content": "FROM sphinxdoc/sphinx:5.3.0\n\nRUN apt-get update && apt-get install -y wget git\n\n# Note: Any golang version that can 'go list -m -f {{.Variable}}' is fine...\nADD https://go.dev/dl/go1.24.1.linux-amd64.tar.gz /\n\nRUN tar -C /usr/local -xzf /go1.24.1.linux-amd64.tar.gz && \\\n    rm /go1*.linux-amd64.tar.gz\n\nENV PATH=$PATH:/usr/local/go/bin\n\nCOPY requirements.txt .\n\nRUN pip3 install -r requirements.txt\n"
  },
  {
    "path": "docs/_templates/layout.html",
    "content": "{%- extends \"!layout.html\" %}\n\n{% block footer %}\n  {% if versions_menu %}\n    <div class=\"rst-versions\" data-toggle=\"rst-versions\" role=\"note\" aria-label=\"versions\">\n      <span class=\"rst-current-version\" data-toggle=\"rst-current-version\">\n        <span class=\"fa fa-book\"> GitHub Pages</span>\n        {{ versions_menu_this_version }}\n        <span class=\"fa fa-caret-down\"></span>\n      </span>\n      <div class=\"rst-other-versions\">\n        <dl id=\"versions\">\n          <dt>{{ _('Versions') }}</dt>\n        </dl>\n        <dl>\n          <dt>\n          <a href=\"/cri-resource-manager/releases\">all releases</a>\n          </dt>\n        </dl>\n      </div>\n    </div>\n  {% endif %}\n  <script src=\"{{ pathto('../versions.js', 1) }}\"></script>\n  <script>\n      var list = document.getElementById('versions')\n      var menuItems = getVersionsMenuItems();\n      for (var i=0; i < menuItems.length; i++) {\n        var item = document.createElement('dd');\n        var anchor = item.appendChild(document.createElement('a'));\n        anchor.appendChild(document.createTextNode(menuItems[i].name));\n        anchor.href = menuItems[i].url;\n        list.appendChild(item);\n      }\n  </script>\n{% endblock %}\n"
  },
  {
    "path": "docs/conf.py",
    "content": "# Configuration file for the Sphinx documentation builder.\n#\n# This file only contains a selection of the most common options. For a full\n# list see the documentation:\n# https://www.sphinx-doc.org/en/master/usage/configuration.html\n\n# -- Path setup --------------------------------------------------------------\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\n# import os\n# import sys\n# sys.path.insert(0, os.path.abspath('.'))\nfrom docutils import nodes\nfrom os.path import isdir, isfile, join, basename, dirname\nfrom os import makedirs, getenv\nfrom shutil import copyfile\nfrom subprocess import run, STDOUT\n\n# -- Project information -----------------------------------------------------\n\nproject = 'CRI Resource Manager'\ncopyright = '2020, various'\nauthor = 'various'\n\nmaster_doc = 'docs/index'\n\n\n##############################################################################\n#\n# This section determines the behavior of links to local items in .md files.\n#\n#  if useGitHubURL == True:\n#\n#     links to local files and directories will be turned into github URLs\n#     using either the baseBranch defined here or using the commit SHA.\n#\n#  if useGitHubURL == False:\n#\n#     local files will be moved to the website directory structure when built\n#     local directories will still be links to github URLs\n#\n#  if built with GitHub workflows:\n#\n#     the GitHub URLs will use the commit SHA (GITHUB_SHA environment variable\n#     is defined by GitHub workflows) to link to the specific commit.\n#\n##############################################################################\n\nbaseBranch = \"master\"\nuseGitHubURL = True\ncommitSHA = getenv('GITHUB_SHA')\ngithubServerURL = getenv('GITHUB_SERVER_URL')\ngithubRepository = getenv('GITHUB_REPOSITORY')\nif githubServerURL and githubRepository:\n    githubBaseURL = join(githubServerURL, githubRepository)\nelse:\n    githubBaseURL = \"https://github.com/intel/cri-resource-manager/\"\n\ngithubFileURL = join(githubBaseURL, \"blob/\")\ngithubDirURL = join(githubBaseURL, \"tree/\")\nif commitSHA:\n    githubFileURL = join(githubFileURL, commitSHA)\n    githubDirURL = join(githubDirURL, commitSHA)\nelse:\n    githubFileURL = join(githubFileURL, baseBranch)\n    githubDirURL = join(githubDirURL, baseBranch)\n\n# Version displayed in the upper left corner of the site\nref = getenv('GITHUB_REF', default=\"\")\nif ref == \"refs/heads/master\":\n    version = \"devel\"\nelif ref.startswith(\"refs/heads/release-\"):\n    # For release branches just show the latest tag name\n    buildVersion = getenv(\"BUILD_VERSION\", default=\"unknown\")\n    version = buildVersion.split('-')[0]\nelif ref.startswith(\"refs/tags/\"):\n    version = ref[len(\"refs/tags/\"):]\nelse:\n    version = getenv(\"BUILD_VERSION\", default=\"unknown\")\n\nrelease = getenv(\"BUILD_VERSION\", default=\"unknown\")\n\n# Versions to show in the version menu\nif getenv('VERSIONS_MENU'):\n    html_context = {\n        'versions_menu': True,\n        'versions_menu_this_version': getenv('VERSIONS_MENU_THIS_VERSION', version)}\n\n# -- General configuration ---------------------------------------------------\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = ['myst_parser', 'sphinx_markdown_tables']\nmyst_enable_extensions = ['substitution']\nsource_suffix = {'.rst': 'restructuredtext','.md': 'markdown'}\n\n# Substitution variables\ndef module_version(module, version):\n    version=version.split('-', 1)[0]\n    if module == 'github.com/intel/goresctrl':\n        version = '.'.join(version.split('.')[0:2]) + '.0'\n    return version\n\ndef gomod_versions(modules):\n    versions = {}\n    gocmd = run(['go', 'list', '-m', '-f', '{{.GoVersion}}'],\n                check=True, capture_output=True, universal_newlines=True)\n    versions['golang'] = gocmd.stdout.strip()\n    for m in modules:\n        gocmd = run(['go', 'list', '-m', '-f', '{{.Version}}', '%s' % m],\n                    check=True, capture_output=True, universal_newlines=True)\n        versions[m] = module_version(m, gocmd.stdout.strip())\n    return versions\n\nmod_versions = gomod_versions(['github.com/intel/goresctrl'])\nmyst_substitutions = {\n    'golang_version': mod_versions['golang'],\n    'goresctrl_version': mod_versions['github.com/intel/goresctrl']\n}\nmyst_heading_anchors = 3\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This pattern also affects html_static_path and html_extra_path.\nexclude_patterns = ['_build', '.github', '_work', 'generate', 'README.md', 'SECURITY.md', 'docs/releases']\n\n\n# -- Options for HTML output -------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages.  See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'sphinx_rtd_theme'\n\nhtml_theme_options = {\n    'display_version': True,\n}\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\n#html_static_path = ['_static']\n\ndef setup(app):\n    app.connect('doctree-resolved',fixLocalMDAnchors)\n    app.connect('missing-reference',fixRSTLinkInMD)\n\n###############################################################################\n#\n#  This section defines callbacks that make markdown specific tweaks to\n#  either:\n#\n#  1. Fix something that recommonmark does wrong.\n#  2. Provide support for .md files that are written as READMEs in a GitHub\n#     repo.\n#\n#  Only use these changes if using the extension ``recommonmark``.\n#\n###############################################################################\n\ndef isHTTPLink(uri):\n    return uri.startswith('http://') or uri.startswith('https://')\n\ndef isMDFileLink(uri):\n    return uri.endswith('.md') or '.md#' in uri\n\ndef isRSTFileLink(uri):\n    return uri.endswith('.rst')\n\n# Callback registerd with 'missing-reference'.\ndef fixRSTLinkInMD(app, env, node, contnode):\n    refTarget = node.get('reftarget')\n\n    if isHTTPLink(refTarget):\n        return\n\n    if isRSTFileLink(refTarget) and not isHTTPLink(refTarget):\n    # This occurs when a .rst file is referenced from a .md file\n    # Currently unable to check if file exists as no file\n    # context is provided and links are relative.\n    #\n    # Example: [Application examples](examples/readme.rst)\n    #\n        contnode['refuri'] = contnode['refuri'].replace('.rst','.html')\n        contnode['internal'] = \"True\"\n        return contnode\n    elif refTarget.startswith(\"/\"):\n    # This occurs when a file is referenced for download from an .md file.\n    # Construct a list of them and short-circuit the warning. The files\n    # are moved later (need file location context). To avoid warnings,\n    # write .md files, make the links absolute. This only marks them fixed\n    # if it can verify that they exist.\n    #\n    # Example: [Makefile](/Makefile)\n    #\n        filePath = refTarget.lstrip(\"/\")\n        if isfile(filePath) or isdir(filePath):\n            return contnode\n\n\ndef normalizePath(docPath,uriPath):\n    if uriPath == \"\":\n        return uriPath\n    if \"#\" in uriPath:\n    # Strip out anchors\n        uriPath = uriPath.split(\"#\")[0]\n    if uriPath.startswith(\"/\"):\n    # It's an absolute path\n        return uriPath.lstrip(\"/\") #path to file from project directory\n    else:\n    # It's a relative path\n        docDir = dirname(docPath)\n        return join(docDir,uriPath) #path to file from referencing file\n\n\n# Callback registerd with 'doctree-resolved'.\ndef fixLocalMDAnchors(app, doctree, docname):\n    for node in doctree.traverse(nodes.reference):\n        uri = node.get('refuri')\n        if uri is None:\n             print(\"fixLocalMDAnchor: skipping anchor with no URI at node: \", node)\n             continue\n\n        if isHTTPLink(uri):\n            continue\n\n        filePath = normalizePath(docname,uri)\n\n        if isfile(filePath):\n        # Only do this if the file exists.\n        #\n        # TODO: Pop a warning if the file doesn't exist.\n        #\n            if isMDFileLink(uri) and not isHTTPLink(uri):\n            # Make sure .md file links that weren't caught are converted.\n            # These occur when creating an explicit link to an .md file\n            # from an .rst file. By default these are not validated by Sphinx\n            # or recommonmark. Only toctree references are validated. recommonmark\n            # also fails to convert links to local Markdown files that include\n            # anchors. This fixes that as well.\n            #\n            # Only include this code if .md files are being converted to html\n            #\n            # Example: `Google Cloud Engine <gce.md>`__\n            #          [configuration options](autotest.md#configuration-options)\n            #\n                node['refuri'] = node['refuri'].replace('.md','.html')\n            else:\n            # Handle the case where markdown is referencing local files in the repo\n            #\n            # Example: [Makefile](/Makefile)\n            #\n                if useGitHubURL:\n                # Replace references to local files with links to the GitHub repo\n                #\n                    newURI = join(githubFileURL, filePath)\n                    print(\"new url: \", newURI)\n                    node['refuri']=newURI\n                else:\n                # If there are links to local files other than .md (.rst files are caught\n                # when warnings are fired), move the files into the Sphinx project, so\n                # they can be accessed.\n                    newFileDir = join(app.outdir,dirname(filePath)) # where to move the file in Sphinx output.\n                    newFilePath = join(app.outdir,filePath)\n                    newURI = uri # if the path is relative no need to change it.\n                    if uri.startswith(\"/\"):\n                    # It's an absolute path. Need to make it relative.\n                        uri = uri.lstrip(\"/\")\n                        docDirDepth = len(docname.split(\"/\")) - 1\n                        newURI = \"../\"*docDirDepth + uri\n                    if not isdir(newFileDir):\n                        makedirs(newFileDir)\n                    copyfile(filePath,newFilePath)\n                    node['refuri'] = newURI\n        elif \"#\" not in uri: # ignore anchors\n        # turn links to directories into links to the repo\n            if isdir(filePath):\n                newURI = join(githubDirURL, filePath)\n                node['refuri']=newURI\n"
  },
  {
    "path": "docs/contributing.md",
    "content": "# Contributing\n\nPlease use the GitHub\\* infrastructure for contributing to\nCRI Resource Manager.\nUse [pull requests](https://github.com/intel/cri-resource-manager/pulls)\nto contribute code, bug fixes, or if you want to discuss your ideas in terms of\ncode. Open [issues](https://github.com/intel/cri-resource-manager/issues) to\nreport bugs, request new features, or if you want to discuss any other topics\nrelated to CRI Resource Manager or orchestration resource management in\ngeneral.\n"
  },
  {
    "path": "docs/demos/blockio.md",
    "content": "# Block I/O Demo\n\nThis demo creates a virtual machine for a single-node Kubernetes\\*\ncluster where container runtime features are extended by `cri-resmgr`.\n\nIn this setup, `cri-resmgr` is configured with block I/O parameters\nthat throttle I/O bandwith of a container that constantly scans\nsystem file checksums.\n\n## Prerequisites\n\nInstall:\n- `docker`\n- `govm`\n\n## Run the demo\n\n```\n./run.sh play\n```\n\nThe demo does not delete the virtual machine so that you can\nexperiment with it. You can login to the virtual machine:\n\n```\n$ govm ssh crirm-demo-blockio\n```\n\n## Clean up - and run the demo from scratch\n\nIn order to run the demo from scratch again, delete the virtual\nmachine:\n\n```\n$ govm delete crirm-demo-blockio\n```\n"
  },
  {
    "path": "docs/demos/index.rst",
    "content": "Demos\n#####\n.. toctree::\n  :maxdepth: 1\n\n  blockio.md\n\n"
  },
  {
    "path": "docs/developers-guide/architecture.md",
    "content": "# Architecture\n\n## Overview\n\nCRI Resource Manager (CRI-RM) is a pluggable add-on for controlling how much\nand which resources are assigned to containers in a Kubernetes\\* cluster.\nIt's an add-on because you install it in addition to the normal selection of\nyour components. It's pluggable since you inject it on the signaling path\nbetween two existing components with the rest of the cluster unaware of its\npresence.\n\nCRI-RM plugs in between kubelet and CRI, the Kubernetes node agent and the\ncontainer runtime implementation. CRI-RM intercepts CRI protocol requests\nfrom the kubelet acting as a non-transparent proxy towards the runtime.\nProxying by CRI-RM is non-transparent in nature because it usually alters\nintercepted protocol messages before forwarding them.\n\nCRI-RM keeps track of the states of all containers running on a Kubernetes\nnode. Whenever it intercepts a CRI request that results in changes to the\nresource allocation of any container (container creation, deletion, or\nresource assignment update request), CRI-RM runs one of its built-in policy\nalgorithms. This policy makes a decision about how the assignment of\nresources should be updated and, eventually, the intercepted request is\nmodified according to this decision. The policy can make changes to any\ncontainer in the system, not just the one associated with the intercepted\nCRI request. Therefore it does not operate directly on CRI requests.\nInstead CRI-RM's internal state tracking cache provides an abstraction for\nmodifying containers and the policy uses this abstraction for recording its\ndecisions.\n\nIn addition to policies, CRI-RM has a number of built-in resource\ncontrollers.\nThese are used to put policy decisions—in practice pending changes made to\ncontainers by a policy—into effect. A special in-band CRI controller is used\nto control all resources that are controllable via the CRI runtime. This\ncontroller handles the practical details of updating the intercepted CRI\nrequest and generating any additional unsolicited update requests for other\nexisting containers updated by the policy decision. Additional out-of-band\ncontrollers exist to exercise control over resources that the current CRI\nruntimes are unable to handle.\n\nTo tell which containers need to be handed off to various controllers for\nupdating, CRI-RM uses the internal state tracking cache's ability to tell\nwhich containers have pending unenforced changes and to which controllers'\ndomain these changes belong. The CRI controller currently handles CPU and\nmemory resources, including huge pages. The level of control covers per\ncontainer CPU sets, CFS parametrization, memory limits, OOM score adjustment,\nand pinning to memory controllers. The two existing out-of-band controllers,\nIntel® Resource Director Technology (Intel® RDT) and Block I/O, handle last-level cache and memory bandwidth allocation,\nand the arbitration of Block I/O bandwidth respectively.\n\nMany of the details of how CRI-RM operates is configurable. These include,\nfor instance, which policy is active within CRI-RM, configuration of the\nresource assignment algorithm for the active policy, and configuration for\nthe various resource controllers. Although CRI-RM can be configured using a\nconfiguration file present on the node running CRI-RM, the preferred way to\nconfigure all CRI-RM instances in a cluster is to use Kubernetes\nConfigMaps and the CRI-RM Node Agent.\n\n<p align=\"center\">\n<!-- # It's a pity the markdown ![]()-syntax does not support aligning... -->\n<img src=\"figures/arch-overview.svg\" title=\"Architecture Overview\">\n</p>\n\n## Components\n\n### [Node Agent](/pkg/agent/)\n\nThe node agent is a component external to CRI-RM itself. All interactions\nby CRI-RM with the Kubernetes Control Plane go through the node agent with\nthe node agent performing any direct interactions on behalf of CRI-RM.\n\nThe node agent communicates with CRI-RM using two gRPC interfaces. The\n[config interface](/pkg/cri/resource-manager/config/api/v1/) is used to:\n  - push updated external configuration data to CRI-RM\n  - push adjustments to container resource assignments to CRI-RM\n\nThe [cluster interface](/pkg/agent/api/v1/) implements the necessary\nlow-level plumbing for the agent interface CRI-RM internally exposes\nfor its policies and other components. This interface in turn implements\nthe following:\n  - updating resource capacity of the node\n  - getting, setting, or removing labels on the node\n  - getting, setting, or removing annotations on the node\n  - getting, setting, or removing taints on the node\n\nThe config interface is defined and has its gRPC server running in\nCRI-RM. The agent acts as a gRPC client for this interface. The low-level\ncluster interface is defined and has its gRPC server running in the agent,\nwith the [convenience layer](/pkg/cri/resource-manager/agent) defined in\nCRI-RM. CRI-RM acts as a gRPC client for the low-level plumbing interface.\n\nAdditionally, the stock node agent that comes with CRI-RM implements schemes\nfor:\n   - configuration management for all CRI-RM instances\n   - management of dynamic adjustments to container resource assignments\n\n<p align=\"center\">\n<!-- # It's a pity the markdown ![]()-syntax does not support aligning... -->\n<img src=\"figures/cri-resmgr.png\" title=\"Architecture Overview\" width=\"50%\">\n</p>\n\n\n### [Resource Manager](/pkg/cri/resource-manager/)\n\nCRI-RM implements a request processing pipeline and an event processing\npipeline.\nThe request processing pipeline takes care of proxying CRI requests and\nresponses between CRI clients and the CRI runtime. The event processing\npipeline processes a set of other events that are not directly related\nto or the result of CRI requests. These events are typically internally\ngenerated within CRI-RM. They can be the result of changes in the state\nof some containers or the utilization of a shared system resource, which\npotentially could warrant an attempt to rebalance the distribution of\nresources among containers to bring the system closer to an optimal state.\nSome events can also be generated by policies.\n\nThe Resource Manager component of CRI-RM implements the basic control\nflow of both of these processing pipelines. It passes control to all the\nnecessary sub-components of CRI-RM at the various phases of processing a\nrequest or an event. Additionally, it serializes the processing of these,\nmaking sure there is at most one (intercepted) request or event being\nprocessed at any point in time.\n\nThe high-level control flow of the request processing pipeline is as\nfollows:\n\nA. If the request does not need policying, let it bypass the processing\npipeline; hand it off for logging, then relay it to the server and the\ncorresponding response back to the client.\n\nB. If the request needs to be intercepted for policying, do the following:\n 1. Lock the processing pipeline serialization lock.\n 2. Look up/create cache objects (pod/container) for the request.\n 3. If the request has no resource allocation consequences, do proxying\n    (step 6).\n 4. Otherwise, invoke the policy layer for resource allocation:\n    - Pass it on to the configured active policy, which will\n    - Allocate resources for the container.\n    - Update the assignments for the container in the cache.\n    - Update any other containers affected by the allocation in the cache.\n 5. Invoke the controller layer for post-policy processing, which will:\n    - Collect controllers with pending changes in their domain of control\n    - for each invoke the post-policy processing function corresponding to\n      the request.\n    - Clear pending markers for the controllers.\n 6. Proxy the request:\n    - Relay the request to the server.\n    - Send update requests for any additional affected containers.\n    - Update the cache if/as necessary based on the response.\n    - Relay the response back to the client.\n 7. Release the processing pipeline serialization lock.\n\nThe high-level control flow of the event processing pipeline is one of the\nfollowing, based on the event type:\n\n - For policy-specific events:\n   1. Engage the processing pipeline lock.\n   2. Call policy event handler.\n   3. Invoke the controller layer for post-policy processing (same as step 5 for requests).\n   4. Release the pipeline lock.\n - For metrics events:\n   1. Perform collection/processing/correlation.\n   2. Engage the processing pipeline lock.\n   3. Update cache objects as/if necessary.\n   4. Request rebalancing as/if necessary.\n   5. Release pipeline lock.\n - For rebalance events:\n   1. Engage the processing pipeline lock.\n   2. Invoke policy layer for rebalancing.\n   3. Invoke the controller layer for post-policy processing (same as step 5 for requests).\n   4. Release the pipeline lock.\n\n\n### [Cache](/pkg/cri/resource-manager/cache/)\n\nThe cache is a shared internal storage location within CRI-RM. It tracks the\nruntime state of pods and containers known to CRI-RM, as well as the state\nof CRI-RM itself, including the active configuration and the state of the\nactive policy. The cache is saved to permanent storage in the filesystem and\nis used to restore the runtime state of CRI-RM across restarts.\n\nThe cache provides functions for querying and updating the state of pods and\ncontainers. This is the mechanism used by the active policy to make resource\nassignment decisions. The policy simply updates the state of the affected\ncontainers in the cache according to the decisions.\n\nThe cache's ability to associate and track changes to containers with\nresource domains is used to enforce policy decisions. The generic controller\nlayer first queries which containers have pending changes, then invokes each\ncontroller for each container. The controllers use the querying functions\nprovided by the cache to decide if anything in their resource/control domain\nneeds to be changed and then act accordingly.\n\nAccess to the cache needs to be serialized. However, this serialization is\nnot provided by the cache itself. Instead, it assumes callers to make sure\nproper protection is in place against concurrent read-write access. The\nrequest and event processing pipelines in the resource manager use a lock to\nserialize request and event processing and consequently access to the cache.\n\nIf a policy needs to do processing unsolicited by the resource manager, IOW\nprocessing other than handling the internal policy backend API calls from the\nresource manager, then it should inject a policy event into the resource\nmanagers event loop. This causes a callback from the resource manager to\nthe policy's event handler with the injected event as an argument and with\nthe cache properly locked.\n\n\n### [Generic Policy Layer](/pkg/cri/resource-manager/policy/policy.go)\n\nThe generic policy layer defines the abstract interface the rest of CRI-RM\nuses to interact with policy implementations and takes care of the details\nof activating and dispatching calls through to the configured active policy.\n\n\n### [Generic Resource Controller Layer](/pkg/cri/resource-manager/control/control.go)\n\nThe generic resource controller layer defines the abstract interface the rest\nof CRI-RM uses to interact with resource controller implementations and takes\ncare of the details of dispatching calls to the controller implementations\nfor post-policy enforcment of decisions.\n\n\n### [Metrics Collector](/pkg/metrics/)\n\nThe metrics collector gathers a set of runtime metrics about the containers\nrunning on the node. CRI-RM can be configured to periodically evaluate this\ncollected data to determine how optimal the current assignment of container\nresources is and to attempt a rebalancing/reallocation if it is deemed\nboth possible and necessary.\n\n\n### [Policy Implementations](/pkg/cri/resource-manager/policy/builtin/)\n\n#### [None](/pkg/cri/resource-manager/policy/builtin/none/)\n\nAn empty policy that makes no policy decisions. It is included\nmerely for the sake of completeness, analoguous to the none policy of the\nCPU Manager in kubelet.\n\n#### [Static Pools](/pkg/cri/resource-manager/policy/builtin/static-pools/)\n\nA backward-compatible reimplementation of\n[CMK](https://github.com/intel/CPU-Manager-for-Kubernetes)\nfor CRI-RM with a few extra features.\n\n#### [Static](/pkg/cri/resource-manager/policy/builtin/static/)\n\nPart of the code from the static policy of CPU Manager in kubelet, that has\nbeen brutally hacked to work within CRI-RM. Serves merely as a\nproof-of-concept that the current policies of kubelet can be implemented in\nCRI-RM.\n\n#### [Static Plus](/pkg/cri/resource-manager/policy/builtin/static-plus/)\n\nA fairly simplistic policy similar in spirit to the static policy of\nCPU Manager in kubelet, with a few extra features.\n\n#### [Topology Aware](/pkg/cri/resource-manager/policy/builtin/topology-aware/)\n\nA topology-aware policy capable of handling multiple tiers/types of memory,\ntypically a DRAM/PMEM combination configured in 2-layer memory mode.\n\n### [Resource Controller Implementations](/pkg/cri/resource-manager/control/)\n\n#### [Intel RDT](/pkg/cri/resource-manager/control/rdt/)\n\nA resource controller implementation responsible for the practical details of\nassociating a container with Intel RDT classes. This class effectively\ndetermines how much last level cache and memory bandwidth will be available\nfor the container. This controller uses the resctrl pseudo filesystem of the\nLinux kernel for control.\n\n#### [Block I/O](/pkg/cri/resource-manager/control/blockio/)\n\nA resource controller implementation responsible for the practical details of\nassociating a container with a Block I/O class. This class effectively\ndetermines how much Block I/O bandwidth will be available for the container.\nThis controller uses the blkio cgroup controller and the cgroupfs pseudo-\nfilesystem of the Linux kernel for control.\n\n#### [CRI](/pkg/cri/resource-manager/control/cri/)\n\nA resource controller responsible for modifying intercepted CRI container\ncreation requests and creating CRI container resource update requests,\naccording to the changes the active policy makes to containers.\n"
  },
  {
    "path": "docs/developers-guide/cri-test.md",
    "content": "# CRI Validation\n\n[This test](/test/critest) runs\n[`critest`](https://github.com/kubernetes-sigs/cri-tools/blob/master/docs/validation.md)\nfrom [cri-tools](https://github.com/kubernetes-sigs/cri-tools/) to\nmake sure that various `cri-resmgr` configurations do not break CRI\nruntime conformance.\n\n## Prerequisites\n\nInstall:\n- `docker`\n- `govm`\n\n## Run the test\n\n```\ncd test/critest\n./run.sh test\n```\n"
  },
  {
    "path": "docs/developers-guide/e2e-test.md",
    "content": "# End-to-End tests\n\n## Prerequisites\n\nInstall:\n- `docker`\n- `govm` v0.95\n  In case of errors in building `govm` with `go get`, or creating a virtual machine (`Error when creating the new VM: repository name must be canonical`), these are the workarounds:\n  ```\n  git clone https://github.com/govm-project/govm -b 0.95 && cd govm && go install && docker build . -t govm/govm:latest\n  ```\n\n## Usage\n\nRun policy tests:\n\n```\n[VAR=VALUE...] ./run_tests.sh policies\n```\n\nRun tests only on certain policy, topology, or only selected test:\n\n```\n[VAR=VALUE...] ./run_tests.sh policies[/POLICY[/TOPOLOGY[/testNN-*]]]\n```\n\nRun custom tests:\n\n```\n[VAR=VALUE...] ./run.sh MODE\n```\n\nGet help on available `VAR=VALUE`'s with `./run.sh\nhelp`. `run_tests.sh` calls `run.sh` in order to execute selected\ntests. Therefore the same `VAR=VALUE` definitions apply both scripts.\n\n## Test phases\n\nIn the *setup phase* `run.sh` creates a virtual machine unless it\nalready exists. When it is running, tests create a single-node cluster\nand launches `cri-resmgr` on it, unless they are already running.\n\nIn the *test phase* `run.sh` runs a test script, or gives a prompt\n(`run.sh> `) asking a user to run test script commands in the\n`interactive` mode. *Test scripts* are `bash` scripts that can use\nhelper functions for running commands and observing the status of the\nvirtual machine and software running on it.\n\nIn the *tear down phase* `run.sh` copies logs from the virtual machine\nand finally stops or deletes the virtual machine, if that is wanted.\n\n## Test modes\n\n- `test` mode runs fast and reports `Test verdict: PASS` or\n  `FAIL`. The exit status is zero if and only if a test passed.\n\n- `play` mode runs the same phases and scripts as the `test` mode, but\n  slower. This is good for following and demonstrating what is\n  happening.\n\n- `interactive` mode runs the setup and tear down phases, but instead\n  of executing a test script it gives an interactive prompt.\n\nPrint help to see clean up, execution speed and other options for all\nmodes.\n\n## Running from scratch and quick rerun in existing virtual machine\n\nThe test will use `govm`-managed virtual machine named in the `vm`\nenvironment variable. The default is `crirm-test-e2e`. If a virtual\nmachine with that name exists, the test will be run on it. Otherwise\nthe test will create a virtual machine with that name from\nscratch. You can delete a virtual machine with `govm delete NAME`.\n\nIf you want rerun the test many times, possibly with different test\ninputs or against different versions of `cri-resmgr`, either use the\n`play` mode or set `cleanup=0` in order to keep the virtual machine\nafter each run. Then tests will run in the same single node cluster,\nand the test script will only delete running pods before launching new\nones.\n\n## Testing locally built cri-resmgr and cri-resmgr from github\n\nIf you make changes to `cri-resmgr` sources and rebuild it, you can\nforce the test script to reinstall newly built `cri-resmgr` to\nexisting virtual machine before rerunning the test:\n\n```\ncri-resource-manager$ make\ncri-resource-manager$ cd test/e2e\ne2e$ reinstall_cri_resmgr=1 speed=1000 ./run.sh play\n```\n\nYou can also let the test script build `cri-resmgr` from the github\nmaster branch. This takes place inside the virtual machine, so your\nlocal git sources will not be affected:\n\n```\ne2e$ reinstall_cri_resmgr=1 binsrc=github ./run.sh play\n```\n\n## Custom tests\n\nYou can run a custom test script in a virtual machine that runs\nsingle-node Kubernetes\\* cluster. Example:\n\n```\n$ cat > myscript.sh << EOF\n# create two pods, each requesting two CPUs\nCPU=2 n=2 create guaranteed\n# create four pods, no resource requests\nn=4 create besteffort\n# show pods\nkubectl get pods\n# check that the first two pods are not allowed to use the same CPUs\nverify 'cpus[\"pod0c0\"].isdisjoint(cpus[\"pod1c0\"])'\nEOF\n$ ./run.sh test myscript.sh\n```\n\n## Custom topologies\n\nIf you change NUMA node topology of an existing virtual machine, you\nmust delete the virtual machine first. Otherwise the `topology` variable\nis ignored and the test will run in the existing NUMA\nconfiguration.\n\nThe `topology` variable is a JSON array of objects. Each object\ndefines one or more NUMA nodes. Keys in objects:\n```\n\"mem\"                 mem (RAM) size on each NUMA node in this group.\n                      The default is \"0G\".\n\"nvmem\"               nvmem (non-volatile RAM) size on each NUMA node\n                      in this group. The default is \"0G\".\n\"cores\"               number of CPU cores on each NUMA node in this group.\n                      The default is 0.\n\"threads\"             number of threads on each CPU core.\n                      The default is 2.\n\"nodes\"               number of NUMA nodes on each die.\n                      The default is 1.\n\"dies\"                number of dies on each package.\n                      The default is 1.\n\"packages\"            number of packages.\n                      The default is 1.\n```\n\n\nExample:\n\nRun the test in a VM with two NUMA nodes. There are 4 CPUs (two cores, two\nthreads per core by default) and 4G RAM in each node\n```\ne2e$ govm delete my2x4 ; vm=my2x4 topology='[{\"mem\":\"4G\",\"cores\":2,\"nodes\":2}]' ./run.sh play\n```\n\nRun the test in a VM with 32 CPUs in total: there are two packages\n(sockets) in the system, each containing two dies. Each die containing\ntwo NUMA nodes, each node containing 2 CPU cores, each core containing\ntwo threads. And with a NUMA node with 16G of non-volatile memory\n(NVRAM) but no CPUs.\n\n```\ne2e$ vm=mynvram topology='[{\"mem\":\"4G\",\"cores\":2,\"nodes\":2,\"dies\":2,\"packages\":2},{\"nvmem\":\"16G\"}]' ./run.sh play\n```\n\n## Test output\n\nAll test output is saved under the directory in the environment\nvariable `outdir`. The default is `./output`.\n\nExecuted commands with their output, exit status and timestamps are\nsaved under the `output/commands` directory.\n\nYou can find Qemu output from Docker\\* logs. For instance, output of the\nmost recent Qemu launced by `govm`:\n```\n$ docker logs $(docker ps | awk '/govm/{print $1; exit}')\n```\n\n## Manual testing and debugging\n\nInteractive mode helps developing and debugging scripts:\n\n```\n$ ./run.sh interactive\n...\nrun.sh> CPU=2 n=2 create guaranteed\n```\n\nYou can get help on functions available in test scripts with `./run.sh\nhelp script`, or with `help` and `help FUNCTION` when in the\ninteractive mode.\n\nIf a test has stopped to a failing `verify`, you can inspect\n`cri-resmgr` cache and allowed OS resources in Python\\* after the test\nrun:\n\n```\n$ PYTHONPATH=<TEST-OUTPUT-DIR> python3\n>>> from pyexec_state import *\n>>> pp(allowed) # allowed OS resources\n>>> pp(pods[\"pod0\"]) # pod entry in cache\n>>> pp(containers[\"pod0c0\"])) # container entry in cache\n```\n\nIf you want to get the interactive prompt in the middle of a test run\nwherever a `verify` or `create` fails, you can set a `on_FUNC_fail` hook to\neither or both of them. Example:\n\n```\n$ on_verify_fail=interactive ./run.sh myscript.sh\n```\n"
  },
  {
    "path": "docs/developers-guide/index.rst",
    "content": "Developer's Guide\n#################\n.. toctree::\n   :maxdepth: 1\n\n   architecture.md\n   policy-writers-guide.md\n   testing.rst\n"
  },
  {
    "path": "docs/developers-guide/policy-writers-guide.md",
    "content": "# Policy Writer's Guide\n\n***WORK IN PROGRESS***\n"
  },
  {
    "path": "docs/developers-guide/testing.rst",
    "content": "Testing\n#######\n\n.. toctree::\n   :maxdepth: 1\n\n   unit-test.md\n   cri-test.md\n   e2e-test.md\n"
  },
  {
    "path": "docs/developers-guide/unit-test.md",
    "content": "# Unit tests\n\nRun unit tests with\n```\nmake test\n```\n\n"
  },
  {
    "path": "docs/index.html",
    "content": "<meta http-equiv=\"refresh\" content=\"0; URL='docs/index.html'\" />\n"
  },
  {
    "path": "docs/index.rst",
    "content": ".. CRI Resource Manager documentation master file\n\nWelcome to CRI Resource Manager's documentation!\n================================================\n\n.. toctree::\n   :maxdepth: 2\n   :caption: Contents:\n\n   introduction.md\n   quick-start.md\n   installation.md\n   setup.md\n   policy/index.rst\n   node-agent.md\n   webhook.md\n\n   developers-guide/index.rst\n\n   migration-to-NRI.md\n\n   demos/index.rst\n\n   reference/index.md\n\n   contributing.md\n   security.md\n\n   Project GitHub repository <https://github.com/intel/cri-resource-manager>\n"
  },
  {
    "path": "docs/installation.md",
    "content": "# Installation\n\n## Installing from packages\n\nYou can install CRI Resource Manager from `deb` or `rpm` packages\nfor supported distros.\n\n  - [download](https://github.com/intel/cri-resource-manager/releases/latest)\n  packages\n  - install them:\n    - for rpm packages: `sudo rpm -Uvh <packages>`\n    - for deb packages: `sudo dpkg -i <packages>`\n\n## Installing from sources\n\nAlthough not recommended, you can install CRI Resource Manager from sources:\n\n  - get the sources: `git clone https://github.com/intel/cri-resource-manager`\n  - build and install: `cd cri-resource-manager; make build && sudo make install`\n\nYou will need at least `git`, {{ '`golang '+ '{}'.format(golang_version) + '`' }} or newer,\n`GNU make`, `bash`, `find`, `sed`, `head`, `date`, and `install` to be able to build and install\nfrom sources.\n\n## Building packages for the distro of your host\n\nYou can build packages for the `$distro` of your host by executing the\nfollowing command:\n\n```\nmake packages\n```\n\nIf the `$version` of your `$distro` is supported, this will leave the\nresulting packages in `packages/$distro-$version`. Building packages\nthis way requires `docker`, but it does not require you to install\nthe full set of build dependencies of CRI Resource Manager to your host.\n\nIf you want to build packages without docker, you can use either\n`make rpm` or `make deb`, depending on which supported distro you are\nrunning. Building this way requires all the build dependencies to be\ninstalled on your host.\n\nYou can check which `$distro`'s and `$version`'s are supported by running\n\n```\nls dockerfiles/cross-build\n```\n\nIf you see a `Dockerfile.$distro-$version` matching your host then your\ndistro is supported.\n\n## Building packages for another distro\n\nYou can cross-build packages of the native `$type` for a particular\n`$version` of a `$distro` by running the following command:\n\n```\nmake cross-$type.$distro-$version\n```\n\nSimilarly to `make packages`, this will build packages using a `Docker\\*`\ncontainer. However, instead of building for your host, it will build them\nfor the specified distro. For instance `make cross-deb.ubuntu-18.04` will\nbuild `deb` packages for `Ubuntu\\* 18.04`.\n\n## Post-install configuration\n\nThe provided packages install `systemd` service files and a sample\nconfiguration. The easiest way to get up and running is to rename the sample\nconfiguration and start CRI Resource Manager using systemd. You can do this\nusing the following commands:\n\n```\nmv /etc/cri-resmgr/fallback.cfg.sample /etc/cri-resmgr/fallback.cfg\nsystemctl start cri-resource-manager\n```\n\nIf you want, you can set CRI Resource Manager to automatically start\nwhen your system boots with this command:\n\n```\nsystemctl enable cri-resource-manager\n```\n\nThe provided packages also install a file for managing the default options\npassed to CRI Resource Manager upon startup. You can change these by editing\nthis file and then restarting CRI Resource Manager, like this:\n\n```\n# On Debian\\*-based systems edit the defaults like this:\n${EDITOR:-vi} /etc/default/cri-resource-manager\n# On rpm-based systems edit the defaults like this:\n${EDITOR:-vi} /etc/sysconfig/cri-resource-manager\n# Restart the service.\nsystemctl restart cri-resource-manager\n```\n"
  },
  {
    "path": "docs/introduction.md",
    "content": "# Introduction\n\nCRI Resource Manager is a Container Runtime Interface Proxy. It sits between\nclients and the actual Container Runtime implementation (containerd, cri-o)\nrelaying requests and responses back and forth. The main purpose of the proxy\nis to apply hardware-aware resource allocation policies to the containers\nrunning in the system.\n\nPolicies are applied by either modifying a request before forwarding it or\nby performing extra actions related to the request during its processing and\nproxying. There are several policies available, each with a different set of\ngoals in mind and implementing different hardware allocation strategies. The\ndetails of whether and how a CRI request is altered or if extra actions are\nperformed depend on which policy is active in CRI Resource Manager and how\nthat policy is configured.\n\nThe current goal for the CRI Resource Manager is to prototype and experiment\nwith new Kubernetes\\* container placement policies. The existing policies are\nwritten with this in mind and the intended setup is for the Resource Manager\nto only act as a proxy for the Kubernetes Node Agent, kubelet.\n"
  },
  {
    "path": "docs/migration-to-NRI.md",
    "content": "# Migrating from CRI-RM to NRI\n\n## Prerequisities\n\n- Up and running CRI Resource Manager\n- One of the two supported policies in use: balloons or topology-aware.\n- For other policies a little bit more work is required and the policies need to be 'ported'. This can be done by just following the example of how balloons or topology-aware were converted.\n\n## Steps for an initial/basic migration test\n\n### Containerd\n\nReplace the containerd version in the system with 1.7 or newer version (NRI server not supported in older versions).\n\nReplace kubelet's --container-runtime-endpoint=/var/run/cri-resmgr/cri-resmgr.sock with --container-runtime-endpoint=/var/run/containerd/containerd.sock\n\nReplacing the runtime endpoint on a node that was setup using Kubeadm:\n```console\n# Get the Kubelet args\nsystemctl cat kubelet <- Look for: EnvironmentFile=/.../kubeadm-flags.env\n\nvim /.../kubeadm-flags.env\n  KUBELET_KUBEADM_ARGS=\"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.k8s.io/pause:3.9\"\n\nvim /etc/sysconfig/kubelet\n  KUBELET_EXTRA_ARGS= --container-runtime-endpoint=/var/run/containerd/containerd.sock <- Remember this aswell\n\nsystemctl restart kubelet\n```\n\nEdit the containerd config file and look for the section [plugins.\"io.containerd.nri.v1.nri\"] and replace \"disable = true\" with \"disable = false\":\n```console\nvim /etc/containerd/config.toml\n```\n```toml\n[plugins.\"io.containerd.nri.v1.nri\"]\n  disable = false\n  disable_connections = false\n  plugin_config_path = \"/etc/nri/conf.d\"\n  plugin_path = \"/opt/nri/plugins\"\n  plugin_registration_timeout = \"5s\"\n  plugin_request_timeout = \"2s\"\n  socket_path = \"/var/run/nri/nri.sock\"\n```\n```console\nsystemctl restart containerd\n```\n\n### CRI-O\n\nEnsure that crio version 1.26.2 or newer is used.\n\nReplace kubelet's --container-runtime-endpoint=/var/run/cri-resmgr/cri-resmgr.sock with --container-runtime-endpoint=/var/run/crio/crio.sock\n\nReplacing the runtime endpoint on a node that was setup using Kubeadm:\n```console\n# Get the Kubelet args\nsystemctl cat kubelet <- Look for: EnvironmentFile=/.../kubeadm-flags.env\n\nvim /.../kubeadm-flags.env\n  KUBELET_KUBEADM_ARGS=\"--container-runtime-endpoint=unix:///var/run/crio/crio.sock --pod-infra-container-image=registry.k8s.io/pause:3.9\"\n\nvim /etc/sysconfig/kubelet\n  KUBELET_EXTRA_ARGS= --container-runtime-endpoint=/var/run/crio/crio.sock <- Remember this aswell\n\nsystemctl restart kubelet\n```\n\nEnable NRI:\n```console\nCRIO_CONF=/etc/crio/crio.conf\ncp $CRIO_CONF $CRIO_CONF.orig\ncrio --enable-nri config > $CRIO_CONF\nsystemctl restart crio\n```\n\n### Build the NRI policies\n\n```console\ngit clone https://github.com/containers/nri-plugins.git\ncd nri-plugins\nmake\n# Build the images, specify your image repo to easily push the image later.\nmake images IMAGE_REPO=my-repo IMAGE_VERSION=my-tag\n```\n\n### Create required CRDs\n\n```console\nkubectl apply -f deployment/base/crds/noderesourcetopology_crd.yaml\n```\n\n### Import the image of the NRI plugin you want to run\n\n<b>Containerd</b>\n\n```console\nctr -n k8s.io images import build/images/nri-resmgr-topology-aware-image-*.tar\n```\n\n<b>CRI-O</b>\n\nSee the section [below](#steps-for-a-more-real-life-migration-using-self-hosted-image-repository) for instructions on how to push the images to a registry, then pull from there.\n\n### Deploy the plugin\n\n  ```console\n  kubectl apply -f build/images/nri-resmgr-topology-aware-deployment.yaml\n  ```\n\n### Deploy a test pod\n\n```console\nkubectl run mypod --image busybox -- sleep inf\nkubectl exec mypod  -- grep allowed_list: /proc/self/status\n```\n\n### See the resources the pod got assigned with\n\n```console\nkubectl exec $pod -c $container  -- grep allowed_list: /proc/self/status\n# Output should look similar to the output of CRI-RM\n```\n\n## Steps for a more real-life migration using self-hosted image repository\n\n- Same steps as above for enabling NRI with Containerd/CRI-O and building the images.\n- Push the images built to your repository:\n  ```console\n  # Replace my-repo and my-tag with the IMAGE_REPO and IMAGE_VERSION you specified when building the images with make images\n  docker push my-repo:my-tag\n  ```\n\n- Remember to change the image name & pull policy in the plugins .yaml file to match your registyr and image, ex:\n  ```console\n  vim build/images/nri-resmgr-topology-aware-deployment.yaml\n  ```\n\n- Then deploy the plugin simlarly to the earlier step.\n\n## Migrating existing configuration\n\n- The ConfigMap used by the ported policies/infra has a different name/naming scheme than the original one used in CRI-RM, ex:\n  - configMapName:\n    ```diff\n    - configmap-name: cri-resmgr-config\n    + configmap-name: nri-resource-policy-config\n    ```\n  - The details of grouping nodes by labeling to share configuration:\n    ```diff\n    - cri-resource-manager.intel.com/group: $GROUP_NAME\n    + resource-policy.nri.io/group: $GROUP_NAME\n    ```\n\n## Migrating existing workloads\n\n- The annotations one can use to customize how a policy treats a workload use slightly different keys than the original ones in CRI-RM. The collective 'key namespace' for policy- and resource-manager-specific annotation has been changed from cri-resource-manager.intel.com to resource-policy.nri.io.\n\n- For instance, an explicit type annotation for the balloons policy, which used to be:\n  ```yaml\n  ...\n  metadata:\n    annotations:\n      balloon.balloons.cri-resource-manager.intel.com/container.$CONTAINER_NAME: $BALLOON_TYPE`\n  ...\n  ```\n\n- Should now be:\n  ```yaml\n  ...\n  metadata:\n    annotations:\n      balloon.balloons.resource-policy.nri.io/container.$CONTAINER_NAME: $BALLOON_TYPE`\n  ...\n  ```\n\n- Similarly a workload opt-out annotation from exclusive CPU allocation for the topology-aware policy, which used to be:\n  ```yaml\n  ...\n  metadata:\n    annotations:\n      prefer-shared-cpus.cri-resource-manager.intel.com/container.$CONTAINER_NAME: \"true\"\n  ...\n  ```\n\n- Should now be:\n  ```yaml\n  ...\n  metadata:\n    annotations:\n      prefer-shared-cpus.resource-policy.nri.io/container.$CONTAINER_NAME: \"true\"\n  ...\n  ```\n\n- Similar changes are needed for any cri-resmgr-specific annotation that uses the same semantic scoping for key syntax.\n\n\nAll of the annotations:\n| Was                                                 | Is now                                      |\n| --------------------------------------------------- | ------------------------------------------- |\n| cri-resource-manager.intel.com/afffinity            | resource-policy.nri.io/affinity             |\n| cri-resource-manager.intel.com/anti-afffinity       | resource-policy.nri.io/anti-affinity        |\n| cri-resource-manager.intel.com/prefer-isolated-cpus | resource-policy.nri.io/prefer-isolated-cpus |\n| cri-resource-manager.intel.com/prefer-shared-cpus   | resource-policy.nri.io/prefer-shared-cpus   |\n| cri-resource-manager.intel.com/cold-start           | resource-policy.nri.io/cold-start           |\n| cri-resource-manager.intel.com/memory-type          | resource-policy.nri.io/ memory-type         |\n| prefer-isolated-cpus.cri-resource-manager.intel.com | prefer-isolated-cpus.resource-policy.nri.io |\n| prefer-shared-cpus.cri-resource-manager.intel.com   | prefer-shared-cpus.resource-policy.nri.io   |\n| memory-type.cri-resource-manager.intel.com          | memory-type.resource-policy.nri.io          |\n| cold-start.cri-resource-manager.intel.com           | cold-start.resource-policy.nri.io           |\n| prefer-reserved-cpus.cri-resource-manager.intel.com | prefer-reserved-cpus.resource-policy.nri.io |\n| rdtclass.cri-resource-manager.intel.com             | rdtclass.resource-policy.nri.io             |\n| blockioclass.cri-resource-manager.intel.com         | blockioclass.resource-policy.nri.io         |\n| toptierlimit.cri-resource-manager.intel.com         | toptierlimit.resource-policy.nri.io         |\n| topologyhints.cri-resource-manager.intel.com        | topologyhints.resource-policy.nri.io        |\n| balloon.balloons.cri-resource-manager.intel.com     | balloon.balloons.resource-policy.nri.io     |\n"
  },
  {
    "path": "docs/node-agent.md",
    "content": "# Node Agent\n\nCRI Resource Manager can be configured dynamically using the CRI Resource\nManager Node Agent and Kubernetes\\* ConfigMaps.\n\n## Running as a DaemonSet\n\nThe agent can be build using the\n[provided Dockerfile](/cmd/cri-resmgr-agent/Dockerfile). It can be\ndeployed as a `DaemonSet` in the cluster using the\n[provided deployment file](/cmd/cri-resmgr-agent/agent-deployment.yaml).\n\nWhen using the provided or a similar deployment, the agent uses a\nreadiness probe to propagate the status of the last configuration\nupdate back to the control plane. If the configuration could not\nbe taken into use for any reason, the agent's probe will fail which\neventually marks the agent as not being `Ready`. In this case, more\ndetails about the failure should be present among the latest messages\nlogged by the agent or the probe itself. if the reason for failure is\na configuration error, once the error is fixed, the agent should become\neventually `Ready` again.\n\n## Running as a Host Service\n\nTo run the agent manually or as a `systemd` service, set the environment\nvariable `NODE_NAME` to the name of the cluster node the agent is running\non. If necessary pass it the credentials for accessing the cluster using\n the `-kubeconfig <file>` command line option.\n\n## ConfigMap to Node Mapping Conventions\n\nThe agent monitors two ConfigMaps for the node, a primary node-specific one\nand a secondary group-specific or default one, depending on whether the node\nbelongs to a configuration group. The node-specific ConfigMap always takes\nprecedence over the others.\n\nThe names of these ConfigMaps are\n\n1. `cri-resmgr-config.node.$NODE_NAME`: primary, node-specific configuration\n2. `cri-resmgr-config.group.$GROUP_NAME`: secondary group-specific node\n    configuration\n3. `cri-resmgr-config.default`: secondary: secondary default node\n    configuration\n\nYou can assign a node to a configuration group by setting the\n`cri-resource-manager.intel.com/group` label on the node to the name of\nthe configuration group. You can remove a node from its group by deleting\nthe node group label.\n\nThere is a\n[sample ConfigMap spec](/sample-configs/cri-resmgr-configmap.example.yaml)\nthat contains a node-specific, a group-specific, and a default ConfigMap\nexample. See [any available policy-specific documentation](policy/index.rst)\nfor more information on the policy configurations.\n\n"
  },
  {
    "path": "docs/policy/balloons.md",
    "content": "# Balloons Policy\n\n## Overview\n\nThe balloons policy implements workload placement into \"balloons\" that\nare disjoint CPU pools. Balloons can be inflated and deflated, that is\nCPUs added and removed, based on the CPU resource requests of\ncontainers. Balloons can be static or dynamically created and\ndestroyed. CPUs in balloons can be configured, for example, by setting\nmin and max frequencies on CPU cores and uncore.\n\n## How It Works\n\n1. User configures balloon types from which the policy instantiates\n   balloons.\n\n2. A balloon has a set of CPUs and a set of containers that run on the\n   CPUs.\n\n3. Every container is assigned to exactly one balloon. A container is\n   allowed to use all CPUs of its balloon and no other CPUs.\n\n4. Every logical CPU belongs to at most one balloon. There can be CPUs\n   that do not belong to any balloon.\n\n5. The number of CPUs in a balloon can change during the lifetime of\n   the balloon. If a balloon inflates, that is CPUs are added to it,\n   all containers in the balloon are allowed to use more CPUs. If a\n   balloon deflates, the opposite is true.\n\n6. When a new container is created on a Kubernetes node, the policy\n   first decides the type of the balloon that will run the\n   container. The decision is based on annotations of the pod, or the\n   namespace if annotations are not given.\n\n7. Next the policy decides which balloon of the decided type will run\n   the container. Options are:\n   - an existing balloon that already has enough CPUs to run its\n     current and new containers\n   - an existing balloon that can be inflated to fit its current and\n     new containers\n   - new balloon.\n\n9. When a CPU is added to a balloon or removed from it, the CPU is\n   reconfigured based on balloon's CPU class attributes, or idle CPU\n   class attributes.\n\n## Deployment\n\n### Install cri-resmgr\n\nDeploy cri-resmgr on each node as you would for any other policy. See\n[installation](../installation.md) for more details.\n\n## Configuration\n\nThe balloons policy is configured using the yaml-based configuration\nsystem of CRI-RM. See [setup and\nusage](../setup.md#setting-up-cri-resource-manager) for more details\non managing the configuration.\n\n### Parameters\n\nBalloons policy parameters:\n\n- `PinCPU` controls pinning a container to CPUs of its balloon. The\n  default is `true`: the container cannot use other CPUs.\n- `PinMemory` controls pinning a container to the memories that are\n  closest to the CPUs of its balloon. The default is `true`: allow\n  using memory only from the closest NUMA nodes. Warning: this may\n  cause kernel to kill workloads due to out-of-memory error when\n  closest NUMA nodes do not have enough memory. In this situation\n  consider switching this option `false`.\n- `IdleCPUClass` specifies the CPU class of those CPUs that do not\n  belong to any balloon.\n- `ReservedPoolNamespaces` is a list of namespaces (wildcards allowed)\n  that are assigned to the special reserved balloon, that is, will run\n  on reserved CPUs. This always includes the `kube-system` namespace.\n- `AllocatorTopologyBalancing` affects selecting CPUs for new\n  balloons. If `true`, new balloons are created using CPUs on\n  NUMA/die/package with most free CPUs, that is, balloons are spread\n  across the hardware topology. This helps inflating balloons within\n  the same NUMA/die/package and reduces interference between workloads\n  in balloons when system is not fully loaded. The default is `false`:\n  pack new balloons tightly into the same NUMAs/dies/packages. This\n  helps keeping large portions of hardware idle and entering into deep\n  power saving states.\n- `PreferSpreadOnPhysicalCores` prefers allocating logical CPUs\n  (possibly hyperthreads) for a balloon from separate physical CPU\n  cores. This prevents workloads in the balloon from interfering with\n  themselves as they do not compete on the resources of the same CPU\n  cores. On the other hand, it allows more interference between\n  workloads in different balloons. The default is `false`: balloons\n  are packed tightly to a minimum number of physical CPU cores. The\n  value set here is the default for all balloon types, but it can be\n  overridden with the balloon type specific setting with the same\n  name.\n- `BalloonTypes` is a list of balloon type definitions. Each type can\n  be configured with the following parameters:\n  - `Name` of the balloon type. This is used in pod annotations to\n    assign containers to balloons of this type.\n  - `Namespaces` is a list of namespaces (wildcards allowed) whose\n    pods should be assigned to this balloon type, unless overridden by\n    pod annotations.\n  - `MinBalloons` is the minimum number of balloons of this type that\n    is always present, even if the balloons would not have any\n    containers. The default is 0: if a balloon has no containers, it\n    can be destroyed.\n  - `MaxBalloons` is the maximum number of balloons of this type that\n    is allowed to co-exist. The default is 0: creating new balloons is\n    not limited by the number of existing balloons.\n  - `MaxCPUs` specifies the maximum number of CPUs in any balloon of\n\tthis type. Balloons will not be inflated larger than this. 0 means\n\tunlimited.\n  - `MinCPUs` specifies the minimum number of CPUs in any balloon of\n    this type. When a balloon is created or deflated, it will always\n    have at least this many CPUs, even if containers in the balloon\n    request less.\n  - `CpuClass` specifies the name of the CPU class according to which\n    CPUs of balloons are configured.\n  - `PreferSpreadingPods`: if `true`, containers of the same pod\n    should be spread to different balloons of this type. The default\n    is `false`: prefer placing containers of the same pod to the same\n    balloon(s).\n  - `PreferPerNamespaceBalloon`: if `true`, containers in the same\n\tnamespace will be placed in the same balloon(s). On the other\n\thand, containers in different namespaces are preferrably placed in\n\tdifferent balloons. The default is `false`: namespace has no\n\teffect on choosing the balloon of this type.\n  - `PreferNewBalloons`: if `true`, prefer creating new balloons over\n    placing containers to existing balloons. This results in\n    preferring exclusive CPUs, as long as there are enough free\n    CPUs. The default is `false`: prefer filling and inflating\n    existing balloons over creating new ones.\n  - `ShareIdleCPUsInSame`: Whenever the number of or sizes of balloons\n    change, idle CPUs (that do not belong to any balloon) are reshared\n    as extra CPUs to workloads in balloons with this option. The value\n    sets locality of allowed extra CPUs that will be common to these\n    workloads.\n    - `system`: workloads are allowed to use idle CPUs available\n      anywhere in the system.\n    - `package`: ...allowed to use idle CPUs in the same package(s)\n    (sockets) as the balloon.\n    - `die`: ...in the same die(s) as the balloon.\n    - `numa`: ...in the same numa node(s) as the balloon.\n    - `core`: ...allowed to use idle CPU threads in the same cores with\n      the balloon.\n  - `PreferSpreadOnPhysicalCores` overrides the policy level option\n    with the same name in the scope of this balloon type.\n  - `AllocatorPriority` (0: High, 1: Normal, 2: Low, 3: None). CPU\n    allocator parameter, used when creating new or resizing existing\n    balloons. If there are balloon types with pre-created balloons\n    (`MinBalloons` > 0), balloons of the type with the highest\n    `AllocatorPriority` are created first.\n\nRelated configuration parameters:\n- `policy.ReservedResources.CPU` specifies the (number of) CPUs in the\n  special `reserved` balloon. By default all containers in the\n  `kube-system` namespace are assigned to the reserved balloon.\n- `cpu.classes` defines CPU classes and their parameters (such as\n  `minFreq`, `maxFreq`, `uncoreMinFreq` and `uncoreMaxFreq`).\n\n### Example\n\nExample configuration that runs all pods in balloons of 1-4 CPUs.\n```yaml\npolicy:\n  Active: balloons\n  ReservedResources:\n    CPU: 1\n  balloons:\n    PinCPU: true\n    PinMemory: true\n    IdleCPUClass: lowpower\n    BalloonTypes:\n      - Name: \"quad\"\n        MinCpus: 1\n        MaxCPUs: 4\n        CPUClass: dynamic\n        Namespaces:\n          - \"*\"\ncpu:\n  classes:\n    lowpower:\n      minFreq: 800\n      maxFreq: 800\n    dynamic:\n      minFreq: 800\n      maxFreq: 3600\n    turbo:\n      minFreq: 3000\n      maxFreq: 3600\n      uncoreMinFreq: 2000\n      uncoreMaxFreq: 2400\n```\n\nSee the [sample configmap](/sample-configs/balloons-policy.cfg) for a\ncomplete example.\n\n## Assigning a Container to a Balloon\n\nThe balloon type of a container can be defined in pod annotations. In\nthe example below, the first annotation sets the balloon type (`BT`)\nof a single container (`CONTAINER_NAME`). The last two annotations set\nthe default balloon type for all containers in the pod.\n\n```yaml\nballoon.balloons.cri-resource-manager.intel.com/container.CONTAINER_NAME: BT\nballoon.balloons.cri-resource-manager.intel.com/pod: BT\nballoon.balloons.cri-resource-manager.intel.com: BT\n```\n\nIf a pod has no annotations, its namespace is matched to the\n`Namespaces` of balloon types. The first matching balloon type is\nused.\n\nIf the namespace does not match, the container is assigned to the\nspecial `default` balloon, that means reserved CPUs unless `MinCPUs`\nor `MaxCPUs` of the `default` balloon type are explicitely defined in\nthe `BalloonTypes` configuration.\n\n## Metrics and Debugging\n\nIn order to enable more verbose logging and metrics exporting from the\nballoons policy, enable instrumentation and policy debugging from the\nCRI-RM global config:\n\n```yaml\ninstrumentation:\n  # The balloons policy exports containers running in each balloon,\n  # and cpusets of balloons. Accessible in command line:\n  # curl --silent http://localhost:8891/metrics\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n```\n"
  },
  {
    "path": "docs/policy/blockio.md",
    "content": "# Block IO\n\n## Overview\n\nBlock IO controller provides means to control\n- block device IO scheduling priority (weight)\n- throttling IO bandwith\n- throttling number of IO operations.\n\nCRI Resource Manager applies block IO contoller parameters to pods via\n[cgroups block io contoller](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/blkio-controller.html).\n\n## Configuration\n\nSee [sample blockio configuration](/sample-configs/blockio.cfg).\n\n## Demo\n\nSee [Block IO demo](../demos/blockio.md)\n"
  },
  {
    "path": "docs/policy/container-affinity.md",
    "content": "# Container Affinity and Anti-Affinity\n\n## Introduction\n\nSome policies allow the user to give hints about how particular containers\nshould be *co-located* within a node. In particular these hints express whether\ncontainers should be located *'close'* to each other or *'far away'* from each\nother, in a hardware topology sense.\n\nSince these hints are interpreted always by a particular *policy implementation*,\nthe exact definitions of 'close' and 'far' are also somewhat *policy-specific*.\nHowever as a general rule of thumb containers running\n\n  - on CPUs within the *same NUMA nodes* are considered *'close'* to each other,\n  - on CPUs within *different NUMA nodes* in the *same socket* are *'farther'*, and\n  - on CPUs within *different sockets* are *'far'* from each other\n\nThese hints are expressed by `container affinity annotations` on the Pod.\nThere are two types of affinities:\n\n  - `affinity` (or `positive affinty`): cause affected containers to *pull* each other closer\n  - `anti-affinity` (or `negative affinity`): cause affected containers to *push* each other further away\n\nPolicies try to place a container\n  - close to those the container has affinity towards\n  - far from those the container has anti-affinity towards.\n\n## Affinity Annotation Syntax\n\n*Affinities* are defined as the `cri-resource-manager.intel.com/affinity` annotation.\n*Anti-affinities* are defined as the `cri-resource-manager.intel.com/anti-affinity`\nannotation. They are specified in the `metadata` section of the `Pod YAML`, under\n`annotations` as a dictionary, with each dictionary key being the name of the\n*container* within the Pod to which the annotation belongs to.\n\n```yaml\nmetadata:\n  anotations:\n    cri-resource-manager.intel.com/affinity: |\n      container1:\n        - scope:\n            key: key-ref\n            operator: op\n            values:\n            - value1\n            ...\n            - valueN\n          match:\n            key: key-ref\n            operator: op\n            values:\n            - value1\n            ...\n            - valueN\n          weight: w\n```\n\nAn anti-affinity is defined similarly but using `cri-resource-manager.intel.com/anti-affinity`\nas the annotation key.\n\n```yaml\nmetadata:\n  anotations:\n    cri-resource-manager.intel.com/anti-affinity: |\n      container1:\n        - scope:\n            key: key-ref\n            operator: op\n            values:\n            - value1\n            ...\n            - valueN\n          match:\n            key: key-ref\n            operator: op\n            values:\n            - value1\n            ...\n            - valueN\n          weight: w\n```\n\n## Affinity Semantics\n\nAn affinity consists of three parts:\n\n  - `scope expression`: defines which containers this affinity is evaluated against\n  - `match expression`: defines for which containers (within the scope) the affinity applies to\n  - `weight`: defines how *strong* a pull or a push the affinity causes\n\n*Affinities* are also sometimes referred to as *positive affinities* while\n*anti-affinities* are referred to as *negative affinities*. The reason for this is\nthat the only difference between these are that affinities have a *positive weight*\nwhile anti-affinities have a *negative weight*.\n\nThe *scope* of an affinity defines the *bounding set of containers* the affinity can\napply to. The affinity *expression* is evaluated against the containers *in scope* and\nit *selects the containers* the affinity really has an effect on. The *weight* specifies\nwhether the effect is a *pull* or a *push*. *Positive* weights cause a *pull* while\n*negative* weights cause a *push*. Additionally, the *weight* specifies *how strong* the\npush or the pull is. This is useful in situations where the policy needs to make some\ncompromises because an optimal placement is not possible. The weight then also acts as\na way to specify preferences of priorities between the various compromises: the heavier\nthe weight the stronger the pull or push and the larger the propbability that it will be\nhonored, if this is possible at all.\n\nThe scope can be omitted from an affinity in which case it implies *Pod scope*, in other\nwords the scope of all containers that belong to the same Pod as the container for which\nwhich the affinity is defined.\n\nThe weight can also be omitted in which case it defaults to -1 for anti-affinities\nand +1 for affinities. Weights are currently limited to the range [-1000,1000].\n\nBoth the affinity scope and the expression select containers, therefore they are identical.\nBoth of them are *expressions*. An expression consists of three parts:\n\n  - key: specifies what *metadata* to pick from a container for evaluation\n  - operation (op): specifies what *logical operation* the expression evaluates\n  - values: a set of *strings* to evaluate the the value of the key against\n\nThe supported keys are:\n\n  - for pods:\n    - `name`\n    - `namespace`\n    - `qosclass`\n    - `labels/<label-key>`\n    - `id`\n    - `uid`\n  - for containers:\n    - `pod/<pod-key>`\n    - `name`\n    - `namespace`\n    - `qosclass`\n    - `labels/<label-key>`\n    - `tags/<tag-key>`\n    - `id`\n\nEssentially an expression defines a logical operation of the form (key op values).\nEvaluating this logical expression will take the value of the key in  which\neither evaluates to true or false.\na boolean true/false result. Currently the following operations are supported:\n\n  - `Equals`: equality, true if the *value of key* equals the single item in *values*\n  - `NotEqual`: inequality, true if the *value of key* is not equal to the single item in *values*\n  - `In`: membership, true if *value of key* equals to any among *values*\n  - `NotIn`: negated membership, true if the *value of key* is not equal to any among *values*\n  - `Exists`: true if the given *key* exists with any value\n  - `NotExists`: true if the given *key* does not exist\n  - `AlwaysTrue`: always evaluates to true, can be used to denote node-global scope (all containers)\n  - `Matches`: true if the *value of key* matches the globbing pattern in values\n  - `MatchesNot`: true if the *value of key* does not match the globbing pattern in values\n  - `MatchesAny`: true if the *value of key* matches any of the globbing patterns in values\n  - `MatchesNone`: true if the *value of key* does not match any of the globbing patterns in values\n\nThe effective affinity between containers C_1 and C_2, A(C_1, C_2) is the sum of the\nweights of all pairwise in-scope matching affinities W(C_1, C_2). To put it another way,\nevaluating an affinity for a container C_1 is done by first using the scope (expression)\nto determine which containers are in the scope of the affinity. Then, for each in-scope\ncontainer C_2 for which the match expression evaluates to true, taking the weight of the\naffinity and adding it to the effective affinity A(C_1, C_2).\n\nNote that currently (for the topology-aware policy) this evaluation is asymmetric:\nA(C_1, C_2) and A(C_2, C_1) can and will be different unless the affinity annotations are\ncrafted to prevent this (by making them fully symmetric). Moreover, A(C_1, C_2) is calculated\nand taken into consideration during resource allocation for C_1, while A(C_2, C_1)\nis calculated and taken into account during resource allocation for C_2. This might be\nchanged in a future version.\n\n\nCurrently affinity expressions lack support for boolean operators (and, or, not).\nSometimes this limitation can be overcome by using joint keys, especially with\nmatching operators. The joint key syntax allows joining the value of several keys\nwith a separator into a single value. A joint key can be specified in a simple or\nfull format:\n\n  - simple: `<colon-separated-subkeys>`, this is equivalent to `:::<colon-separated-subkeys>`\n  - full:   `<ksep><vsep><ksep-separated-keylist>`\n\nA joint key evaluates to the values of all the `<ksep>`-separated subkeys joined by `<vsep>`.\nA non-existent subkey evaluates to the empty string. For instance the joint key\n\n  `:pod/qosclass:pod/name:name`\n\nevaluates to\n\n  `<qosclass>:<pod name>:<container name>`\n\nFor existence operators, a joint key is considered to exist if any of its subkeys exists.\n\n\n## Examples\n\nPut the container `peter` close to the container `sheep` but far away from the\ncontainer `wolf`.\n\n```yaml\nmetadata:\n  annotations:\n    cri-resource-manager.intel.com/affinity: |\n      peter:\n      - match:\n          key: name\n          operator: Equals\n          values:\n          - sheep\n        weight: 5\n    cri-resource-manager.intel.com/anti-affinity: |\n      peter:\n      - match:\n          key: name\n          operator: Equals\n          values:\n          - wolf\n        weight: 5\n```\n\n## Shorthand Notation\n\nThere is an alternative shorthand syntax for what is considered to be the most common\ncase: defining affinities between containers within the same pod. With this notation\none needs to give just the names of the containers, like in the example below.\n\n```yaml\n  annotations:\n    cri-resource-manager.intel.com/affinity: |\n      container3: [ container1 ]\n    cri-resource-manager.intel.com/anti-affinity: |\n      container3: [ container2 ]\n      container4: [ container2, container3 ]\n```\n\n\nThis shorthand notation defines:\n  - `container3` having\n    - affinity (weight 1) to `container1`\n    - `anti-affinity` (weight -1) to `container2`\n  - `container4` having\n    - `anti-affinity` (weight -1) to `container2`, and `container3`\n\nThe equivalent annotation in full syntax would be\n\n```yaml\nmetadata:\n  annotations:\n    cri-resource-manager.intel.com/affinity: |+\n      container3:\n      - match:\n          key: labels/io.kubernetes.container.name\n          operator: In\n          values:\n          - container1\n    cri-resource-manager.intel.com/anti-affinity: |+\n      container3:\n      - match:\n          key: labels/io.kubernetes.container.name\n          operator: In\n          values:\n          - container2\n      container4:\n      - match:\n          key: labels/io.kubernetes.container.name\n          operator: In\n          values:\n          - container2\n          - container3\n```\n"
  },
  {
    "path": "docs/policy/cpu-allocator.md",
    "content": "# CPU Allocator\n\nCRI Resource Manager has a separate CPU allocator component that helps policies\nmake educated allocation of CPU cores for workloads.  Currently all policies\nexcept for [static-pools](static-pools.md) utilize the built-in CPU allocator.\nSee policy specific documentation for more details.\n\n## Topology Based Allocation\n\nThe CPU allocator tries to optimize the allocation of CPUs in terms of the\nhardware topology. More specifically, it aims at packing all CPUs of one\nrequest \"near\" each other in order to minimize memory latencies between CPUs.\n\n## CPU Prioritization\n\nThe CPU allocator also does automatic CPU prioritization by detecting CPU\nfeatures and their configuration parameters. Currently, CRI Resource Manager\nsupports CPU priority detection based on the `intel_pstate` scaling\ndriver in the Linux CPUFreq subsystem, and, Intel Speed Select Technology\n(SST).\n\nCPUs are divided into three priority classes, i.e. *high*, *normal* and *low*.\nPolicies utilizing the CPU allocator may choose to prefer certain priority\nclass for certain types of workloads. For example, prefer (and preserve) high\npriority CPUs for high priority workloads.\n\n### Intel Speed Select Technology (SST)\n\nCRI Resource Manager supports detection of all Intel Speed Select Technology\n(SST) features, i.e. Speed Select Technology Performance Profile (SST-PP), Base\nFrequency (SST-BF), Turbo Frequency (SST-TF) and Core Power (SST-CP).\n\nCPU prioritization is based on detection of the currently active SST features\nand their parameterization:\n\n1. If SST-TF has been enabled, all CPUs prioritized by SST-TF are flagged as\n   high priority.\n1. If SST-CP is enabled but SST-TF disabled, the CPU allocator examines the\n   active Classes of Service (CLOSes) and their parameters. CPUs associated\n   with the highest priority CLOS will be flagged as high priority, lowest\n   priority CLOS will be flagged as low priority and possible \"middle priority\"\n   CLOS as normal priority.\n1. If SST-BF has been enabled and SST-TF and SST-CP are inactive, all BF high\n   priority cores (having higher guaranteed base frequency) will be flagged\n   as high priority.\n\n### Linux CPUFreq\n\nCPUFreq based prioritization only takes effect if Intel Speed Select Technology\n(SST) is disabled (or not supported). CRI-RM divides CPU cores into priority\nclasses based on two parameters:\n\n- base frequency\n- EPP (Energy-Performance Preference)\n\nCPU cores with high base frequency (relative to the other cores in the system)\nwill be flagged as high priority.  Low base frequency will map to low priority,\ncorrespondingly.\n\nCPU cores with high EPP priority (relative to the other cores in the system)\nwill be marked as high priority cores.\n"
  },
  {
    "path": "docs/policy/dynamic-pools.md",
    "content": "# Dynamic-Pools Policy\n\n## Overview\n\nThe dynamic-pools policy can put the workload into different dynamic-pools. Each dynamic-pool contains several CPUs and can be resized dynamically in terms of the specific algorithms.\n\nThe main idea of the algorithm is: on the premise that the CPUs in each dynamic-pool can meet the requests of the pods in the dynamic-pool, the CPUs are allocated based on the CPU utilization of the workload. Dynamic-pools policy try to keep CPU utilization balanced.\n\nCPUs in dynamic-pools can be configured, for example, by setting min and max frequencies on CPU cores and uncore.\n\n## How It Works\n\n1. The user configures the dynamic-pool types from which the policy instantiates dynamic-pools. In addition to the dynamic-pools configured by the user, there is also a built-in dynamic-pool named shared pool.\n2. A dynamic-pool has a set of CPUs and a set of containers running on the CPUs.\n3. Every container is assigned to a dynamic-pool. Dynamic-pools policy allows a container to use all CPUs of its pool and no other CPUs.\n4. Each logical CPU belongs to exactly one dynamic-pool. There cannot be CPUs that do not belong to any dynamic-pool.\n5. The number of CPUs in a dynamic-pool can change. If CPUs are added to a dynamic-pool, then all containers in the dynamic-pool can use more CPUs. The opposite is true if the CPUs are removed.\n6. As CPUs are added to or removed from the dynamic-pool, the CPUs are reconfigured according to the dynamic-pool's CPU class attributes or the idle CPU class attributes.\n7. Updating the number of CPUs in dynamic-pools:\n   - The dynamic-pools policy needs to update the number of CPUs in dynamic-pools when starting policy, creating pods, deleting pods, updating configurations, and at regular intervals.\n   - The number of CPUs in the dynamic-pools is determined by the requests of containers and CPU utilization in the dynamic-pools.\n   - The number of CPUs allocated in each dynamic-pool is the sum of the requests of the containers in the dynamic pool and the CPUs allocated based on the CPU utilization of the workload.\n8. When a new container is created on a Kubernetes node, the policy first decides the type of the dynamic-pool that will run the container. The decision is based on the annotation of the pod, or the namespace if annotations are not given.\n\n## Deployment\n\n### Install cri-resmgr\n\nDeploy cri-resmgr on each node as you would for any other policy. See [installation](https://intel.github.io/cri-resource-manager/stable/docs/installation.html) for more details.\n\n## Configuration\n\nThe dynamic-pools policy is configured using the yaml-based configuration system of CRI-RM. See [setup and usage](https://intel.github.io/cri-resource-manager/stable/docs/setup.html#setting-up-cri-resource-manager) for more details on managing the configuration.\n\n### Parameters\n\nDynamic-pools policy parameters:\n\n* `PinCPU` controls pinning a container to CPUs of its dynamic-pool. The default is  `true`: the container cannot use other CPUs.\n* `PinMemory` controls pinning a container to the memories that are closest to the CPUs of its dynamic-pool. The default is `true`: allow using memory only from the closest NUMA nodes. Warning: this may cause kernel to kill workloads due to out-of-memory error when closest NUMA nodes do not have enough memory. In this situation consider switching this option `false`.\n* `ReservedPoolNamespaces` is a list of namespaces (wildcards allowed) that are assigned to the special reserved dynamic-pool, that is, will run on reserved CPUs. This always includes the `kube-system` namespace.\n* `DynamicPoolTypes` is a list of dynamic-pool type definitions. Each type can be configured with the following parameters:\n  - `Name` of the dynamic-pool type. This is used in pod annotations to assign containers to dynamic-pool of this type.\n  - `Namespaces` is a list of namespaces (wildcards allowed) whose pods should be assigned to this dynamic-pool type, unless overridden by pod annotations.\n  - `CpuClass` specifies the name of the CPU class according to which CPUs of dynamic-pools are configured.\n  - `AllocatorPriority` (0: High, 1: Normal, 2: Low, 3: None). CPU allocator parameter, used when creating new or resizing existing dynamic-pools.\n\nRelated configuration parameters:\n\n* `policy.ReservedResources.CPU` specifies the (number of) CPUs in the special `reserved` dynamic-pool. By default all containers in the `kube-system` namespace are assigned to the reserved dynamic-pool.\n* `policy.AvailableResources.CPU` specifies the CPUs that can be used by the policy, including `policy.ReservedResources.CPU`.\n* `cpu.classes` defines CPU classes and their parameters (such as `minFreq`, `maxFreq`, `uncoreMinFreq` and `uncoreMaxFreq`).\n\n### Example\n\n```yaml\ncpu:\n  classes:\n    pool1-cpuclass:\n      maxFreq: 1500000\n      minFreq: 2000000\n    pool2-cpuclass:\n      maxFreq: 2000000\n      minFreq: 2500000\npolicy:\n  Active: dynamic-pools\n  ReservedResources:\n      CPU: cpuset:0\n  dynamic-pools:\n    PinCPU: true\n    PinMemory: true\n    DynamicPoolTypes:\n      - Name: \"pool1\"\n        Namespaces:\n          - \"pool1\"\n        CPUClass: \"pool1-cpuclass\"\n      - Name: \"pool2\"\n        Namespaces:\n          - \"pool2\"\n        CPUClass: \"pool2-cpuclass\"\n```\n\n### Update Dynamic-Pools at Regular Intervals\n\nThe dynamic-pools policy can be set at regular intervals, based on the cpu utilization of the workload in each pool, to update the cpu allocation, and use the `--rebalance-interval` option to set the interval.\n\n### Assigning a Container to a Dynamic-pool\n\nThe dynamic-pool type of a container can be defined in pod annotations. In the example below, the first annotation sets the dynamic-pool type (`DPT`) of a single container (`CONTAINER_NAME`). The last two annotations set the default dynamic-pools type for all containers in the pod.\n\n```yaml\ndynamic-pool.dynamic-pools.cri-resource-manager.intel.com/container.CONTAINER_NAME: DPT\ndynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: DPT\ndynamic-pool.dynamic-pools.cri-resource-manager.intel.com: DPT\n```\n\nIf a pod has no annotations, its namespace is matched to the `namespace` of dynamic-pool types. The first matching dynamic-pool type is used.\n\nIf the namespace does not match, the container is assigned to the `shared` dynamic-pool.\n\n## Metrics and Debugging\n\nIn order to enable more verbose logging and metrics exporting from the dynamic-pools policy, enable instrumentation and policy debugging from the CRI-RM global config:\n\n```yaml\ninstrumentation:\n  # The dynamic-pools policy exports containers running in each dynamic-pool,\n  # and cpusets of dynamic-pools. Accessible in command line:\n  # curl --silent http://localhost:8891/metrics\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n```\n\nUse the `--metrics-interval` option to set the interval for updating metrics data."
  },
  {
    "path": "docs/policy/index.rst",
    "content": "Policies\n########\n\n.. toctree::\n   :maxdepth: 1\n\n   topology-aware.md\n   static-pools.md\n   balloons.md\n   podpools.md\n   container-affinity.md\n   blockio.md\n   rdt.md\n   cpu-allocator.md\n   dynamic-pools.md"
  },
  {
    "path": "docs/policy/podpools.md",
    "content": "# Podpools Policy\n\n## Overview\n\nThe podpools policy implements pod-level workload placement. It\nassigns all containers of a pod to the same CPU/memory pool. The\nnumber of CPUs in a pool is configurable by user.\n\n## Deployment\n\n### Install cri-resmgr\n\nDeploy cri-resmgr on each node as you would for any other policy. See\n[installation](../installation.md) for more details.\n\n## Configuration\n\nThe policy is configured using the yaml-based configuration system of CRI-RM.\nSee [setup and usage](../setup.md#setting-up-cri-resource-manager) for more\ndetails on managing the configuration.\n\nAt minimum, you need to specify the active policy in the\nconfiguration, and define at least one pod pool. For example, the\nfollowing configuration dedicates 95 % of non-reserved CPUs on the\nnode to be used by `dualcpu` pools. Every pool instance (`dualcpu[0]`,\n`dualcpu[1]`, ...) contains two exclusive CPUs and has a capacity\n(`MaxPods`) of one pod. The CPUs are used only by containers of pods\nassigned to the pool. Remaining CPUs will be used for running pods\nthat are not `dualcpu` or `kube-system` pods.\n\n```yaml\npolicy:\n  Active: podpools\n  ReservedResources:\n    CPU: 1\n  podpools:\n    Pools:\n      - Name: dualcpu\n        CPU: 2\n        MaxPods: 1\n        Instances: 95 %\n```\n\nNote that the configuration above allocates two exclusive CPUs for\neach pod assigned to the pool. To align with kube-scheduler resource\naccounting, requested CPUs of all containers in this kind of pods must\nsum up to CPU/MaxPods, that is 2000m CPU in this case.\n\nSee the [sample configmap](/sample-configs/podpools-policy.cfg) for a\ncomplete example.\n\n### Debugging\n\nIn order to enable more verbose logging for the podpools policy enable\npolicy debug from the CRI-RM global config:\n\n```yaml\nlogger:\n  Debug: policy\n```\n\n## Running Pods in Podpools\n\nThe podpools policy assigns a pod to a pod pool instance if the pod\nhas annotation\n\n```yaml\npool.podpools.cri-resource-manager.intel.com: POOLNAME\n```\n\nFollowing Pod runs in a `dualcpu` pool. This example assumes that\n`dualcpu` pools include two CPUs per pod, as in the above\nconfiguration example. Therefore containers in the yaml request 2000m\nCPUs in total.\n\n```yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: podpools-test\n  annotations:\n    pool.podpools.cri-resource-manager.intel.com: dualcpu\nspec:\n  containers:\n  - name: testcont0\n    image: busybox\n    command:\n      - \"sh\"\n      - \"-c\"\n      - \"while :; do grep _allowed_list /proc/self/status; sleep 5; done\"\n    resources:\n      requests:\n        cpu: 1200m\n  - name: testcont1\n    image: busybox\n    command:\n      - \"sh\"\n      - \"-c\"\n      - \"while :; do grep _allowed_list /proc/self/status; sleep 5; done\"\n    resources:\n      requests:\n        cpu: 800m\n```\n\nIf a pod is not annotated to run on any specific pod pool and it is\nnot a `kube-system` pod, it will be run on shared CPUs. Shared CPUs\ninclude left-over CPUs after creating user-defined pools. If all CPUs\nwere allocated to other pools, reserved CPUs will be used as shared,\ntoo.\n"
  },
  {
    "path": "docs/policy/rdt.md",
    "content": "# RDT (Intel® Resource Director Technology)\n\n## Background\n\nIntel® RDT provides capabilities for cache and memory allocation and\nmonitoring. In Linux system the functionality is exposed to the user space via\nthe [resctrl](https://docs.kernel.org/x86/resctrl.html)\nfilesystem. Cache and memory allocation in RDT is handled by using resource\ncontrol groups. Resource allocation is specified on the group level and each\ntask (process/thread) is assigned to one group. In the context of CRI Resource\nwe use the term 'RDT class' instead of 'resource control group'.\n\nCRI Resource Manager supports all available RDT technologies, i.e. L2 and L3\nCache Allocation (CAT) with Code and Data Prioritization (CDP) and Memory\nBandwidth Allocation (MBA) plus Cache Monitoring (CMT) and Memory Bandwidth\nMonitoring (MBM).\n\n\n## Overview\n\nRDT configuration in CRI-RM is class-based. Each container gets assigned to an\nRDT class. In turn, all processes of the container will be assigned to the RDT\nClasses of Service (CLOS) (under `/sys/fs/resctrl`) corresponding the RDT class. CRI-RM will configure\nthe CLOSes according to its configuration at startup or whenever the\nconfiguration changes.\n\nCRI-RM maintains a direct mapping between Pod QoS classes and RDT classes. If\nRDT is enabled CRI-RM tries to assign containers into an RDT class with a name\nmatching their Pod QoS class. This default behavior can be overridden with\npod annotations.\n\n## Class Assignment\n\nBy default, containers get an RDT class with the same name as its Pod QoS class\n(Guaranteed, Burstable or Besteffort). If the RDT class is missing the\ncontainer will be assigned to the system root class.\n\nThe default behavior can be overridden with pod annotations:\n\n- `rdtclass.cri-resource-manager.intel.com/pod: <class-name>` specifies a\n  pod-level default that will be used for all containers of a pod\n- `rdtclass.cri-resource-manager.intel.com/container.<container-name>: <class-name>`\n   specifies container-specific assignment, taking preference over possible\n   pod-level annotation (above)\n\nWith pod annotations it is possible to specify RDT classes other than\nGuaranteed, Burstable or Besteffort.\n\nThe default assignment could also be overridden by a policy but currently none\nof the builtin policies do that.\n\n## Configuration\n\n### Operating Modes\n\nThe RDT controller supports three operating modes, controlled by\n`rdt.options.mode` configuration option.\n\n- Disabled: RDT controller is effectively disabled and containers will not be\n  assigned and no monitoring groups will be created. Upon activation of this\n  mode all CRI-RM specific control and monitoring groups from the resctrl\n  filesystem are removed.\n- Discovery: RDT controller detects existing non-CRI-RM specific classes from\n  the resctrl filesystem and uses these. The configuration of the discovered\n  classes is considered read-only and it will not be altered. Upon activation\n  of this mode all CRI-RM specific control groups from the resctrl filesystem\n  are removed.\n- Full: Full operating mode. The controller manages the configuration of the\n  resctrl filesystem according to the rdt class definitions in the CRI-RM\n  configuration. This is the default operating mode.\n\n### RDT Classes\n\nThe RDT class configuration in CRI-RM is a two-level hierarchy consisting of\npartitions and classes. It specifies a set of partitions each having a set of\nclasses.\n\n#### Partitions\n\nPartitions represent a logical grouping of the underlying classes, each\npartition specifying a portion of the available resources (L2/L3/MB) which will\nbe shared by the classes under it. Partitions guarantee non-overlapping\nexclusive cache allocation - i.e. no overlap on the cache ways between\npartitions is allowed. However, by technology, MB allocations are not\nexclusive. Thus, it is possible to assign all partitions 100% of memory\nbandwidth, for example.\n\n#### Classes\n\nClasses represent the actual RDT classes containers are assigned to. In\ncontrast to partitions, cache allocation between classes under a specific\npartition may overlap (and they usually do).\n\nThe set of RDT classes can be freely specified, but, it should be ensured that\nclasses corresponding to the Pod QoS classes are specified. Also, the maximum\nnumber of classes (CLOSes) supported by the underlying hardware must not be\nexceeded.\n\n### Example\n\nBelow is a config snippet that would allocate 60% of the L3 cache lines\nexclusively to the Guarenteed class. The remaining 40% L3 is for Burstable and\nBesteffort, Besteffort getting only 50% of this. Guaranteed class gets full\nmemory bandwidth whereas the other classes are throttled to 50%.\n\n```yaml\nrdt:\n  # Common options\n  options:\n    # One of Full, Discovery or Disabled\n    mode: Full\n    # Set to true to disable creation of monitoring groups\n    monitoringDisabled: false\n    l3:\n      # Make this false if L3 CAT must be available\n      optional: true\n    mb:\n      # Make this false if MBA must be available\n      optional: true\n\n  # Configuration of classes\n  partitions:\n    exclusive:\n      # Allocate 60% of all L3 cache to the \"exclusive\" partition\n      l3Allocation: \"60%\"\n      mbAllocation: [\"100%\"]\n      classes:\n        Guaranteed:\n          # Allocate all of the partitions cache lines to \"Guaranteed\"\n          l3Allocation: \"100%\"\n    shared:\n      # Allocate 40% L3 cache IDs to the \"shared\" partition\n      # These will NOT overlap with the cache lines allocated for \"exclusive\" partition\n      l3Allocation: \"40%\"\n      mbAllocation: [\"50%\"]\n      classes:\n        Burstable:\n          # Allow \"Burstable\" to use all cache lines of the \"shared\" partition\n          l3Allocation: \"100%\"\n        BestEffort:\n          # Allow \"Besteffort\" to use only half of the L3 cache # lines of the \"shared\" partition.\n          # These will overlap with those used by \"Burstable\"\n          l3Allocation: \"50%\"\n```\n\nThe configuration also supports far more fine-grained control, e.g. per\ncache-ID configuration (i.e. different sockets having different allocation) and\nCode and Data Prioritization (CDP) allowing different cache allocation for code\nand data paths. If the hardware details are known, raw bitmasks or bit numbers\n(\"0x1f\" or 0-4) can be used instead of percentages in order to be able to\nconfigure cache allocations exactly as required. For detailed description of the RDT configuration format with examples see the\n{{ '[goresctrl library documentation](https://github.com/intel/goresctrl/blob/{}/doc/rdt.md)'.format(goresctrl_version) }}\n\nSee `rdt` in the [example ConfigMap spec](/sample-configs/cri-resmgr-configmap.example.yaml)\nfor another example configuration.\n\n### Dynamic Configuration\n\nRDT supports dynamic configuration i.e. the resctrl filesystem is reconfigured\nwhenever a configuration update e.g. via the [Node Agent](../node-agent.md) is\nreceived. However, the configuration update is rejected if it is incompatible\nwith the set of currently running containers - e.g. the new config is missing a\nclass that a running container has been assigned to.\n"
  },
  {
    "path": "docs/policy/static-pools.md",
    "content": "# Static-Pools (STP) Policy\n\n## Overview\n\nThe `static-pools` (STP) builtin policy was inspired by [CMK (CPU Manager for\nKubernetes)][cmk]. It is an example policy demonstrating capabilities of\n`cri-resource-manager` and not considered as production ready.\n\nBasically, the STP policy aims to replicate the functionality of the `cmk\nisolate` command of CMK. It also has compatibility features to function as\na drop-in replacement in order to allow easier testing and prototyping.\n\nFeatures:\n\n- arbitrary number of configurable CPU list pools\n- dynamic configuration updates via the [node agent](../node-agent.md)\n\nPlease see the documentation of\n[CMK][cmk] for a more detailed\ndescription of the terminology and functionality.\n\nCMK compatibility features:\n\n- supports the same environment variables as the original CMK, except for:\n  - `CMK_LOCK_TIMEOUT` and `CMK_PROC_FS`: configuration variables that are not\n    applicable in cri-resmgr context\n  - `CMK_LOG_LEVEL`: not implemented, yet\n  - `CMK_NUM_CORES`: not needed in cri-resmgr as we take this value directly\n    from the container resource request\n- supports the existing configuration directory format of CMK for retrieving\n  the pool configuration\n- parses the container command/args in an attempt to retrieve command line\n  options of `cmk isolate`\n- supports generating CMK-specific node label and taint (off by default)\n\n## Deployment\n\n### Install cri-resmgr\n\nDeploy cri-resmgr on each node as you would for any other policy. See\n[installation](../installation.md) for more details.\n\n### Deploy Node Agent\n\nThe CRI-RM node agent is required in order to communicate with the Kubernetes\ncontrol plane. In particular, the STP policy needs this capability for\nupdating the extended resource (that represents exclusive cores) as well as\nmanaging legacy CMK node annotation and taint. In addition, the node agent\nenables dynamic configuration updates.\n\nSee [node agent](../node-agent.md) for detailed instructions for set-up and\nusage.\n\n### Deploy Admission Webhook\n\nYou need to run and enable the cri-resmgr mutating admission webhook which\ncreates pod annotations consumed by CRI-RM. This is required so that the STP\npolicy is able to inspect the extended resources (in this case, exclusive CPU\ncores) requested by containers.\n\nSee the [webhook](../webhook.md) for instructions how to set it up.\n\n## Configuration\n\nThe policy is configured using the yaml-based configuration system of CRI-RM.\nSee [setup and usage](../setup.md#setting-up-cri-resource-manager) for more\ndetails on managing the configuration.\n\nAt minimum, you need to specify the active policy in the configuration.\nPolicy-specific options control the pool configuration and legacy node label\nand taint.\n\n```yaml\npolicy:\n  Active: static-pools\n  static-pools:\n    # Set to true to create CMK node label\n    #LabelNode: false\n    # Set to true to create CMK node taint\n    #TaintNode: false\n  ...\n```\n\nSee the [sample configmap](/sample-configs/cri-resmgr-configmap.example.yaml)\nfor a complete example containing all available configuration options.\n\nIf dynamic configuration via the [node agent](../node-agent.md) is in use the\npolicy options, including pools configuration, may be altered at runtime.\n\n**NOTE**: the active policy (`policy.Active`) cannot be changed at runtime. In\norder to change the active policy cri-resmgr needs to be restarted.\n\n### Pools Configuration\n\nThere are three possible sources of the pools configuration, in decreasing\npriority order:\n\n1. CRI-RM global config\n1. stand-alone static-pools config file\n1. CMK directory tree\n\nThe configuration is fully evaluated whenever a re-configuration event is\nreceived (e.g. from the node agent). Thus, a valid pools config appearing in\nthe CRI-RM global config will take precedence over a directory tree based\nconfig that was previously active. Similarly, removing pools config from the\nCRI-RM global config will make a local config (file or directory tree)\neffective.\n\n**NOTE:** cri-resmgr does not have any utility for generating a pool\nconfiguration. Thus, you need to either manually write one by yourself, or, run\nthe `cmk init` command (of the original CMK) in order to create a legacy\nconfiguration directory structure.\n\n#### Global Config\n\nConfiguration from the global CRI-RM config takes the highest preference, if\nspecified (under `policy.static-pools.pools`). A referential example:\n\n```yaml\npolicy:\n  static-pools:\n    pools:\n      exclusive:\n        exclusive: true\n        cpuLists:\n        ...\n      shared:\n        cpuLists:\n        ...\n      infra:\n        cpuLists:\n        ...\n\n```\n\n#### Stand-alone YAML File\n\nPath to a stand-alone configuration file can be specified by the\n`policy.static-pools.ConfFilePath` option (empty by default) in the CRI-RM\nglobal config:\n\n```yaml\npolicy:\n  static-pools:\n    ConfFilePath: \"/path/to/conf.yaml\"\n```\n\nFormat of the configuration file is similar to the pools config used in the\nglobal CRI-RM config. You can also see the\n[example config file](/sample-configs/static-pools-policy.conf.example)\nfor a starting point.\n\n#### CMK Directory Tree\n\nThe STP policy also supports configuration directory format of the original\nCMK. It reads the configuration from a location specified by the\n`policy.static-pools.ConfFileDir` field (`/etc/cmk` by default) in the CRI-RM\nglobal config:\n\n```yaml\npolicy:\n  static-pools:\n    ConfFileDir: \"/etc/cmk\"\n```\n\n### Debugging\n\nIn order to enable more verbose logging for the STP policy set the\n`LOGGER_DEBUG=static-pools` environment variable or enable debug from the CRI-RM\nglobal config:\n\n```yaml\nlogger:\n  Debug: static-pools\n\n```\n\n## Running Workloads\n\nThe preferred way to specify the pod configuration is through environment\nvariables. However, exclusive cores must be reserved by making a request of the\n`cmk.intel.com/exclusive-cores` extended resource. Naming of the extended\nresource has `cmk` prefix in order to provide backwards compatibility with the\noriginal CMK.\n\n### Pod Configuration Using Env Variables\n\nThe following environment variables are recognized:\n\n| Name            | Description                                                |\n| --------------- | ---------------------------------------------------------- |\n| STP_NO_AFFINITY | Do not set cpu affinity. The workload is responsible for reading the `CMK_CPUS_ASSIGNED` environment variable and setting the affinity itself.\n| STP_POOL        | Name of the pool to run in\n| STP_SOCKET_ID   | Socket where cores should be allocated. Set to -1 to accept any socket.\n\nAn example Pod spec for running a workload in the `exclusive` pool with one\ncore reserved from socket id 0:\n\n```yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: stp-test\nspec:\n  containers:\n  - name: stp-test\n    image: busybox\n    env:\n      - name: STP_POOL\n        value: \"exclusive\"\n      - name: STP_SOCKET_ID\n        value: \"0\"\n    command:\n      - \"sh\"\n      - \"-c\"\n      - \"while :; do echo ASSIGNED: $CMK_CPUS_ASSIGNED; sleep 1; done\"\n    resources:\n      requests:\n        cmk.intel.com/exclusive-cores: \"1\"\n      limits:\n        cmk.intel.com/exclusive-cores: \"1\"\n```\n\n### Backwards Compatibility for `cmk isolate`\n\nThe STP policy parses the container command/args in an attempt to\nretrieve the Pod configuration (from `cmk isolate` options). This is to provide\nbackwards compatibility with existing CMK workload specs. It manipulates the\ncontainer command and args so that `cmk isolate` and all it's arguments are\nremoved.\n\nIn the example below STP policy will run `sh -c \"sleep 10000\"` in the `infra`\npool.\n\n```yaml\napiVersion: v1\nkind: Pod\nmetadata:\n  name: cmk-test\nspec:\n  containers:\n  - name: cmk-test\n    image: busybox\n    command:\n      - \"sh\"\n      - \"-c\"\n    args:\n      - \"/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=infra sleep 10000\"\n```\n\n<!-- Links -->\n[cmk]: https://github.com/intel/CPU-Manager-for-Kubernetes\n"
  },
  {
    "path": "docs/policy/topology-aware.md",
    "content": "# Topology-Aware Policy\n\n## Background\n\nOn server-grade hardware the CPU cores, I/O devices and other peripherals\nform a rather complex network together with the memory controllers, the\nI/O bus hierarchy and the CPU interconnect. When a combination of these\nresources are allocated to a single workload, the performance of that\nworkload can vary greatly, depending on how efficiently data is transferred\nbetween them or, in other words, on how well the resources are aligned.\n\nThere are a number of inherent architectural hardware properties that,\nunless properly taken into account, can cause resource misalignment and\nworkload performance degradation. There are a multitude of CPU cores\navailable to run workloads. There are a multitude of memory controllers\nthese workloads can use to store and retrieve data from main memory. There\nare a multitude of I/O devices attached to a number of I/O buses the same\nworkloads can access. The CPU cores can be divided into a number of groups,\nwith each group having different access latency and bandwidth to each\nmemory controller and I/O device.\n\nIf a workload is not assigned to run with a properly aligned set of CPU,\nmemory and devices, it will not be able to achieve optimal performance.\nGiven the idiosyncrasies of hardware, allocating a properly aligned set\nof resources for optimal workload performance requires identifying and\nunderstanding the multiple dimensions of access latency locality present\nin hardware or, in other words, hardware topology awareness.\n\n## Overview\n\nThe `topology-aware` policy automatically builds a tree of pools based on the\ndetected hardware topology. Each pool has a set of CPUs and memory zones\nassigned as their resources. Resource allocation for workloads happens by\nfirst picking the pool which is considered to fit the best the resource\nrequirements of the workload and then assigning CPU and memory from this pool.\n\nThe pool nodes at various depths from bottom to top represent the NUMA nodes,\ndies, sockets, and finally the whole of the system at the root node. Leaf NUMA\nnodes are assigned the memory behind their controllers / zones and CPU cores\nwith the smallest distance / access penalty to this memory. If the machine\nhas multiple types of memory separately visible to both the kernel and user\nspace, for instance both DRAM and [PMEM](https://www.intel.com/content/www/us/en/products/memory-storage/optane-dc-persistent-memory.html), each zone of special type of memory\nis assigned to the closest NUMA node pool.\n\nEach non-leaf pool node in the tree is assigned the union of the resources of\nits children. So in practice, dies nodes end up containing all the CPU cores\nand the memory zones in the corresponding die, sockets nodes end up containing\nthe CPU cores and memory zones in the corresponding socket's dies, and the root\nends up containing all CPU cores and memory zones in all sockets.\n\nWith this setup, each pool in the tree has a topologically aligned set of CPU\nand memory resources. The amount of available resources gradually increases in\nthe tree from bottom to top, while the strictness of alignment is gradually\nrelaxed. In other words, as one moves from bottom to top in the tree, it is\ngetting gradually easier to fit in a workload, but the price paid for this is\na gradually increasing maximum potential cost or penalty for memory access and\ndata transfer between CPU cores.\n\nAnother property of this setup is that the resource sets of sibling pools at\nthe same depth in the tree are disjoint while the resource sets of descendant\npools along the same path in the tree partially overlap, with the intersection\ndecreasing as the the distance between pools increases. This makes it easy to \nisolate workloads from each other. As long as workloads are assigned to pools\nwhich has no other common ancestor than the root, the resources of these\nworkloads should be as well isolated from each other as possible on the given\nhardware.\n\nWith such an arrangement, this policy should handle topology-aware alignment\nof resources without any special or extra configuration. When allocating\nresources, the policy\n\n  - filters out all pools with insufficient free capacity\n  - runs a scoring algorithm for the remaining ones\n  - picks the one with the best score\n  - assigns resources to the workload from there\n\nAlthough the details of the scoring algorithm are subject to change as the\nimplementation evolves, its basic principles are roughly\n\n  - prefer pools lower in the tree, IOW stricter alignment and lower latency\n  - prefer idle pools over busy ones, IOW more remaining free capacity and\n    fewer workloads\n  - prefer pools with better overall device alignment\n\n## Features\n\nThe `topology-aware` policy has the following features:\n\n  - topologically aligned allocation of CPU and memory\n    * assign CPU and memory to workloads with tightest available alignment\n  - aligned allocation of devices\n    * pick pool for workload based on locality of devices already assigned\n  - shared allocation of CPU cores\n    * assign workload to shared subset of pool CPUs\n  - exclusive allocation of CPU cores\n    * dynamically slice off CPU cores from shared subset and assign to workload\n  - mixed allocation of CPU cores\n    * assign both exclusive and shared CPU cores to workload\n  - discovering and using kernel-isolated CPU cores (['isolcpus'](https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html#cpu-lists))\n    * use kernel-isolated CPU cores for exclusively assigned CPU cores\n  - exposing assigned resources to workloads\n  - notifying workloads about changes in resource assignment\n  - dynamic relaxation of memory alignment to prevent OOM\n    * dynamically widen workload memory set to avoid pool/workload OOM\n  - multi-tier memory allocation\n    * assign workloads to memory zones of their preferred type\n    * the policy knows about three kinds of memory:\n      - DRAM is regular system main memory\n      - PMEM is large-capacity memory, such as\n        [Intel® Optane™ memory](https://www.intel.com/content/www/us/en/products/memory-storage/optane-dc-persistent-memory.html)\n      - [HBM](https://en.wikipedia.org/wiki/High_Bandwidth_Memory) is high speed memory,\n        typically found on some special-purpose computing systems\n  - cold start\n    * pin workload exclusively to PMEM for an initial warm-up period\n  - dynamic page demotion\n    * forcibly migrate read-only and idle container memory pages to PMEM\n\n## Activating the Policy\n\nYou can activate the `topology-aware` policy by using the following configuration\nfragment in the configuration for `cri-resmgr`:\n\n```yaml\npolicy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\n```\n\n## Configuring the Policy\n\nThe policy has a number of configuration options which affect its default behavior.\nThese options can be supplied as part of the\n[dynamic configuration](../setup.md#using-cri-resource-manager-agent-and-a-configmap)\nreceived via the [`node agent`](../node-agent.md), or in a fallback or forced\n[configuration file](../setup.md#using-a-local-configuration-from-a-file). These\nconfiguration options are\n\n  - `PinCPU`\n    * whether to pin workloads to assigned pool CPU sets\n  - `PinMemory`\n    * whether to pin workloads to assigned pool memory zones\n  - `PreferIsolatedCPUs`\n    * whether isolated CPUs are preferred by default for workloads that are\n      eligible for exclusive CPU allocation\n  - `PreferSharedCPUs`\n    * whether shared allocation is preferred by default for workloads that\n      would be otherwise eligible for exclusive CPU allocation\n  - `ReservedPoolNamespaces`\n    * list of extra namespaces (or glob patters) that will be allocated to reserved CPUs\n  - `ColocatePods`\n    * whether try to allocate containers in a pod to the same or close by topology pools\n  - `ColocateNamespaces`\n    * whether try to allocate containers in a namespace to the same or close by topology pools\n\n## Policy CPU Allocation Preferences\n\nThere are a number of workload properties this policy actively checks to decide\nif the workload could potentially benefit from extra resource allocation\noptimizations. Unless configured differently, containers fulfilling certain\ncorresponding criteria are considered eligible for these optimizations. This\nwill be reflected in the assigned resources whenever that is possible at the\ntime the container's creation / resource allocation request hits the policy.\n\nThe set of these extra optimizations consist of\n\n  - assignment of `kube-reserved` CPUs\n  - assignment of exclusively allocated CPU cores\n  - usage of kernel-isolated CPU cores (for exclusive allocation)\n\nThe policy uses a combination of the QoS class and the resource requirements of\nthe container to decide if any of these extra allocation preferences should be\napplied. Containers are divided into five groups, with each group having a\nslightly different set of criteria for eligibility.\n\n  - `kube-system` group\n    * all containers in the `kube-system` namespace\n  - `low-priority` group\n    * containers in the `BestEffort` or `Burstable` QoS class\n  - `sub-core` group\n    * Guaranteed QoS class containers with `CPU request < 1 CPU`\n  - `mixed` group\n    * Guaranteed QoS class containers with `1 <= CPU request < 2`\n  - `multi-core` group\n    * Guaranteed QoS class containers with `CPU request >= 2`\n\nThe eligibility rules for extra optimization are slightly different among these\ngroups.\n\n  - `kube-system`\n    * not eligible for extra optimizations\n    * eligible to run on `kube-reserved` CPU cores\n    * always run on shared CPU cores\n  - `low-priority`\n    * not eligible for extra optimizations\n    * always run on shared CPU cores\n  - `sub-core`\n    * not eligible for extra optimizations\n    * always run on shared CPU cores\n  - `mixed`\n    * by default eligible for exclusive and isolated allocation\n    * not eligible for either if `PreferSharedCPUs` is set to true\n    * not eligible for either if annotated to opt out from exclusive allocation\n    * not eligible for isolated allocation if annotated to opt out\n  - `multi-core`\n    * CPU request fractional (`(CPU request % 1000 milli-CPU) != 0`):\n      - by default not eligible for extra optimizations\n      - eligible for exclusive and isolated allocation if annotated to opt in\n    * CPU request not fractional:\n      - by default eligible for exclusive allocation\n      - by default not eligible for isolated allocation\n      - not eligible for exclusive allocation if annotated to opt out\n      - eligible for isolated allocation if annotated to opt in\n\nEligibility for kube-reserved CPU core allocation should always be possible to\nhonor. If this is not the case, it is probably due to an incorrect configuration\nwhich underdeclares `ReservedResources`. In that case, ordinary shared CPU cores\nwill be used instead of kube-reserved ones.\n\nEligibility for exclusive CPU allocation should always be possible to honor.\nEligibility for isolated core allocation is only honored if there are enough\nisolated cores available to fulfill the exclusive part of the container's CPU\nrequest with isolated cores alone. Otherwise ordinary CPUs will be allocated,\nby slicing them off for exclusive usage from the shared subset of CPU cores in\nthe container's assigned pool.\n\nContainers in the kube-system group are pinned to share all kube-reserved CPU\ncores. Containers in the low-priority or sub-core groups, and containers which\nare only eligible for shared CPU core allocation in the mixed and multi-core\ngroups, are all pinned to run on the shared subset of CPU cores in the\ncontainer's assigned pool. This shared subset can and usually does change\ndynamically as exclusive CPU cores are allocated and released in the pool.\n\n## Container CPU Allocation Preference Annotations\n\nContainers can be annotated to diverge from the default CPU allocation\npreferences the policy would otherwise apply to them. These Pod annotations\ncan be given both with per pod and per container resolution. If for any\ncontainer both of these exist, the container-specific one takes precedence.\n\n### Shared, Exclusive, and Isolated CPU Preference\n\nA container can opt in to or opt out from shared CPU allocation using the\nfollowing Pod annotation.\n\n```yaml\nmetadata:\n  annotations:\n    # opt in container C1 to shared CPU core allocation\n    prefer-shared-cpus.cri-resource-manager.intel.com/container.C1: \"true\"\n    # opt in the whole pod to shared CPU core allocation\n    prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"true\"\n    # selectively opt out container C2 from shared CPU core allocation\n    prefer-shared-cpus.cri-resource-manager.intel.com/container.C2: \"false\"\n```\n\nOpting in to exclusive allocation happens by opting out from shared allocation,\nand opting out from exclusive allocation happens by opting in to shared\nallocation.\n\nA container can opt in to or opt out from isolated exclusive CPU core\nallocation using the following Pod annotation.\n\n```yaml\nmetadata:\n  annotations:\n    # opt in container C1 to isolated exclusive CPU core allocation\n    prefer-isolated-cpus.cri-resource-manager.intel.com/container.C1: \"true\"\n    # opt in the whole pod to isolated exclusive CPU core allocation\n    prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"\n    # selectively opt out container C2 from isolated exclusive CPU core allocation\n    prefer-isolated-cpus.cri-resource-manager.intel.com/container.C2: \"false\"\n```\n\nThese Pod annotations have no effect on containers which are not eligible for\nexclusive allocation.\n\n### Implicit Hardware Topology Hints\n\n`CRI Resource Manager` automatically generates HW `Topology Hints` for devices\nassigned to a container, prior to handing the container off to the active policy\nfor resource allocation. The `topology-aware` policy is hint-aware and normally\ntakes topology hints into account when picking the best pool to allocate\nresources. Hints indicate optimal `HW locality` for device access and they can\nalter significantly which pool gets picked for a container.\n\nSince device topology hints are implicitly generated, there are cases where one\nwould like the policy to disregard them altogether. For instance, when a local\nvolume is used by a container but not in any performance critical manner.\n\nContainers can be annotated to opt out from and selectively opt in to hint-aware\npool selection using the following Pod annotations.\n\n```yaml\nmetadata:\n  annotations:\n    # only disregard hints for container C1\n    topologyhints.cri-resource-manager.intel.com/container.C1: \"false\"\n    # disregard hints for all containers by default\n    topologyhints.cri-resource-manager.intel.com/pod: \"false\"\n    # but take hints into account for container C2\n    topologyhints.cri-resource-manager.intel.com/container.C2: \"true\"\n```\n\nTopology hint generation is globally enabled by default. Therefore, using the\nPod annotation as opt in only has an effect when the whole pod is annotated to\nopt out from hint-aware pool selection.\n\n### Implicit Topological Co-location for Pods and Namespaces\n\nThe `ColocatePods` or `ColocateNamespaces` configuration options control whether\nthe policy will try to co-locate, that is allocate topologically close, containers\nwithin the same Pod or K8s namespace.\n\nBoth of these options are false by default. Setting them to true is a shorthand\nfor adding to each container an affinity of weight 10 for all other containers\nin the same pod or namespace.\n\nContainers with user-defined affinities are never extended with either of these\nco-location affinities. However, such containers can still have affinity effects\non other containers that do get extended with co-location. Therefore mixing user-\ndefined affinities with implicit co-location requires both careful consideration\nand a thorough understanding of affinity evaluation, or it should be avoided\naltogether.\n\n## Cold Start\n\nThe `topology-aware` policy supports \"cold start\" functionality. When cold start\nis enabled and the workload is allocated to a topology node with both DRAM and\nPMEM memory, the initial memory controller is only the PMEM controller. DRAM\ncontroller is added to the workload only after the cold start timeout is\ndone. The effect of this is that allocated large unused memory areas of\nmemory don't need to be migrated to PMEM, because it was allocated there to\nbegin with. Cold start is configured like this in the pod metadata:\n\n```yaml\nmetadata:\n  annotations:\n    memory-type.cri-resource-manager.intel.com/container.container1: dram,pmem\n    cold-start.cri-resource-manager.intel.com/container.container1: |\n      duration: 60s\n```\n\nAgain, alternatively you can use the following deprecated Pod annotation syntax\nto achieve the same, but support for this syntax is subject to be dropped in a\nfuture release:\n\n```yaml\nmetadata:\n  annotations:\n    cri-resource-manager.intel.com/memory-type: |\n      container1: dram,pmem\n    cri-resource-manager.intel.com/cold-start: |\n      container1:\n        duration: 60s\n```\n\nIn the above example, `container1` would be initially granted only PMEM\nmemory controller, but after 60 seconds the DRAM controller would be\nadded to the container memset.\n\n## Dynamic Page Demotion\n\nThe `topology-aware` policy also supports dynamic page demotion. With dynamic\ndemotion enabled, rarely-used pages are periodically moved from DRAM to PMEM\nfor those workloads which are assigned to use both DRAM and PMEM memory types.\nThe configuration for this feature is done using three configuration keys:\n`DirtyBitScanPeriod`, `PageMovePeriod`, and `PageMoveCount`. All of these\nparameters need to be set to non-zero values in order for dynamic page demotion\nto get enabled. See this configuration file fragment as an example:\n\n```yaml\npolicy:\n  Active: topology-aware\n  topology-aware:\n    DirtyBitScanPeriod: 10s\n    PageMovePeriod: 2s\n    PageMoveCount: 1000\n```\n\nIn this setup, every pid in every container in every non-system pod\nfulfilling the memory container requirements would have their page ranges\nscanned for non-accessed pages every ten seconds. The result of the scan\nwould be fed to a page-moving loop, which would attempt to move 1000 pages\nevery two seconds from DRAM to PMEM.\n\n## Container memory requests and limits\n\nDue to inaccuracies in how `cri-resmgr` calculates memory requests for\npods in QoS class `Burstable`, you should either use `Limit` for setting\nthe amount of memory for containers in `Burstable` pods or run the\n[resource-annotating webhook](../webhook.md) to provide `cri-resmgr` with\nan exact copy of the resource requirements from the Pod Spec as an extra\nPod annotation.\n\n## Reserved pool namespaces\n\nUser is able to mark certain namespaces to have a reserved CPU allocation.\nContainers belonging to such namespaces will only run on CPUs set aside\naccording to the global CPU reservation, as configured by the ReservedResources\nconfiguration option in the policy section.\nThe `ReservedPoolNamespaces` option is a list of namespace globs that will be\nallocated to reserved CPU class.\n\nFor example:\n\n```yaml\npolicy:\n  Active: topology-aware\n  topology-aware:\n    ReservedPoolNamespaces: [\"my-pool\",\"reserved-*\"]\n```\n\nIn this setup, all the workloads in `my-pool` namespace and those namespaces\nstarting with `reserved-` string are allocated to reserved CPU class.\nThe workloads in `kube-system` are automatically assigned to reserved CPU\nclass so no need to mention `kube-system` in this list.\n\n## Reserved CPU annotations\n\nUser is able to mark certain pods and containers to have a reserved CPU allocation\nby using annotations. Containers having a such annotation will only run on CPUs set\naside according to the global CPU reservation, as configured by the ReservedResources\nconfiguration option in the policy section.\n\nFor example:\n\n```yaml\nmetadata:\n  annotations:\n    prefer-reserved-cpus.cri-resource-manager.intel.com/pod: \"true\"\n    prefer-reserved-cpus.cri-resource-manager.intel.com/container.special: \"false\"\n```\n"
  },
  {
    "path": "docs/quick-start.md",
    "content": "# Quick-start\n\nThe following describes the minimum number of steps to get started with CRI\nResource Manager.\n\n## Pre-requisites\n\n- containerd container runtime installed and running\n- kubelet installed on your nodes\n\n## Setup CRI-Resmgr\n\nFirst, install and setup cri-resource-manager.\n\n### Install package\n\n#### Fedora\\*, and SUSE\\*\n\n```\nCRIRM_VERSION=`curl -s \"https://api.github.com/repos/intel/cri-resource-manager/releases/latest\" | \\\n               jq .tag_name | tr -d '\"v'`\nsource /etc/os-release\n[ \"$ID\" = \"sles\" ] && export ID=opensuse-leap\nsudo rpm -Uvh https://github.com/intel/cri-resource-manager/releases/download/v${CRIRM_VERSION}/cri-resource-manager-${CRIRM_VERSION}-0.${ID}-${VERSION_ID}.x86_64.rpm\n```\n\n#### Ubuntu\\* and Debian\\*\n```\nCRIRM_VERSION=`curl -s \"https://api.github.com/repos/intel/cri-resource-manager/releases/latest\" | \\\n               jq .tag_name | tr -d '\"v'`\nsource /etc/os-release\npkg=cri-resource-manager_${CRIRM_VERSION}_${ID}-${VERSION_ID}_amd64.deb; curl -LO https://github.com/intel/cri-resource-manager/releases/download/v${CRIRM_VERSION}/${pkg}; sudo dpkg -i ${pkg}; rm ${pkg}\n```\n\n\n### Setup and verify\n\nCreate configuration and start cri-resource-manager\n```\nsudo cp /etc/cri-resmgr/fallback.cfg.sample /etc/cri-resmgr/fallback.cfg\nsudo systemctl enable cri-resource-manager && sudo systemctl start cri-resource-manager\n```\n\nSee that cri-resource-manager is running\n```\nsystemctl status cri-resource-manager\n```\n\n## Kubelet setup\n\nNext, you need to configure kubelet to use cri-resource-manager as it's\ncontainer runtime endpoint.\n\n### Existing cluster\n\nWhen integrating into an existing cluster you need to change kubelet to use\ncri-resmgr instead of the existing container runtime (expecting containerd\nhere).\n\n#### Fedora, and SUSE\n```\nsudo sed '/KUBELET_EXTRA_ARGS/ s!$! --container-runtime-endpoint=/var/run/cri-resmgr/cri-resmgr.sock!' -i /etc/sysconfig/kubelet\nsudo systemctl restart kubelet\n```\n\n#### Ubuntu and Debian\n```\nsudo sed '/KUBELET_EXTRA_ARGS/ s!$! --container-runtime-endpoint=/var/run/cri-resmgr/cri-resmgr.sock!' -i /etc/default/kubelet\nsudo systemctl restart kubelet\n```\n\n### New Cluster\n\nWhen in the process of setting up a new cluster you simply point the kubelet\nto use the cri-resmgr cri sockets on cluster node setup time. Here's an\nexample with kubeadm:\n```\nkubeadm join --cri-socket /var/run/cri-resmgr/cri-resmgr.sock \\\n...\n\n```\n\n## What Next\n\nCongratulations, you now have cri-resource-manager running on your system and\npolicying container resource allocations. Next, you could see:\n- [Installation](installation.md) for more installation options and\n  detailed installation instructions\n- [Setup](setup.md) for details on setup and usage\n- [Node Agent](node-agent.md) for setting up cri-resmgr-agent for dynamic\n  configuration and more\n- [Webhook](webhook.md) for setting up our resource-annotating webhook\n- [Support for Kata Containers\\*](setup.md#kata-containers) for setting up\n  CRI-RM with Kata Containers\n"
  },
  {
    "path": "docs/reference/agent-command-line-reference.md",
    "content": "# CRI-Resmgr-Agent Command-line Reference\n\n***WORK IN PROGRESS***\n"
  },
  {
    "path": "docs/reference/configuration-reference.md",
    "content": "# Configuration Reference\n\n## Configuration file\n\n***WORK IN PROGRESS***\n\n### `policy`\n\n**Active** specifies the active policy.\n```yaml\npolicy:\n  Active: static\n```\n\n**AvailableResources** specifies the available hardware resources.\n\n**ReservedResources** specifies the hardware resources reserved for system and\nkube tasks.\n\nCurrently, only CPU resources are supported. CPUs may be specified as a cpuset\nor as a numerical value, similar to Kubernetes resource quantities. Not all\npolicies use these configuration settings. See the policy-specific documentation\nfor details.\n\n```yaml\npolicy:\n  AvailableResources:\n    cpu: cpuset:0-63\n  ReservedResources:\n    cpu: cpuset:0-3\n    # Alternative ways to specify CPUs:\n    #cpu: 4\n    #cpu: 4000m\n```\n\n### `policy.static`\n\n**RelaxedIsolation** controls whether isolated CPUs are preferred for Guarenteed\nPods.\n\n```yaml\npolicy:\n  static:\n    RelaxedIsolation: true\n```\n\n### `policy.static-plus`\n\n### `policytopology-aware`\n\n### `policy.static-pools`\n\n### `policy.eda`\n\n### `control`\n\n### `control.blockio`\n\n### `control.rdt`\n\n### `blockio`\n\n### `rdt`\n\n### `instrumentation`\n\n### `rdt`\n\n### `blockio`\n\n### `log`\n\n### `dump`\n"
  },
  {
    "path": "docs/reference/index.rst",
    "content": "Reference\n#########\n.. toctree::\n   :maxdepth: 1\n\n   resmgr-command-line-reference.md\n   agent-command-line-reference.md\n   configuration-reference.md\n"
  },
  {
    "path": "docs/reference/resmgr-command-line-reference.md",
    "content": "# CRI-Resmgr Command-line Reference\n\n***WORK IN PROGRESS***\n"
  },
  {
    "path": "docs/releases/conf.py",
    "content": "# Configuration file for the Sphinx documentation builder.\n#\n# This file only contains a selection of the most common options. For a full\n# list see the documentation:\n# https://www.sphinx-doc.org/en/master/usage/configuration.html\n\n# -- Path setup --------------------------------------------------------------\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport os\n# sys.path.insert(0, os.path.abspath('.'))\n\n\n# -- Project information -----------------------------------------------------\n\nproject = 'CRI Resource Manager'\ncopyright = '2020, various'\nauthor = 'various'\n\n# Versions to show in the version menu\nversion = \"all releases\"\nif os.getenv('VERSIONS_MENU'):\n    html_context = {\n        'versions_menu': True,\n        'versions_menu_this_version': version}\n\n\n# -- General configuration ---------------------------------------------------\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n        'myst_parser',\n        'sphinx_markdown_tables'\n        ]\nsource_suffix = {\n        '.rst': 'restructuredtext',\n        '.md': 'markdown'\n        }\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['../_templates']\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This pattern also affects html_static_path and html_extra_path.\nexclude_patterns = []\n\n\n# -- Options for HTML output -------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages.  See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'sphinx_rtd_theme'\n\nhtml_theme_options = {\n    'display_version': True,\n}\n\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\n#html_static_path = ['_static']\n\n# Callbacks for recommonmark\ndef setup(app):\n    app.connect('missing-reference',ignoreMissingRefs)\n\ndef ignoreMissingRefs(app, env, node, contnode):\n    return contnode\n"
  },
  {
    "path": "docs/releases/index.md",
    "content": "# Releases\n\nFor up-to-date user documentation see the [documentation site](/cri-resource-manager)\n\n## Documentation for Released Versions\n<div id=\"releases\">\n</div>\n<script src=\"releases.js\"></script>\n<script>\n  var list = document.getElementById('releases').appendChild(document.createElement(\"ul\"));\n  var releaseItems = getReleaseListItems();\n  for (var i=0; i < releaseItems.length; i++) {\n    var item = document.createElement('li');\n    var paragraph = item.appendChild(document.createElement(\"p\"));\n    var anchor = paragraph.appendChild(document.createElement('a'));\n    anchor.appendChild(document.createTextNode(releaseItems[i].name));\n    anchor.href = releaseItems[i].url;\n    anchor.class = \"reference external\";\n    list.appendChild(item);\n  }\n</script>\n"
  },
  {
    "path": "docs/requirements.txt",
    "content": "sphinx==5.3.0\nsphinx_rtd_theme\nmyst-parser==0.18.1\nsphinx-markdown-tables\nPygments==2.15.1\n"
  },
  {
    "path": "docs/security.md",
    "content": "# Reporting a Potential Security Vulnerability\n\nPlease visit [intel.com/security](https://intel.com/security) to report\nsecurity issues.\n"
  },
  {
    "path": "docs/setup.md",
    "content": "# Setup and Usage\n\nIf you want to give CRI Resource Manager a try, here is the list of things\nyou need to do, assuming you already have a Kubernetes\\* cluster up and\nrunning, using either `containerd` or `cri-o` as the runtime.\n\n  0. [Install](installation.md) CRI Resource Manager.\n  1. Set up kubelet to use CRI Resource Manager as the runtime.\n  2. Set up CRI Resource Manager to use the runtime with a policy.\n\nFor kubelet you do this by altering its command line options like this:\n\n```\n   kubelet <other-kubelet-options> --container-runtime=remote \\\n     --container-runtime-endpoint=unix:///var/run/cri-resmgr/cri-resmgr.sock\n```\n\nFor CRI Resource Manager, you need to provide a configuration file, and also\na socket path if you don't use `containerd` or you run it with a different\nsocket path.\n\n```\n   # for containerd with default socket path\n   cri-resmgr --force-config <config-file> --runtime-socket unix:///var/run/containerd/containerd.sock\n   # for cri-o\n   cri-resmgr --force-config <config-file> --runtime-socket unix:///var/run/crio/crio.sock\n```\n\nThe choice of policy to use along with any potential parameters specific to\nthat policy are taken from the configuration file. You can take a look at the\n[sample configurations](/sample-configs) for some minimal/trivial examples.\nFor instance, you can use\n[sample-configs/topology-aware-policy.cfg](/sample-configs/topology-aware-policy.cfg)\nas `<config-file>` to activate the topology aware policy with memory\ntiering support.\n\n**NOTE**: Currently, the available policies are a work in progress.\n\n## Setting up kubelet to use CRI Resource Manager as the runtime\n\nTo let CRI Resource Manager act as a proxy between kubelet and the CRI\nruntime, you need to configure kubelet to connect to CRI Resource Manager\ninstead of the runtime. You do this by passing extra command line options to\nkubelet as shown below:\n\n```\n   kubelet <other-kubelet-options> --container-runtime=remote \\\n     --container-runtime-endpoint=unix:///var/run/cri-resmgr/cri-resmgr.sock\n```\n\n## Setting up CRI Resource Manager\n\nSetting up CRI Resource Manager involves pointing it to your runtime and\nproviding it with a configuration. Pointing to the runtime is done using\nthe `--runtime-socket <path>` and, optionally, the `--image-socket <path>`.\n\nFor providing a configuration there are two options:\n\n  1. use a local configuration YAML file\n  2. use the [CRI Resource Manager Node Agent][agent] and a `ConfigMap`\n\nThe former is easier to set up and it is also the preferred way to run CRI\nResource Manager for development, and in some cases testing. Setting up the\nlatter is a bit more involved but it allows you to\n\n  - manage policy configuration for your cluster as a single source, and\n  - dynamically update that configuration\n\n### Using a local configuration from a file\n\nThis is the easiest way to run CRI Resource Manager for development or\ntesting. You can do it with the following command:\n\n```\n   cri-resmgr --force-config <config-file> --runtime-socket <path>\n```\n\nWhen started this way, CRI Resource Manager reads its configuration from the\ngiven file. It does not fetch external configuration from the node agent and\nalso disables the config interface for receiving configuration updates.\n\n### Using CRI Resource Manager Agent and a ConfigMap\n\nThis setup requires an extra component, the\n[CRI Resource Manager Node Agent][agent],\nto monitor and fetch configuration from the ConfigMap and pass it on to CRI\nResource Manager. By default, CRI Resource Manager automatically tries to\nuse the agent to acquire configuration, unless you override this by forcing\na static local configuration using the `--force-config <config-file>` option.\nWhen using the agent, it is also possible to provide an initial fallback for\nconfiguration using the `--fallback-config <config-file>`. This file is\nused before the very first configuration is successfully acquired from the\nagent.\n\nWhenever a new configuration is acquired from the agent and successfully\ntaken into use, this configuration is stored in the cache and becomes\nthe default configuration to take into use the next time CRI Resource\nManager is restarted (unless that time the --force-config option is used).\nWhile CRI Resource Manager is shut down, any cached configuration can be\ncleared from the cache using the --reset-config command line option.\n\nSee the [Node Agent][agent] about how to set up and configure the agent.\n\n\n### Changing the active policy\n\nCurrently, CRI Resource Manager disables changing the active policy using\nthe [agent][agent]. That is, once the active policy is recorded in the cache,\nany configuration received through the agent that requests a different policy\nis rejected. This limitation will be removed in a future version of\nCRI Resource Manager.\n\nHowever, by default CRI Resource Manager allows you to change policies during\nits startup phase. If you want to disable this, you can pass the command line\noption `--disable-policy-switch` to CRI Resource Manager.\n\nIf you run CRI Resource Manager with disabled policy switching, you can still\nswitch policies by clearing any policy-specific data stored in the cache while\nCRI Resource Manager is shut down. You can do this by using the command line\noption `--reset-policy`. The whole sequence of switching policies this way is\n\n  - stop cri-resmgr (`systemctl stop cri-resource-manager`)\n  - reset policy data (`cri-resmgr --reset-policy`)\n  - change policy (`$EDITOR /etc/cri-resource-manager/fallback.cfg`)\n  - start cri-resmgr (`systemctl start cri-resource-manager`)\n\n\n### Container adjustments\n\nWhen the [agent][agent] is in use, it is also possible to `adjust` container\n`resource assignments` externally, using dedicated `Adjustment`\n`Custom Resources` in the `adjustments.criresmgr.intel.com` group. You can\nuse the [provided schema](/pkg/apis/resmgr/v1alpha1/adjustment-schema.yaml)\nto define the `Adjustment` resource. Then you can copy and modify the\n[sample adjustment CR](/sample-configs/external-adjustment.yaml) as a\nstarting point to test some overrides.\n\nAn `Adjustment` consists of the following:\n- `scope`:\n  - the nodes and containers to which the adjustment applies\n- adjustment data:\n  - updated native/compute resources (`cpu`/`memory` `requests` and `limits`)\n  - updated `RDT` and/or `Block I/O` class\n  - updated top tier (practically now DRAM) memory limit\n\nAll adjustment data is optional. An adjustment can choose to set any or all\nof them as necessary. The current handling of adjustment update updates the\nresource assignments of containers, marks all existing containers as having\npending changes in all controller domains, and then triggers a rebalancing in\nthe active policy. This causes all containers to be updated.\n\nThe scope defines to which containers on what nodes the adjustment applies.\nNodes are currently matched/picked by name, but a trailing wildcard (`*`) is\nallowed and matches all nodes with the given prefix in their names.\n\nContainers are matched by expressions. These are exactly the same as the\nexpressions for defining [affinity scopes](policy/container-affinity.md). A\nsingle adjustment can specify multiple node/container match pairs. An\nadjustment applies to all containers in its scope. If an adjustment/update\nresults in conflicts for some container, that is at least one container is\nin the scope of multiple adjustments, the adjustment is rejected and the\nwhole update is ignored.\n\n#### Commands for declaring, creating, deleting, and examining adjustments\n\nYou can declare the custom resource for adjustments with this command:\n\n```\nkubectl apply -f pkg/apis/resmgr/v1alpha1/adjustment-schema.yaml\n```\n\nYou can then add adjustments with a command like this:\n\n```\nkubectl apply -f sample-configs/external-adjustment.yaml\n```\n\nYou can list existing adjustments with the following command. Use the correct\n`-n namespace` option according to the namespace you use for the agent, for\nthe configuration, and in your adjustment specifications.\n\n```\nkubectl get adjustments.criresmgr.intel.com -n kube-system\n```\n\nYou can examine the contents of a single adjustment with these commands:\n\n```\nkubectl describe adjustments external-adjustment -n kube-system\nkubectl get adjustments.criresmgr.intel.com/<adjustment-name> -n kube-system -oyaml\n```\n\nOr you can examine the contents of all adjustments using this command:\n\n```\nkubectl get adjustments.criresmgr.intel.com -n kube-system -oyaml\n```\n\nFinally, you can delete an adjustment with commands like these:\n\n```\nkubectl delete -f sample-configs/external-adjustment.yaml\nkubectl delete adjustments.criresmgr.intel.com/<adjustment-name> -n kube-system\n```\n\nThe status of adjustment updates is propagated back to the `Adjustment`\n`Custom Resources`, more specifically into their `Status` fields. With the\nhelp of `jq`, you can easily examine the status of external adjustments\nusing a command like this:\n\n```\nkli@r640-1:~> kubectl get -n kube-system adjustments.criresmgr.intel.com -ojson | jq '.items[].status'\n{\n  \"nodes\": {\n    \"r640-1\": {\n      \"errors\": {}\n    }\n  }\n}\n{\n  \"nodes\": {\n    \"r640-1\": {\n      \"errors\": {}\n    }\n  }\n}\n```\n\nThe above response is what you get for adjustments applied without conflicts\nor errors. You can see here that only node *r640-1* is in the scope of both\nof your existing adjustments and those applied without errors.\n\nIf your adjustments resulted in errors, the output will look something like\nthis:\n\n```\nklitkey1@r640-1:~> kubectl get -n kube-system adjustments.criresmgr.intel.com -ojson | jq '.items[].status'\n{\n  \"nodes\": {\n    \"r640-1\": {\n      \"errors\": {\n        \"b71a93523e58cb4ba0310aa225b2e2a329cef895ca4b96fcd9d12b375337ea35\": \"cache: conflicting adjustments for my-pod-r640-1:my-container: adjustment-1,adjustment-2\"\n      }\n    }\n  }\n}\n{\n  \"nodes\": {\n    \"r640-1\": {\n      \"errors\": {\n        \"b71a93523e58cb4ba0310aa225b2e2a329cef895ca4b96fcd9d12b375337ea35\": \"cache: conflicting adjustments for my-pod-r640-1:my-container: adjustment-1,adjustment-2\"\n      }\n    }\n  }\n}\n```\n\nIn the sample above, you can see that on node *r640-1* the container with\n`ID`*b71a93523e58cb4ba0310aa225b2e2a329cef895ca4b96fcd9d12b375337ea35*, or\n*my-container* of *my-pod-r640-1*, had a conflict. Moreover you can see that\nthe reason of the conflict is that the container is in the scope of both\n*adjustment-1* and *adjustment-2*.\n\nYou can now fix those adjustments to resolve/remove the conflict, then\nreapply the adjustments, and finally verify that the conflicts are gone.\n\n```\nkli@r640-1:~> $EDITOR adjustment-1.yaml adjustment-2.yaml\nkli@r640-1:~> kubectl apply -f adjustment-1.yaml && kubectl apply -f adjustment-1.yaml && sleep 2\nkli@r640-1:~> kubectl get -n kube-system adjustments.criresmgr.intel.com -ojson | jq '.items[].status'\n{\n  \"nodes\": {\n    \"r640-1\": {\n      \"errors\": {}\n    }\n  }\n}\n{\n  \"nodes\": {\n    \"r640-1\": {\n      \"errors\": {}\n    }\n  }\n}\n```\n\n\n## Using CRI Resource Manager as a message dumper\n\nYou can use CRI Resource Manager to simply inspect all proxied CRI requests\nand responses without applying any policy. Run CRI Resource Manager with the\nprovided [sample configuration](/sample-configs/cri-full-message-dump.cfg)\nfor doing this.\n\n\n## Kata Containers\n\n[Kata Containers](https://katacontainers.io/) is an open source container\nruntime, building lightweight virtual machines that seamlessly plug into the\ncontainers ecosystem.\n\nIn order to enable Kata Containers in a Kubernetes-CRI-RM stack, both\nKubernetes and the Container Runtime need to be aware of the new runtime\nenvironment:\n\n  * The Container Runtime can only be CRI-O or containerd, and needs to\n   have the runtimes enabled in their configuration files.\n  * Kubernetes must be made aware of the CRI-O/containerd runtimes via a\n   \"RuntimeClass\"\n   [resource](https://kubernetes.io/docs/concepts/containers/runtime-class/)\n\nAfter these prerequisites are satisfied, the configuration file for the\ntarget  Kata Container, must have the flag \"SandboxCgroupOnly\" set to true.\nAs of Kata 2.0 this is the only way Kata Containers can work with the\nKubernetes cgroup naming conventions.\n\n   ```toml\n   ...\n   # If enabled, the runtime will add all the kata processes inside one dedicated cgroup.\n   # The container cgroups in the host are not created, just one single cgroup per sandbox.\n   # The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.\n   # The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.\n   # The sandbox cgroup is constrained if there is no container type annotation.\n   # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType\n   sandbox_cgroup_only=true\n   ...\n   ```\n\n### Reference\n\nIf you have a pre-existing Kubernetes cluster, for an easy deployement\nfollow this [document](https://github.com/kata-containers/packaging/blob/master/kata-deploy/README.md#kubernetes-quick-start).\n\n\nStarting from scratch:\n\n   * [Kata installation guide](https://github.com/kata-containers/kata-containers/tree/2.0-dev/docs/install#manual-installation)\n   * [Kata Containers + CRI-O](https://github.com/kata-containers/documentation/blob/master/how-to/run-kata-with-k8s.md)\n   * [Kata Containers + containerd](https://github.com/kata-containers/documentation/blob/master/how-to/containerd-kata.md)\n   * [Kubernetes Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/)\n   * [Cgroup and Kata containers](https://github.com/kata-containers/kata-containers/blob/stable-2.0.0/docs/design/host-cgroups.md)\n\n\n## Running with Untested Runtimes\n\nCRI Resource Manager is tested with `containerd` and `CRI-O`. If any other runtime is\ndetected during startup, `cri-resmgr` will refuse to start. This default behavior can\nbe changed using the `--allow-untested-runtimes` command line option.\n\n## Logging and debugging\n\nYou can control logging with the klog command line options or by setting the\ncorresponding environment variables. You can get the name of the environment\nvariable for a command line option by prepending the `LOGGER_` prefix to the\ncapitalized option name without any leading dashes. For instance, setting the\nenvironment variable `LOGGER_SKIP_HEADERS=true` has the same effect as using\nthe `-skip_headers` command line option.\n\nAdditionally, the `LOGGER_DEBUG` environment variable controls debug logs.\nThese are globally disabled by default. You can turn on full debugging by\nsetting `LOGGER_DEBUG='*'`.\n\nWhen using environment variables, be careful which configuration you pass to\nCRI Resource Manager using a file or ConfigMap. The environment is treated\nas default configuration but a file or a ConfigMap has higher precedence.\nIf something is configured in both, the environment will only be in effect\nuntil the configuration is applied. However, in such a case if you later\npush an updated configuration to CRI Resource Manager with the overlapping\nsettings removed, the original ones from the environment will be in effect\nagain.\n\nFor debug logs, the settings from the configuration are applied in addition\nto any settings in the environment. That said, if you turn something on in\nthe environment but off in the configuration, it will be turned off\neventually.\n\n<!-- Links -->\n[agent]: node-agent.md\n"
  },
  {
    "path": "docs/webhook.md",
    "content": "# Webhook\n\nBy default, CRI Resource Manager does not see the original container\n*resource requirements* specified in the *Pod Spec*. It tries to calculate\nthese for `cpu` and `memory` *compute resource*s using the related parameters\npresent in the CRI container creation request. The resulting estimates are\nnormally accurate for `cpu`, and also for `memory` `limits`. However, it is\nnot possible to use these parameters to estimate `memory` `request`s or any\n*extended resource*s.\n\nIf you want to make sure that CRI Resource Manager uses the origin *Pod Spec*\n*resource requirement*s, you need to duplicate these as *annotations* on the\nPod. This is necessary if you plan using or writing a policy which needs\n*extended resource*s.\n\nThis process can be fully automated using the\n[CRI Resource Manager Annotating Webhook](/cmd/cri-resmgr-webhook). Once you\nbuilt the Docker\\* image for it using the\n[provided Dockerfile](/cmd/cri-resmgr-webhook/Dockerfile) and published it,\nyou can set up the webhook as follows:\n- Fill in the `IMAGE_PLACEHOLDER` in\n  [webhook-deployment.yaml](/cmd/cri-resmgr-webhook/webhook-deployment.yaml)\n  to match the image.\n- Create a `cri-resmgr-webhook-secret` that carries a key and a certificate\n  to `cri-resmgr-webhook`. You can create a key, a self-signed certificate\n  and the secret that holds them with the following commands:\n  ```bash\n  SVC=cri-resmgr-webhook NS=cri-resmgr\n  openssl req -x509 -newkey rsa:2048 -sha256 -days 365 -nodes \\\n    -keyout cmd/cri-resmgr-webhook/server-key.pem \\\n    -out cmd/cri-resmgr-webhook/server-crt.pem \\\n    -subj \"/CN=$SVC.$NS.svc\" \\\n    -addext \"subjectAltName=DNS:$SVC,DNS:$SVC.$NS,DNS:$SVC.$NS.svc\"\n  cat >cmd/cri-resmgr-webhook/webhook-secret.yaml <<EOF\n  apiVersion: v1\n  kind: Secret\n  metadata:\n    name: cri-resmgr-webhook-secret\n    namespace: $NS\n  data:\n    svc.crt: $(base64 -w0 < cmd/cri-resmgr-webhook/server-crt.pem)\n    svc.key: $(base64 -w0 < cmd/cri-resmgr-webhook/server-key.pem)\n  type: Opaque\n  EOF\n  kubectl create namespace $NS\n  kubectl create -f cmd/cri-resmgr-webhook/webhook-secret.yaml\n  ```\n- Fill in the `CA_BUNDLE_PLACEHOLDER` in\n  [mutating-webhook-config.yaml](/cmd/cri-resmgr-webhook/mutating-webhook-config.yaml).\n  If you created the key and the certificate with the commands above,\n  you can do this with the following command:\n  ```bash\n  sed -e \"s/CA_BUNDLE_PLACEHOLDER/$(base64 -w0 < cmd/cri-resmgr-webhook/server-crt.pem)/\" \\\n      -i cmd/cri-resmgr-webhook/mutating-webhook-config.yaml\n  ```\n- Finally set up the webhook with these commands:\n  ```bash\n  kubectl apply -f cmd/cri-resmgr-webhook/webhook-deployment.yaml\n  kubectl wait --for=condition=Available -n cri-resmgr deployments/cri-resmgr-webhook\n  kubectl apply -f cmd/cri-resmgr-webhook/mutating-webhook-config.yaml\n  ```\n"
  },
  {
    "path": "elf/avx512.c",
    "content": "#include <uapi/linux/bpf.h>\n#include <asm/page_types.h>\n#include <asm/fpu/types.h>\n\n#define SEC(NAME) __attribute__((section(NAME), used))\n\n#ifndef KERNEL_VERSION\n    #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))\n#endif\n\n#define BUF_SIZE_MAP_NS 256\n\ntypedef struct bpf_map_def {\n\tunsigned int type;\n\tunsigned int key_size;\n\tunsigned int value_size;\n\tunsigned int max_entries;\n\tunsigned int map_flags;\n\tunsigned int pinning;\n\tchar namespace[BUF_SIZE_MAP_NS];\n} bpf_map_def;\n\nstatic int (*bpf_probe_read)(void *dst, u64 size, const void *unsafe_ptr) =\n\t(void *)BPF_FUNC_probe_read;\n\nstatic u64 (*bpf_get_current_cgroup_id)(void) = (void *)\n\tBPF_FUNC_get_current_cgroup_id;\n\nstatic u64 (*bpf_ktime_get_ns)(void) = (void *)\n\tBPF_FUNC_ktime_get_ns;\n\nstatic int (*bpf_map_update_elem)(void *map, void *key, void *value,\n\t\t\t\t  u64 flags) = (void *)BPF_FUNC_map_update_elem;\n\nstatic void *(*bpf_map_lookup_elem)(void *map, void *key) = (void *)\n\tBPF_FUNC_map_lookup_elem;\n\nstruct bpf_map_def\n\tSEC(\"maps/all_context_switch_count\") all_context_switch_count_hash = {\n\t\t.type = BPF_MAP_TYPE_HASH,\n\t\t.key_size = sizeof(u64),\n\t\t.value_size = sizeof(u32),\n\t\t.max_entries = 1024,\n\t};\n\nstruct bpf_map_def\n\tSEC(\"maps/avx_context_switch_count\") avx_context_switch_count_hash = {\n\t\t.type = BPF_MAP_TYPE_PERCPU_HASH,\n\t\t.key_size = sizeof(u64),\n\t\t.value_size = sizeof(u32),\n\t\t.max_entries = 1024,\n\t};\n\nstruct bpf_map_def\n\tSEC(\"maps/avx_timestamp\") avx_timestamp_hash = {\n\t\t.type = BPF_MAP_TYPE_HASH,\n\t\t.key_size = sizeof(u64),\n\t\t.value_size = sizeof(u32),\n\t\t.max_entries = 1024,\n\t};\n\nstruct bpf_map_def\n\tSEC(\"maps/last_update_ns\") last_update_ns_hash = {\n\t\t.type = BPF_MAP_TYPE_HASH,\n\t\t.key_size = sizeof(u64),\n\t\t.value_size = sizeof(u64),\n\t\t.max_entries = 1024,\n\t};\n\nSEC(\"tracepoint/sched/sched_switch\")\nint tracepoint__sched_switch(void *args)\n{\n\tu64 cgroup_id = bpf_get_current_cgroup_id();\n\tu32 *count, *found;\n\tu32 new_count = 1;\n\n\tfound = bpf_map_lookup_elem(&avx_context_switch_count_hash, &cgroup_id);\n\n\t/* store sched_switch counts only for cgroups that have AVX activity */\n\tif (!found) {\n\t\treturn 0;\n\t}\n\n\tcount = bpf_map_lookup_elem(&all_context_switch_count_hash, &cgroup_id);\n\tif (count) {\n\t\t__sync_fetch_and_add(count, 1);\n\t} else {\n\t\tbpf_map_update_elem(&all_context_switch_count_hash, &cgroup_id,\n\t\t\t\t    &new_count, BPF_ANY);\n\t}\n\treturn 0;\n}\n\nstruct x86_fpu_args {\n\tu64 pad;\n\tstruct fpu *fpu;\n\tbool load_fpu;\n\tu64 xfeatures;\n\tu64 xcomp_bv;\n};\n\nSEC(\"tracepoint/x86_fpu/x86_fpu_regs_deactivated\")\nint tracepoint__x86_fpu_regs_deactivated(struct x86_fpu_args *args)\n{\n\tu32 *counter;\n\tu32 ts;\n\tbpf_probe_read(&ts, sizeof(u32), (void *)&args->fpu->avx512_timestamp);\n\n\tif (ts == 0) {\n\t\treturn 0;\n\t}\n\n\tu64 cgroup_id = bpf_get_current_cgroup_id();\n\n\tu32 ts_prev;\n\tu32 *tsp;\n\ttsp = bpf_map_lookup_elem(&avx_timestamp_hash, &cgroup_id);\n\n\tts_prev = tsp ? *tsp : 0;\n\n\tif (ts == ts_prev) {\n\t\treturn 0;\n\t}\n\tbpf_map_update_elem(&avx_timestamp_hash, &cgroup_id, &ts, BPF_ANY);\n\n\tu32 count = 1;\n\tcounter = bpf_map_lookup_elem(&avx_context_switch_count_hash, &cgroup_id);\n\tif (counter) {\n\t\t__sync_fetch_and_add(counter, 1);\n\t} else {\n\t\tbpf_map_update_elem(&avx_context_switch_count_hash, &cgroup_id,\n\t\t\t\t    &count, BPF_ANY);\n\t}\n\n\tu64 last = bpf_ktime_get_ns();\n\tbpf_map_update_elem(&last_update_ns_hash, &cgroup_id, &last, BPF_ANY);\n\n\treturn 0;\n}\n\nchar _license[] SEC(\"license\") = \"GPL\";\n\n/*\nNotes about Linux version:\n   * We don't check LINUX_VERSION_CODE build time. It's user's responsibility to provide new enough headers.\n   * Build failures may happen due to too old kernel headers (currently, Linux >= 5.1 headers are needed).\n   * Our dependency to Kernel ABI is x86_fpu tracepoint parameters and struct fpu.\n   * The host kernel needs to run Linux >= 5.2 and the version is checked upon eBPF loading.\n   * We build the minimum supported version in SEC(\"version\") section.\n   * Max supported version is not checked but the check may be added later.\n*/\nunsigned int _version SEC(\"version\") = KERNEL_VERSION(5, 2, 0);\n"
  },
  {
    "path": "go.mod",
    "content": "module github.com/intel/cri-resource-manager\n\ngo 1.24\n\nrequire (\n\tcontrib.go.opencensus.io/exporter/jaeger v0.2.1\n\tcontrib.go.opencensus.io/exporter/prometheus v0.4.2\n\tgithub.com/cilium/ebpf v0.12.3\n\tgithub.com/evanphx/json-patch v5.7.0+incompatible\n\tgithub.com/google/go-cmp v0.6.0\n\tgithub.com/intel/cri-resource-manager/pkg/topology v0.0.0\n\tgithub.com/intel/goresctrl v0.5.0\n\tgithub.com/pkg/errors v0.9.1\n\tgithub.com/prometheus/client_golang v1.18.0\n\tgithub.com/prometheus/client_model v0.5.0\n\tgithub.com/prometheus/common v0.45.0\n\tgithub.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92\n\tgithub.com/stretchr/testify v1.8.4\n\tgo.opencensus.io v0.24.0\n\tgolang.org/x/sys v0.31.0\n\tgolang.org/x/time v0.5.0\n\tgoogle.golang.org/grpc v1.60.1\n\tgoogle.golang.org/protobuf v1.33.0\n\tk8s.io/api v0.29.0\n\tk8s.io/apimachinery v0.29.0\n\tk8s.io/client-go v0.29.0\n\tk8s.io/cri-api v0.29.0\n\tk8s.io/klog/v2 v2.110.1\n\tk8s.io/utils v0.0.0-20240102154912-e7106e64919e\n\tsigs.k8s.io/yaml v1.4.0\n)\n\nrequire (\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.2.0 // indirect\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/emicklei/go-restful/v3 v3.11.0 // indirect\n\tgithub.com/go-kit/log v0.2.1 // indirect\n\tgithub.com/go-logfmt/logfmt v0.6.0 // indirect\n\tgithub.com/go-logr/logr v1.3.0 // indirect\n\tgithub.com/go-openapi/jsonpointer v0.19.6 // indirect\n\tgithub.com/go-openapi/jsonreference v0.20.2 // indirect\n\tgithub.com/go-openapi/swag v0.22.3 // indirect\n\tgithub.com/gogo/protobuf v1.3.2 // indirect\n\tgithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect\n\tgithub.com/golang/protobuf v1.5.3 // indirect\n\tgithub.com/google/gnostic-models v0.6.8 // indirect\n\tgithub.com/google/gofuzz v1.2.0 // indirect\n\tgithub.com/google/uuid v1.5.0 // indirect\n\tgithub.com/imdario/mergo v0.3.12 // indirect\n\tgithub.com/josharian/intern v1.0.0 // indirect\n\tgithub.com/json-iterator/go v1.1.12 // indirect\n\tgithub.com/mailru/easyjson v0.7.7 // indirect\n\tgithub.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.2 // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgithub.com/prometheus/procfs v0.12.0 // indirect\n\tgithub.com/prometheus/statsd_exporter v0.26.0 // indirect\n\tgithub.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect\n\tgithub.com/spf13/pflag v1.0.5 // indirect\n\tgithub.com/uber/jaeger-client-go v2.25.0+incompatible // indirect\n\tgolang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect\n\tgolang.org/x/net v0.38.0 // indirect\n\tgolang.org/x/oauth2 v0.27.0 // indirect\n\tgolang.org/x/sync v0.12.0 // indirect\n\tgolang.org/x/term v0.30.0 // indirect\n\tgolang.org/x/text v0.23.0 // indirect\n\tgoogle.golang.org/api v0.155.0 // indirect\n\tgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect\n\tgopkg.in/inf.v0 v0.9.1 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tk8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect\n\tsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect\n\tsigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect\n)\n\nreplace (\n\tgithub.com/intel/cri-resource-manager/pkg/topology v0.0.0 => ./pkg/topology\n\n\tgo.opentelemetry.io/contrib => go.opentelemetry.io/contrib v0.20.0\n\tgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0\n\tgo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0\n\tgo.opentelemetry.io/otel => go.opentelemetry.io/otel v0.20.0\n\tgo.opentelemetry.io/otel/exporters/otlp => go.opentelemetry.io/otel/exporters/otlp v0.20.0\n\tgo.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.20.0\n\tgo.opentelemetry.io/otel/oteltest => go.opentelemetry.io/otel/oteltest v0.20.0\n\tgo.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v0.20.0\n\tgo.opentelemetry.io/otel/sdk/export/metric => go.opentelemetry.io/otel/sdk/export/metric v0.20.0\n\tgo.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.20.0\n\tgo.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v0.20.0\n\n\tk8s.io/api => k8s.io/api v0.29.0\n\tk8s.io/apimachinery => k8s.io/apimachinery v0.29.0\n\tk8s.io/apiserver => k8s.io/apiserver v0.29.0\n\tk8s.io/client-go => k8s.io/client-go v0.29.0\n\tk8s.io/component-base => k8s.io/component-base v0.29.0\n\tk8s.io/cri-api => k8s.io/cri-api v0.29.0\n)\n"
  },
  {
    "path": "go.sum",
    "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=\ncloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=\ncloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=\ncloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=\ncloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=\ncloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=\ncloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=\ncloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=\ncloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=\ncloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=\ncloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=\ncloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=\ncloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=\ncloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=\ncloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=\ncloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=\ncloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=\ncloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=\ncloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=\ncloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=\ncloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=\ncloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=\ncloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=\ncloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=\ncloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=\ncloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=\ncloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=\ncloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=\ncloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=\ncloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=\ncontrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI=\ncontrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0=\ncontrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg=\ncontrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ=\ndmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=\ngithub.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=\ngithub.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=\ngithub.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4=\ngithub.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=\ngithub.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=\ngithub.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\ngithub.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=\ngithub.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=\ngithub.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=\ngithub.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=\ngithub.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=\ngithub.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=\ngithub.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=\ngithub.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=\ngithub.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=\ngithub.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=\ngithub.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=\ngithub.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=\ngithub.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=\ngithub.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=\ngithub.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=\ngithub.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=\ngithub.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=\ngithub.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=\ngithub.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=\ngithub.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=\ngithub.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=\ngithub.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=\ngithub.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=\ngithub.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=\ngithub.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=\ngithub.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=\ngithub.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=\ngithub.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=\ngithub.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=\ngithub.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=\ngithub.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=\ngithub.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=\ngithub.com/intel/goresctrl v0.5.0 h1:kcDhjE3ZF/mNrJuRzLS3LY2Hp6atFaF1XVFBT7SVL2g=\ngithub.com/intel/goresctrl v0.5.0/go.mod h1:mIe63ggylWYr0cU/l8n11FAkesqfvuP3oktIsxvu0T0=\ngithub.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=\ngithub.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\ngithub.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=\ngithub.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=\ngithub.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=\ngithub.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=\ngithub.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=\ngithub.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=\ngithub.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=\ngithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=\ngithub.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=\ngithub.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=\ngithub.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=\ngithub.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=\ngithub.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=\ngithub.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=\ngithub.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=\ngithub.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=\ngithub.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=\ngithub.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=\ngithub.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=\ngithub.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=\ngithub.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=\ngithub.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=\ngithub.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=\ngithub.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=\ngithub.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=\ngithub.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=\ngithub.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=\ngithub.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=\ngithub.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=\ngithub.com/prometheus/statsd_exporter v0.26.0 h1:SQl3M6suC6NWQYEzOvIv+EF6dAMYEqIuZy+o4H9F5Ig=\ngithub.com/prometheus/statsd_exporter v0.26.0/go.mod h1:GXFLADOmBTVDrHc7b04nX8ooq3azG61pnECNqT7O5DM=\ngithub.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=\ngithub.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=\ngithub.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=\ngithub.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=\ngithub.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M=\ngithub.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 h1:OfRzdxCzDhp+rsKWXuOO2I/quKMJ/+TQwVbIP/gltZg=\ngithub.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6S6I7v6WXb+IjhMuZEYfH/RJ5RwEWnEo5BMg=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=\ngithub.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=\ngithub.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngithub.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=\ngithub.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=\ngithub.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc=\ngithub.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=\ngithub.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=\ngithub.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngo.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=\ngo.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=\ngo.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=\ngo.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=\ngo.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=\ngolang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=\ngolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=\ngolang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=\ngolang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=\ngolang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=\ngolang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=\ngolang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=\ngolang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=\ngolang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=\ngolang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=\ngolang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=\ngolang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=\ngolang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=\ngolang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=\ngolang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=\ngolang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=\ngolang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=\ngolang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=\ngolang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=\ngolang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=\ngolang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=\ngolang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=\ngolang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=\ngolang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=\ngolang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=\ngolang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=\ngoogle.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=\ngoogle.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=\ngoogle.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=\ngoogle.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA=\ngoogle.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=\ngoogle.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=\ngoogle.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=\ngoogle.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=\ngoogle.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=\ngoogle.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=\ngoogle.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=\ngoogle.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=\ngoogle.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=\ngoogle.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=\ngoogle.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=\ngoogle.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=\ngoogle.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\ngopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=\ngopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=\nhonnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nhonnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nk8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=\nk8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=\nk8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=\nk8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=\nk8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=\nk8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=\nk8s.io/cri-api v0.29.0 h1:atenAqOltRsFqcCQlFFpDnl/R4aGfOELoNLTDJfd7t8=\nk8s.io/cri-api v0.29.0/go.mod h1:Rls2JoVwfC7kW3tndm7267kriuRukQ02qfht0PCRuIc=\nk8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=\nk8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=\nk8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=\nk8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=\nk8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=\nk8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=\nrsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=\nrsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=\nrsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=\nsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=\nsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=\nsigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=\nsigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=\nsigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=\nsigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=\n"
  },
  {
    "path": "packaging/deb.in/changelog",
    "content": "__PACKAGE__ (__VERSION__) unstable; urgency=low\n\n  * Release build of __PACKAGE__ __VERSION__ for debian/ubuntu.\n\n -- __AUTHOR__ <__EMAIL__>  __DATE__\n"
  },
  {
    "path": "packaging/deb.in/compat",
    "content": "11\n"
  },
  {
    "path": "packaging/deb.in/control",
    "content": "Source: __PACKAGE__\nMaintainer: __AUTHOR__ <__EMAIL__>\n\nPackage: __PACKAGE__\nArchitecture: any\nDescription: A CRI Proxy for Hardware Resource Management"
  },
  {
    "path": "packaging/deb.in/rules",
    "content": "#!/usr/bin/make -f\n#-*- make -*-\n\nDISTRIBUTION = $(shell sed -n \"s/^VERSION_CODENAME=//p\" /etc/os-release)\nVERSION = __VERSION__\nPACKAGEVERSION = $(VERSION)\nTARBALL = __TARBALL__\nURL = http://github.com/intel/cri-resource-manager\n\n%:\n\tdh $@\n\noverride_dh_auto_clean:\noverride_dh_auto_test:\noverride_dh_auto_build:\noverride_dh_auto_install:\n\texport PATH=\"$$PATH:$$(go env GOPATH)/bin\"; \\\n\tmake BUILD_DIRS=cri-resmgr install DESTDIR=debian/__PACKAGE__\n\tmake BUILD_DIRS=cri-resmgr install-licenses DESTDIR=debian/__PACKAGE__/usr/share/doc/__PACKAGE__\n\tcp README.md docs/*.md cmd/*/*.sample \\\n\t    debian/__PACKAGE__/usr/share/doc/__PACKAGE__\n\noverride_dh_gencontrol:\n\tdh_gencontrol -- -v$(PACKAGEVERSION)\n"
  },
  {
    "path": "packaging/rpm/cri-resource-manager.spec.in",
    "content": "Name:    cri-resource-manager\nVersion: __VERSION__\nRelease: 0\nSummary: CRI Resource Manager, a CRI proxy with various in-node workload placement policies\nLicense: ASL 2.0 \nURL:     https://github.com/intel/cri-resource-manager\nSource0: https://github.com/intel/cri-resource-manager/archive/cri-resource-manager-__TARVERSION__.tar.gz\nBuildRequires: coreutils, make, kernel-devel\n\n# Disable the building of debug package(s).\n%define debug_package %{nil}\n\n%description\nKubernetes Container Runtime Interface proxy service with hardware resource aware workload\nplacement policies.\n\n%prep\n%setup -q -n cri-resource-manager-__TARVERSION__\n\n%build\nmake build BUILD_DIRS=cri-resmgr\nmake install-licenses BUILD_DIRS=cri-resmgr DESTDIR=.\n\n%install\n%make_install UNITDIR=%{_unitdir} SYSCONFDIR=%{_sysconfdir} BUILD_DIRS=cri-resmgr\ninstall -m 0700 -d %{?buildroot}%{_sharedstatedir}/cri-resmgr\n\n%files\n%defattr(-,root,root,-)\n%{_bindir}/*\n%{_sysconfdir}/sysconfig/*\n%{_unitdir}/*\n%dir %attr(0700,root,root) %{_sharedstatedir}/cri-resmgr\n%dir %attr(0700,root,root) %{_sysconfdir}/cri-resmgr\n%config(noreplace) %{_sysconfdir}/cri-resmgr/*\n%license licenses/cri-resmgr/*\n%doc README.md docs/*.md\n%doc cmd/*/*.sample\n"
  },
  {
    "path": "pkg/agent/agent.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n\tk8sclient \"k8s.io/client-go/kubernetes\"\n\n\tresmgrcs \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1\"\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n)\n\n// Get cri-resmgr config\ntype configInterface interface {\n\tgetConfig() resmgrConfig\n\tgetError() error\n}\n\n// resmgrConfig represents cri-resmgr configuration\ntype resmgrConfig map[string]string\n\n// resmgrAdjustment represents external adjustments for the resource-manager\ntype resmgrAdjustment map[string]*resmgr.Adjustment\n\n// resmgrStatus represents the status of an external adjustment update\ntype resmgrStatus struct {\n\trequest error\n\terrors  map[string]string\n}\n\n// ResourceManagerAgent is the interface exposed for the CRI Resource Manager Congig Agent\ntype ResourceManagerAgent interface {\n\tRun() error\n}\n\n// agent implements ResourceManagerAgent\ntype agent struct {\n\tlog.Logger                      // Our logging interface\n\tcli        *k8sclient.Clientset // K8s client\n\textCli     *resmgrcs.CriresmgrV1alpha1Client\n\tserver     agentServer   // gRPC server listening for requests from cri-resource-manager\n\twatcher    k8sWatcher    // Watcher monitoring events in K8s cluster\n\tupdater    configUpdater // Client sending config updates to cri-resource-manager\n}\n\n// NewResourceManagerAgent creates a new instance of ResourceManagerAgent\nfunc NewResourceManagerAgent() (ResourceManagerAgent, error) {\n\tvar err error\n\n\ta := &agent{\n\t\tLogger: log.NewLogger(\"resource-manager-agent\"),\n\t}\n\n\tif a.cli, a.extCli, err = a.getK8sClient(opts.kubeconfig); err != nil {\n\t\treturn nil, agentError(\"failed to get k8s client: %v\", err)\n\t}\n\n\tif a.watcher, err = newK8sWatcher(a.cli, a.extCli); err != nil {\n\t\treturn nil, agentError(\"failed to initialize watcher instance: %v\", err)\n\t}\n\n\tif a.server, err = newAgentServer(a.cli, a); err != nil {\n\t\treturn nil, agentError(\"failed to initialize gRPC server\")\n\t}\n\n\tif a.updater, err = newConfigUpdater(); err != nil {\n\t\treturn nil, agentError(\"failed to initialize config updater instance: %v\", err)\n\t}\n\n\treturn a, nil\n}\n\n// Start starts the resource manager.\nfunc (a *agent) Run() error {\n\ta.Info(\"starting CRI Resource Manager Agent\")\n\n\tif err := a.server.Start(opts.agentSocket); err != nil {\n\t\treturn agentError(\"failed to start gRPC server: %v\", err)\n\t}\n\n\tif err := a.watcher.Start(); err != nil {\n\t\treturn agentError(\"failed to start watcher: %v\", err)\n\t}\n\n\tif err := a.updater.Start(); err != nil {\n\t\treturn agentError(\"failed to start config updater: %v\", err)\n\t}\n\n\tfor {\n\t\tselect {\n\t\tcase config, ok := <-a.watcher.ConfigChan():\n\t\t\tif ok {\n\t\t\t\ta.updater.UpdateConfig(&config)\n\t\t\t}\n\t\tcase adjust, ok := <-a.watcher.AdjustmentChan():\n\t\t\tif ok {\n\t\t\t\ta.updater.UpdateAdjustment(&adjust)\n\t\t\t}\n\t\tcase status, ok := <-a.updater.StatusChan():\n\t\t\tif ok {\n\t\t\t\ta.Info(\"got status %v\", status)\n\t\t\t\tif err := a.watcher.UpdateStatus(status); err != nil {\n\t\t\t\t\ta.Error(\"failed to update adjustment node status: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (a *agent) getConfig() resmgrConfig {\n\tif a.watcher == nil {\n\t\treturn nil\n\t}\n\treturn a.watcher.GetConfig()\n}\n\nfunc (a *agent) getError() error {\n\tif a.updater == nil {\n\t\treturn nil\n\t}\n\treturn a.updater.GetError()\n}\n\nfunc agentError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(format, args...)\n}\n"
  },
  {
    "path": "pkg/agent/api/v1/api.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage v1\n\nimport (\n\t\"encoding/json\"\n)\n\nvar _ json.Marshaler = &JsonPatch{}\n\n// MarshalJSON marshals JsonPatch to valid Json\nfunc (j *JsonPatch) MarshalJSON() ([]byte, error) {\n\t// Don't really encode anything. Op and Path are ascii strings and value\n\t// is assumed to be in marshaled format\n\tif len(j.Value) == 0 {\n\t\treturn []byte(`{\"op\":\"` + j.Op + `\",\"path\":\"` + j.Path + `\"}`), nil\n\t}\n\treturn []byte(`{\"op\":\"` + j.Op + `\",\"path\":\"` + j.Path + `\",\"value\":` + j.Value + `}`), nil\n}\n"
  },
  {
    "path": "pkg/agent/api/v1/api.pb.go",
    "content": "//\n//Copyright 2019 Intel Corporation\n//\n//Licensed under the Apache License, Version 2.0 (the \"License\");\n//you may not use this file except in compliance with the License.\n//You may obtain a copy of the License at\n//\n//http://www.apache.org/licenses/LICENSE-2.0\n//\n//Unless required by applicable law or agreed to in writing, software\n//distributed under the License is distributed on an \"AS IS\" BASIS,\n//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n//See the License for the specific language governing permissions and\n//limitations under the License.\n\n// Code generated by protoc-gen-go. DO NOT EDIT.\n// versions:\n// \tprotoc-gen-go v1.28.0\n// \tprotoc        v3.20.1\n// source: pkg/agent/api/v1/api.proto\n\npackage v1\n\nimport (\n\tprotoreflect \"google.golang.org/protobuf/reflect/protoreflect\"\n\tprotoimpl \"google.golang.org/protobuf/runtime/protoimpl\"\n\treflect \"reflect\"\n\tsync \"sync\"\n)\n\nconst (\n\t// Verify that this generated code is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)\n\t// Verify that runtime/protoimpl is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)\n)\n\ntype GetNodeRequest struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n}\n\nfunc (x *GetNodeRequest) Reset() {\n\t*x = GetNodeRequest{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[0]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *GetNodeRequest) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*GetNodeRequest) ProtoMessage() {}\n\nfunc (x *GetNodeRequest) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[0]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use GetNodeRequest.ProtoReflect.Descriptor instead.\nfunc (*GetNodeRequest) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{0}\n}\n\ntype GetNodeReply struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tNode string `protobuf:\"bytes,1,opt,name=node,proto3\" json:\"node,omitempty\"`\n}\n\nfunc (x *GetNodeReply) Reset() {\n\t*x = GetNodeReply{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[1]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *GetNodeReply) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*GetNodeReply) ProtoMessage() {}\n\nfunc (x *GetNodeReply) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[1]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use GetNodeReply.ProtoReflect.Descriptor instead.\nfunc (*GetNodeReply) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{1}\n}\n\nfunc (x *GetNodeReply) GetNode() string {\n\tif x != nil {\n\t\treturn x.Node\n\t}\n\treturn \"\"\n}\n\n// JsonPatch holds on JSON patch\ntype JsonPatch struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tOp    string `protobuf:\"bytes,1,opt,name=op,proto3\" json:\"op,omitempty\"`\n\tPath  string `protobuf:\"bytes,2,opt,name=path,proto3\" json:\"path,omitempty\"`\n\tValue string `protobuf:\"bytes,3,opt,name=value,proto3\" json:\"value,omitempty\"`\n}\n\nfunc (x *JsonPatch) Reset() {\n\t*x = JsonPatch{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[2]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *JsonPatch) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*JsonPatch) ProtoMessage() {}\n\nfunc (x *JsonPatch) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[2]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use JsonPatch.ProtoReflect.Descriptor instead.\nfunc (*JsonPatch) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{2}\n}\n\nfunc (x *JsonPatch) GetOp() string {\n\tif x != nil {\n\t\treturn x.Op\n\t}\n\treturn \"\"\n}\n\nfunc (x *JsonPatch) GetPath() string {\n\tif x != nil {\n\t\treturn x.Path\n\t}\n\treturn \"\"\n}\n\nfunc (x *JsonPatch) GetValue() string {\n\tif x != nil {\n\t\treturn x.Value\n\t}\n\treturn \"\"\n}\n\ntype PatchNodeRequest struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\t// List of JSON patches to apply on the node\n\tPatches []*JsonPatch `protobuf:\"bytes,1,rep,name=patches,proto3\" json:\"patches,omitempty\"`\n}\n\nfunc (x *PatchNodeRequest) Reset() {\n\t*x = PatchNodeRequest{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[3]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *PatchNodeRequest) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*PatchNodeRequest) ProtoMessage() {}\n\nfunc (x *PatchNodeRequest) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[3]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use PatchNodeRequest.ProtoReflect.Descriptor instead.\nfunc (*PatchNodeRequest) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{3}\n}\n\nfunc (x *PatchNodeRequest) GetPatches() []*JsonPatch {\n\tif x != nil {\n\t\treturn x.Patches\n\t}\n\treturn nil\n}\n\ntype PatchNodeReply struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n}\n\nfunc (x *PatchNodeReply) Reset() {\n\t*x = PatchNodeReply{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[4]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *PatchNodeReply) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*PatchNodeReply) ProtoMessage() {}\n\nfunc (x *PatchNodeReply) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[4]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use PatchNodeReply.ProtoReflect.Descriptor instead.\nfunc (*PatchNodeReply) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{4}\n}\n\ntype UpdateNodeCapacityRequest struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\t// Name-value map of status.capacity to update\n\tCapacities map[string]string `protobuf:\"bytes,1,rep,name=capacities,proto3\" json:\"capacities,omitempty\" protobuf_key:\"bytes,1,opt,name=key,proto3\" protobuf_val:\"bytes,2,opt,name=value,proto3\"`\n}\n\nfunc (x *UpdateNodeCapacityRequest) Reset() {\n\t*x = UpdateNodeCapacityRequest{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[5]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *UpdateNodeCapacityRequest) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*UpdateNodeCapacityRequest) ProtoMessage() {}\n\nfunc (x *UpdateNodeCapacityRequest) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[5]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use UpdateNodeCapacityRequest.ProtoReflect.Descriptor instead.\nfunc (*UpdateNodeCapacityRequest) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{5}\n}\n\nfunc (x *UpdateNodeCapacityRequest) GetCapacities() map[string]string {\n\tif x != nil {\n\t\treturn x.Capacities\n\t}\n\treturn nil\n}\n\ntype UpdateNodeCapacityReply struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n}\n\nfunc (x *UpdateNodeCapacityReply) Reset() {\n\t*x = UpdateNodeCapacityReply{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[6]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *UpdateNodeCapacityReply) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*UpdateNodeCapacityReply) ProtoMessage() {}\n\nfunc (x *UpdateNodeCapacityReply) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[6]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use UpdateNodeCapacityReply.ProtoReflect.Descriptor instead.\nfunc (*UpdateNodeCapacityReply) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{6}\n}\n\ntype HealthCheckRequest struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tQuery string `protobuf:\"bytes,1,opt,name=query,proto3\" json:\"query,omitempty\"`\n}\n\nfunc (x *HealthCheckRequest) Reset() {\n\t*x = HealthCheckRequest{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[7]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *HealthCheckRequest) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*HealthCheckRequest) ProtoMessage() {}\n\nfunc (x *HealthCheckRequest) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[7]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead.\nfunc (*HealthCheckRequest) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{7}\n}\n\nfunc (x *HealthCheckRequest) GetQuery() string {\n\tif x != nil {\n\t\treturn x.Query\n\t}\n\treturn \"\"\n}\n\ntype HealthCheckReply struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\tError string `protobuf:\"bytes,1,opt,name=error,proto3\" json:\"error,omitempty\"`\n}\n\nfunc (x *HealthCheckReply) Reset() {\n\t*x = HealthCheckReply{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[8]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *HealthCheckReply) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*HealthCheckReply) ProtoMessage() {}\n\nfunc (x *HealthCheckReply) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_agent_api_v1_api_proto_msgTypes[8]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use HealthCheckReply.ProtoReflect.Descriptor instead.\nfunc (*HealthCheckReply) Descriptor() ([]byte, []int) {\n\treturn file_pkg_agent_api_v1_api_proto_rawDescGZIP(), []int{8}\n}\n\nfunc (x *HealthCheckReply) GetError() string {\n\tif x != nil {\n\t\treturn x.Error\n\t}\n\treturn \"\"\n}\n\nvar File_pkg_agent_api_v1_api_proto protoreflect.FileDescriptor\n\nvar file_pkg_agent_api_v1_api_proto_rawDesc = []byte{\n\t0x0a, 0x1a, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f,\n\t0x76, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x31,\n\t0x22, 0x10, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,\n\t0x73, 0x74, 0x22, 0x22, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x70,\n\t0x6c, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,\n\t0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, 0x45, 0x0a, 0x09, 0x4a, 0x73, 0x6f, 0x6e, 0x50, 0x61,\n\t0x74, 0x63, 0x68, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,\n\t0x02, 0x6f, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28,\n\t0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,\n\t0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x3b, 0x0a,\n\t0x10, 0x50, 0x61, 0x74, 0x63, 0x68, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,\n\t0x74, 0x12, 0x27, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03,\n\t0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x63,\n\t0x68, 0x52, 0x07, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x22, 0x10, 0x0a, 0x0e, 0x50, 0x61,\n\t0x74, 0x63, 0x68, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0xa9, 0x01, 0x0a,\n\t0x19, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x61, 0x70, 0x61, 0x63,\n\t0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4d, 0x0a, 0x0a, 0x63, 0x61,\n\t0x70, 0x61, 0x63, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d,\n\t0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x61,\n\t0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x61,\n\t0x70, 0x61, 0x63, 0x69, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x63,\n\t0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x43, 0x61, 0x70,\n\t0x61, 0x63, 0x69, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,\n\t0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,\n\t0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,\n\t0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x19, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61,\n\t0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x52, 0x65,\n\t0x70, 0x6c, 0x79, 0x22, 0x2a, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65,\n\t0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65,\n\t0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22,\n\t0x28, 0x0a, 0x10, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65,\n\t0x70, 0x6c, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,\n\t0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x86, 0x02, 0x0a, 0x05, 0x41, 0x67,\n\t0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12,\n\t0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,\n\t0x73, 0x74, 0x1a, 0x10, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x52,\n\t0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x09, 0x50, 0x61, 0x74, 0x63, 0x68, 0x4e,\n\t0x6f, 0x64, 0x65, 0x12, 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x63, 0x68, 0x4e, 0x6f,\n\t0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x50,\n\t0x61, 0x74, 0x63, 0x68, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12,\n\t0x52, 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x61, 0x70,\n\t0x61, 0x63, 0x69, 0x74, 0x79, 0x12, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74,\n\t0x65, 0x4e, 0x6f, 0x64, 0x65, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71,\n\t0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,\n\t0x4e, 0x6f, 0x64, 0x65, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x52, 0x65, 0x70, 0x6c,\n\t0x79, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65,\n\t0x63, 0x6b, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68,\n\t0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x76, 0x31, 0x2e,\n\t0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79,\n\t0x22, 0x00, 0x42, 0x07, 0x5a, 0x05, 0x2e, 0x2e, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f,\n\t0x74, 0x6f, 0x33,\n}\n\nvar (\n\tfile_pkg_agent_api_v1_api_proto_rawDescOnce sync.Once\n\tfile_pkg_agent_api_v1_api_proto_rawDescData = file_pkg_agent_api_v1_api_proto_rawDesc\n)\n\nfunc file_pkg_agent_api_v1_api_proto_rawDescGZIP() []byte {\n\tfile_pkg_agent_api_v1_api_proto_rawDescOnce.Do(func() {\n\t\tfile_pkg_agent_api_v1_api_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_agent_api_v1_api_proto_rawDescData)\n\t})\n\treturn file_pkg_agent_api_v1_api_proto_rawDescData\n}\n\nvar file_pkg_agent_api_v1_api_proto_msgTypes = make([]protoimpl.MessageInfo, 10)\nvar file_pkg_agent_api_v1_api_proto_goTypes = []interface{}{\n\t(*GetNodeRequest)(nil),            // 0: v1.GetNodeRequest\n\t(*GetNodeReply)(nil),              // 1: v1.GetNodeReply\n\t(*JsonPatch)(nil),                 // 2: v1.JsonPatch\n\t(*PatchNodeRequest)(nil),          // 3: v1.PatchNodeRequest\n\t(*PatchNodeReply)(nil),            // 4: v1.PatchNodeReply\n\t(*UpdateNodeCapacityRequest)(nil), // 5: v1.UpdateNodeCapacityRequest\n\t(*UpdateNodeCapacityReply)(nil),   // 6: v1.UpdateNodeCapacityReply\n\t(*HealthCheckRequest)(nil),        // 7: v1.HealthCheckRequest\n\t(*HealthCheckReply)(nil),          // 8: v1.HealthCheckReply\n\tnil,                               // 9: v1.UpdateNodeCapacityRequest.CapacitiesEntry\n}\nvar file_pkg_agent_api_v1_api_proto_depIdxs = []int32{\n\t2, // 0: v1.PatchNodeRequest.patches:type_name -> v1.JsonPatch\n\t9, // 1: v1.UpdateNodeCapacityRequest.capacities:type_name -> v1.UpdateNodeCapacityRequest.CapacitiesEntry\n\t0, // 2: v1.Agent.GetNode:input_type -> v1.GetNodeRequest\n\t3, // 3: v1.Agent.PatchNode:input_type -> v1.PatchNodeRequest\n\t5, // 4: v1.Agent.UpdateNodeCapacity:input_type -> v1.UpdateNodeCapacityRequest\n\t7, // 5: v1.Agent.HealthCheck:input_type -> v1.HealthCheckRequest\n\t1, // 6: v1.Agent.GetNode:output_type -> v1.GetNodeReply\n\t4, // 7: v1.Agent.PatchNode:output_type -> v1.PatchNodeReply\n\t6, // 8: v1.Agent.UpdateNodeCapacity:output_type -> v1.UpdateNodeCapacityReply\n\t8, // 9: v1.Agent.HealthCheck:output_type -> v1.HealthCheckReply\n\t6, // [6:10] is the sub-list for method output_type\n\t2, // [2:6] is the sub-list for method input_type\n\t2, // [2:2] is the sub-list for extension type_name\n\t2, // [2:2] is the sub-list for extension extendee\n\t0, // [0:2] is the sub-list for field type_name\n}\n\nfunc init() { file_pkg_agent_api_v1_api_proto_init() }\nfunc file_pkg_agent_api_v1_api_proto_init() {\n\tif File_pkg_agent_api_v1_api_proto != nil {\n\t\treturn\n\t}\n\tif !protoimpl.UnsafeEnabled {\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*GetNodeRequest); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*GetNodeReply); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*JsonPatch); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*PatchNodeRequest); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*PatchNodeReply); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*UpdateNodeCapacityRequest); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*UpdateNodeCapacityReply); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*HealthCheckRequest); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_agent_api_v1_api_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*HealthCheckReply); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\ttype x struct{}\n\tout := protoimpl.TypeBuilder{\n\t\tFile: protoimpl.DescBuilder{\n\t\t\tGoPackagePath: reflect.TypeOf(x{}).PkgPath(),\n\t\t\tRawDescriptor: file_pkg_agent_api_v1_api_proto_rawDesc,\n\t\t\tNumEnums:      0,\n\t\t\tNumMessages:   10,\n\t\t\tNumExtensions: 0,\n\t\t\tNumServices:   1,\n\t\t},\n\t\tGoTypes:           file_pkg_agent_api_v1_api_proto_goTypes,\n\t\tDependencyIndexes: file_pkg_agent_api_v1_api_proto_depIdxs,\n\t\tMessageInfos:      file_pkg_agent_api_v1_api_proto_msgTypes,\n\t}.Build()\n\tFile_pkg_agent_api_v1_api_proto = out.File\n\tfile_pkg_agent_api_v1_api_proto_rawDesc = nil\n\tfile_pkg_agent_api_v1_api_proto_goTypes = nil\n\tfile_pkg_agent_api_v1_api_proto_depIdxs = nil\n}\n"
  },
  {
    "path": "pkg/agent/api/v1/api.proto",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\nsyntax = \"proto3\";\n\npackage v1;\noption go_package = \"../v1\";\n\nservice Agent{\n    rpc GetNode(GetNodeRequest) returns (GetNodeReply) {}\n    rpc PatchNode(PatchNodeRequest) returns (PatchNodeReply) {}\n    rpc UpdateNodeCapacity(UpdateNodeCapacityRequest) returns (UpdateNodeCapacityReply) {}\n    rpc HealthCheck(HealthCheckRequest) returns (HealthCheckReply) {}\n}\n\nmessage GetNodeRequest {\n}\n\nmessage GetNodeReply {\n    string node = 1;\n}\n\n// JsonPatch holds on JSON patch\nmessage JsonPatch {\n    string op = 1;\n    string path = 2;\n    string value = 3;\n}\n\nmessage PatchNodeRequest {\n    // List of JSON patches to apply on the node\n    repeated JsonPatch patches = 1;\n}\n\nmessage PatchNodeReply {\n}\n\nmessage UpdateNodeCapacityRequest {\n    // Name-value map of status.capacity to update\n    map<string, string> capacities = 1;\n}\n\nmessage UpdateNodeCapacityReply {\n}\n\nmessage HealthCheckRequest {\n    string query = 1;\n}\n\nmessage HealthCheckReply {\n    string error = 1;\n}\n"
  },
  {
    "path": "pkg/agent/api/v1/api_grpc.pb.go",
    "content": "// Code generated by protoc-gen-go-grpc. DO NOT EDIT.\n// versions:\n// - protoc-gen-go-grpc v1.2.0\n// - protoc             v3.20.1\n// source: pkg/agent/api/v1/api.proto\n\npackage v1\n\nimport (\n\tcontext \"context\"\n\tgrpc \"google.golang.org/grpc\"\n\tcodes \"google.golang.org/grpc/codes\"\n\tstatus \"google.golang.org/grpc/status\"\n)\n\n// This is a compile-time assertion to ensure that this generated file\n// is compatible with the grpc package it is being compiled against.\n// Requires gRPC-Go v1.32.0 or later.\nconst _ = grpc.SupportPackageIsVersion7\n\n// AgentClient is the client API for Agent service.\n//\n// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.\ntype AgentClient interface {\n\tGetNode(ctx context.Context, in *GetNodeRequest, opts ...grpc.CallOption) (*GetNodeReply, error)\n\tPatchNode(ctx context.Context, in *PatchNodeRequest, opts ...grpc.CallOption) (*PatchNodeReply, error)\n\tUpdateNodeCapacity(ctx context.Context, in *UpdateNodeCapacityRequest, opts ...grpc.CallOption) (*UpdateNodeCapacityReply, error)\n\tHealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckReply, error)\n}\n\ntype agentClient struct {\n\tcc grpc.ClientConnInterface\n}\n\nfunc NewAgentClient(cc grpc.ClientConnInterface) AgentClient {\n\treturn &agentClient{cc}\n}\n\nfunc (c *agentClient) GetNode(ctx context.Context, in *GetNodeRequest, opts ...grpc.CallOption) (*GetNodeReply, error) {\n\tout := new(GetNodeReply)\n\terr := c.cc.Invoke(ctx, \"/v1.Agent/GetNode\", in, out, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn out, nil\n}\n\nfunc (c *agentClient) PatchNode(ctx context.Context, in *PatchNodeRequest, opts ...grpc.CallOption) (*PatchNodeReply, error) {\n\tout := new(PatchNodeReply)\n\terr := c.cc.Invoke(ctx, \"/v1.Agent/PatchNode\", in, out, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn out, nil\n}\n\nfunc (c *agentClient) UpdateNodeCapacity(ctx context.Context, in *UpdateNodeCapacityRequest, opts ...grpc.CallOption) (*UpdateNodeCapacityReply, error) {\n\tout := new(UpdateNodeCapacityReply)\n\terr := c.cc.Invoke(ctx, \"/v1.Agent/UpdateNodeCapacity\", in, out, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn out, nil\n}\n\nfunc (c *agentClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckReply, error) {\n\tout := new(HealthCheckReply)\n\terr := c.cc.Invoke(ctx, \"/v1.Agent/HealthCheck\", in, out, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn out, nil\n}\n\n// AgentServer is the server API for Agent service.\n// All implementations must embed UnimplementedAgentServer\n// for forward compatibility\ntype AgentServer interface {\n\tGetNode(context.Context, *GetNodeRequest) (*GetNodeReply, error)\n\tPatchNode(context.Context, *PatchNodeRequest) (*PatchNodeReply, error)\n\tUpdateNodeCapacity(context.Context, *UpdateNodeCapacityRequest) (*UpdateNodeCapacityReply, error)\n\tHealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckReply, error)\n\tmustEmbedUnimplementedAgentServer()\n}\n\n// UnimplementedAgentServer must be embedded to have forward compatible implementations.\ntype UnimplementedAgentServer struct {\n}\n\nfunc (UnimplementedAgentServer) GetNode(context.Context, *GetNodeRequest) (*GetNodeReply, error) {\n\treturn nil, status.Errorf(codes.Unimplemented, \"method GetNode not implemented\")\n}\nfunc (UnimplementedAgentServer) PatchNode(context.Context, *PatchNodeRequest) (*PatchNodeReply, error) {\n\treturn nil, status.Errorf(codes.Unimplemented, \"method PatchNode not implemented\")\n}\nfunc (UnimplementedAgentServer) UpdateNodeCapacity(context.Context, *UpdateNodeCapacityRequest) (*UpdateNodeCapacityReply, error) {\n\treturn nil, status.Errorf(codes.Unimplemented, \"method UpdateNodeCapacity not implemented\")\n}\nfunc (UnimplementedAgentServer) HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckReply, error) {\n\treturn nil, status.Errorf(codes.Unimplemented, \"method HealthCheck not implemented\")\n}\nfunc (UnimplementedAgentServer) mustEmbedUnimplementedAgentServer() {}\n\n// UnsafeAgentServer may be embedded to opt out of forward compatibility for this service.\n// Use of this interface is not recommended, as added methods to AgentServer will\n// result in compilation errors.\ntype UnsafeAgentServer interface {\n\tmustEmbedUnimplementedAgentServer()\n}\n\nfunc RegisterAgentServer(s grpc.ServiceRegistrar, srv AgentServer) {\n\ts.RegisterService(&Agent_ServiceDesc, srv)\n}\n\nfunc _Agent_GetNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {\n\tin := new(GetNodeRequest)\n\tif err := dec(in); err != nil {\n\t\treturn nil, err\n\t}\n\tif interceptor == nil {\n\t\treturn srv.(AgentServer).GetNode(ctx, in)\n\t}\n\tinfo := &grpc.UnaryServerInfo{\n\t\tServer:     srv,\n\t\tFullMethod: \"/v1.Agent/GetNode\",\n\t}\n\thandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\treturn srv.(AgentServer).GetNode(ctx, req.(*GetNodeRequest))\n\t}\n\treturn interceptor(ctx, in, info, handler)\n}\n\nfunc _Agent_PatchNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {\n\tin := new(PatchNodeRequest)\n\tif err := dec(in); err != nil {\n\t\treturn nil, err\n\t}\n\tif interceptor == nil {\n\t\treturn srv.(AgentServer).PatchNode(ctx, in)\n\t}\n\tinfo := &grpc.UnaryServerInfo{\n\t\tServer:     srv,\n\t\tFullMethod: \"/v1.Agent/PatchNode\",\n\t}\n\thandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\treturn srv.(AgentServer).PatchNode(ctx, req.(*PatchNodeRequest))\n\t}\n\treturn interceptor(ctx, in, info, handler)\n}\n\nfunc _Agent_UpdateNodeCapacity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {\n\tin := new(UpdateNodeCapacityRequest)\n\tif err := dec(in); err != nil {\n\t\treturn nil, err\n\t}\n\tif interceptor == nil {\n\t\treturn srv.(AgentServer).UpdateNodeCapacity(ctx, in)\n\t}\n\tinfo := &grpc.UnaryServerInfo{\n\t\tServer:     srv,\n\t\tFullMethod: \"/v1.Agent/UpdateNodeCapacity\",\n\t}\n\thandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\treturn srv.(AgentServer).UpdateNodeCapacity(ctx, req.(*UpdateNodeCapacityRequest))\n\t}\n\treturn interceptor(ctx, in, info, handler)\n}\n\nfunc _Agent_HealthCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {\n\tin := new(HealthCheckRequest)\n\tif err := dec(in); err != nil {\n\t\treturn nil, err\n\t}\n\tif interceptor == nil {\n\t\treturn srv.(AgentServer).HealthCheck(ctx, in)\n\t}\n\tinfo := &grpc.UnaryServerInfo{\n\t\tServer:     srv,\n\t\tFullMethod: \"/v1.Agent/HealthCheck\",\n\t}\n\thandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\treturn srv.(AgentServer).HealthCheck(ctx, req.(*HealthCheckRequest))\n\t}\n\treturn interceptor(ctx, in, info, handler)\n}\n\n// Agent_ServiceDesc is the grpc.ServiceDesc for Agent service.\n// It's only intended for direct use with grpc.RegisterService,\n// and not to be introspected or modified (even as a copy)\nvar Agent_ServiceDesc = grpc.ServiceDesc{\n\tServiceName: \"v1.Agent\",\n\tHandlerType: (*AgentServer)(nil),\n\tMethods: []grpc.MethodDesc{\n\t\t{\n\t\t\tMethodName: \"GetNode\",\n\t\t\tHandler:    _Agent_GetNode_Handler,\n\t\t},\n\t\t{\n\t\t\tMethodName: \"PatchNode\",\n\t\t\tHandler:    _Agent_PatchNode_Handler,\n\t\t},\n\t\t{\n\t\t\tMethodName: \"UpdateNodeCapacity\",\n\t\t\tHandler:    _Agent_UpdateNodeCapacity_Handler,\n\t\t},\n\t\t{\n\t\t\tMethodName: \"HealthCheck\",\n\t\t\tHandler:    _Agent_HealthCheck_Handler,\n\t\t},\n\t},\n\tStreams:  []grpc.StreamDesc{},\n\tMetadata: \"pkg/agent/api/v1/api.proto\",\n}\n"
  },
  {
    "path": "pkg/agent/api/v1/constants.go",
    "content": "/*\nCopyright Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage v1\n\nconst (\n\t// ConfigStatus queries the status of the last configuration push to resmgr.\n\tConfigStatus = \"config-status\"\n)\n"
  },
  {
    "path": "pkg/agent/config-updater.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\t\"time\"\n\n\t\"context\"\n\t\"encoding/json\"\n\n\t\"google.golang.org/grpc\"\n\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tresmgr_v1 \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/config/api/v1\"\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// configuration update rate-limiting timeout\n\trateLimitTimeout = 2 * time.Second\n\t// setConfigTimeout is the duration we wait at most for a SetConfig reply\n\tsetConfigTimeout = 5 * time.Second\n\t// retryTimeout is the timeout after we retry sending configuration updates upon failure\n\tretryTimeout = 5 * time.Second\n)\n\n// configUpdater handles sending configuration to cri-resmgr\ntype configUpdater interface {\n\tStart() error\n\tStop()\n\tUpdateConfig(*resmgrConfig)\n\tUpdateAdjustment(*resmgrAdjustment)\n\tStatusChan() chan *resmgrStatus\n\tGetError() error\n}\n\n// updater implements configUpdater\ntype updater struct {\n\tlog.Logger\n\tresmgrCli     resmgr_v1.ConfigClient\n\tnewConfig     chan *resmgrConfig\n\tnewAdjustment chan *resmgrAdjustment\n\tnewStatus     chan *resmgrStatus\n\tcfgErrLock    sync.RWMutex\n\tcfgErr        error\n}\n\nfunc newConfigUpdater() (configUpdater, error) {\n\tu := &updater{Logger: log.NewLogger(\"config-updater\")}\n\n\tc, err := newResmgrCli(opts.resmgrSocket)\n\tif err != nil {\n\t\treturn nil, agentError(\"failed to create connection to cri-resmgr\")\n\t}\n\tu.resmgrCli = c\n\n\tu.newConfig = make(chan *resmgrConfig)\n\tu.newAdjustment = make(chan *resmgrAdjustment)\n\tu.newStatus = make(chan *resmgrStatus)\n\n\treturn u, nil\n}\n\nfunc (u *updater) Start() error {\n\tu.Info(\"Starting config-updater\")\n\tgo func() {\n\t\tvar pendingConfig *resmgrConfig\n\t\tvar pendingAdjustment *resmgrAdjustment\n\n\t\tvar ratelimit <-chan time.Time\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase cfg := <-u.newConfig:\n\t\t\t\tu.Info(\"scheduling update after %v rate-limiting timeout...\", rateLimitTimeout)\n\t\t\t\tpendingConfig = cfg\n\t\t\t\tratelimit = time.After(rateLimitTimeout)\n\n\t\t\tcase adjust := <-u.newAdjustment:\n\t\t\t\tu.Info(\"scheduling update after %v rate-limiting timeout...\", rateLimitTimeout)\n\t\t\t\tpendingAdjustment = adjust\n\t\t\t\tratelimit = time.After(rateLimitTimeout)\n\n\t\t\tcase _ = <-ratelimit:\n\t\t\t\tif pendingConfig != nil {\n\t\t\t\t\tmgrErr, err := u.setConfig(pendingConfig)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tu.Error(\"failed to send configuration update: %v\", err)\n\t\t\t\t\t\tratelimit = time.After(retryTimeout)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif mgrErr != nil {\n\t\t\t\t\t\t\tu.Error(\"cri-resmgr configuration error: %v\", mgrErr)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tpendingConfig = nil\n\t\t\t\t\t\tratelimit = nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif pendingAdjustment != nil {\n\t\t\t\t\terrors, err := u.setAdjustment(pendingAdjustment)\n\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tu.Error(\"failed to update adjustments: %+v\", err)\n\t\t\t\t\t}\n\t\t\t\t\tif len(errors) > 0 {\n\t\t\t\t\t\tu.Error(\"some adjustment updates failed: %+v\", errors)\n\t\t\t\t\t}\n\n\t\t\t\t\tu.newStatus <- &resmgrStatus{\n\t\t\t\t\t\trequest: err,\n\t\t\t\t\t\terrors:  errors,\n\t\t\t\t\t}\n\n\t\t\t\t\tpendingAdjustment = nil\n\t\t\t\t\tratelimit = nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn nil\n}\n\nfunc (u *updater) Stop() {\n}\n\nfunc (u *updater) UpdateConfig(c *resmgrConfig) {\n\tu.newConfig <- c\n}\n\nfunc (u *updater) UpdateAdjustment(c *resmgrAdjustment) {\n\tu.newAdjustment <- c\n}\n\nfunc (u *updater) StatusChan() chan *resmgrStatus {\n\treturn u.newStatus\n}\n\nfunc (u *updater) setError(err error) error {\n\tu.cfgErrLock.Lock()\n\tdefer u.cfgErrLock.Unlock()\n\tu.cfgErr = err\n\treturn err\n}\n\nfunc (u *updater) GetError() error {\n\tu.cfgErrLock.RLock()\n\tdefer u.cfgErrLock.RUnlock()\n\treturn u.cfgErr\n}\n\nfunc (u *updater) setConfig(cfg *resmgrConfig) (error, error) {\n\tctx, cancel := context.WithTimeout(context.Background(), setConfigTimeout)\n\tdefer cancel()\n\n\treq := &resmgr_v1.SetConfigRequest{NodeName: nodeName, Config: *cfg}\n\tu.Debug(\"sending SetConfig request to cri-resmgr\")\n\n\treply, err := u.resmgrCli.SetConfig(ctx, req, []grpc.CallOption{grpc.FailFast(false)}...)\n\n\tswitch {\n\tcase err != nil:\n\t\treturn nil, u.setError(err)\n\tcase reply.Error != \"\":\n\t\treturn u.setError(fmt.Errorf(\"%s\", reply.Error)), nil\n\tdefault:\n\t\treturn u.setError(nil), nil\n\t}\n}\n\nfunc (u *updater) setAdjustment(adjust *resmgrAdjustment) (map[string]string, error) {\n\tctx, cancel := context.WithTimeout(context.Background(), setConfigTimeout)\n\tdefer cancel()\n\n\tspecs := map[string]*resmgr.AdjustmentSpec{}\n\tfor name, p := range *adjust {\n\t\tspecs[name] = &resmgr.AdjustmentSpec{\n\t\t\tScope:        p.Spec.NodeScope(nodeName),\n\t\t\tResources:    p.Spec.Resources,\n\t\t\tClasses:      p.Spec.Classes,\n\t\t\tToptierLimit: p.Spec.ToptierLimit,\n\t\t}\n\t}\n\tencoded, err := json.Marshal(specs)\n\tif err != nil {\n\t\treturn nil, agentError(\"setAdjustment: failed to encode AdjustmentSpec: %v\", err)\n\t}\n\n\treq := &resmgr_v1.SetAdjustmentRequest{NodeName: nodeName, Adjustment: string(encoded)}\n\tu.Debug(\"sending SetAdjustment request to cri-resmgr\")\n\n\treply, err := u.resmgrCli.SetAdjustment(ctx, req, []grpc.CallOption{grpc.FailFast(false)}...)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn reply.Errors, nil\n}\n\nfunc newResmgrCli(socket string) (resmgr_v1.ConfigClient, error) {\n\tdialOpts := []grpc.DialOption{\n\t\tgrpc.WithInsecure(),\n\t\tgrpc.WithDialer(func(sock string, timeout time.Duration) (net.Conn, error) {\n\t\t\treturn net.Dial(\"unix\", socket)\n\t\t}),\n\t}\n\tconn, err := grpc.Dial(socket, dialOpts...)\n\tif err != nil {\n\t\treturn nil, agentError(\"failed to connect to cri-resmgr: %v\", err)\n\t}\n\treturn resmgr_v1.NewConfigClient(conn), nil\n}\n"
  },
  {
    "path": "pkg/agent/flags.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\t\"flag\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/sockets\"\n)\n\ntype options struct {\n\tkubeconfig    string\n\tagentSocket   string\n\tresmgrSocket  string\n\tconfigNs      string\n\tconfigMapName string\n\tlabelName     string\n}\n\nvar opts = options{}\n\nfunc init() {\n\tflag.StringVar(&opts.agentSocket, \"agent-socket\", sockets.ResourceManagerAgent, \"Socket for incoming requests from cri-resmgr\")\n\tflag.StringVar(&opts.resmgrSocket, \"cri-resmgr-socket\", sockets.ResourceManagerConfig, \"cri-resmgr socket to connect to\")\n\tflag.StringVar(&opts.kubeconfig, \"kubeconfig\", \"\", \"Kubeconfig to use, empty string implies in-cluster config (i.e. running inside a Pod)\")\n\tflag.StringVar(&opts.configNs, \"config-ns\", \"kube-system\", \"Kubernetes namespace where to look for config\")\n\tflag.StringVar(&opts.configMapName, \"configmap-name\", \"cri-resmgr-config\", \"Name of the K8s ConfigMap to watch\")\n\tflag.StringVar(&opts.labelName, \"label-name\", kubernetes.ResmgrKey(\"group\"), \"Name of the label used to assign a node to a configuration group.\")\n}\n"
  },
  {
    "path": "pkg/agent/kubernetes.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"time\"\n\n\tcore_v1 \"k8s.io/api/core/v1\"\n\tmeta_v1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/types\"\n\tk8swatch \"k8s.io/apimachinery/pkg/watch\"\n\tk8sclient \"k8s.io/client-go/kubernetes\"\n\n\t\"k8s.io/client-go/rest\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1\"\n\n\tagent_v1 \"github.com/intel/cri-resource-manager/pkg/agent/api/v1\"\n)\n\ntype namespace string\n\n// nodeName contains the name of the k8s we're running on\nvar nodeName string\n\n// getK8sClient initializes a new Kubernetes client\nfunc (a *agent) getK8sClient(kubeconfig string) (*k8sclient.Clientset, *resmgr.CriresmgrV1alpha1Client, error) {\n\tvar config *rest.Config\n\tvar err error\n\n\tif kubeconfig == \"\" {\n\t\ta.Info(\"using in-cluster kubeconfig\")\n\t\tconfig, err = rest.InClusterConfig()\n\t} else {\n\t\tconfig, err = clientcmd.BuildConfigFromFlags(\"\", kubeconfig)\n\t}\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tgenCli, err := k8sclient.NewForConfig(config)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tresmgr, err := resmgr.NewForConfig(config)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\treturn genCli, resmgr, nil\n}\n\n// getNodeObject gets a k8s Node object\nfunc getNodeObject(cli *k8sclient.Clientset) (*core_v1.Node, error) {\n\tnode, err := cli.CoreV1().Nodes().Get(context.TODO(), nodeName, meta_v1.GetOptions{})\n\tif err != nil {\n\t\treturn nil, agentError(\"failed to get node object for node %q: %v\", nodeName, err)\n\t}\n\treturn node, nil\n}\n\n// patchNodeObject is a helper for patching a k8s Node object\nfunc patchNode(cli *k8sclient.Clientset, patchList []*agent_v1.JsonPatch) error {\n\t// Convert patch list into bytes\n\tdata, err := json.Marshal(patchList)\n\tif err != nil {\n\t\treturn agentError(\"failed to marshal Node patches: %v\", err)\n\t}\n\n\t// Patch our node\n\tpt := types.JSONPatchType\n\t_, err = cli.CoreV1().Nodes().Patch(context.TODO(), nodeName, pt, data, meta_v1.PatchOptions{})\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// patchNodeStatus is a helper for patching the status of a k8s Node object\nfunc patchNodeStatus(cli *k8sclient.Clientset, fields map[string]string) error {\n\tpatch, sep := fmt.Sprintf(`{\"status\": {`), \"\"\n\tfor f, v := range fields {\n\t\tpatch += sep + fmt.Sprintf(`\"%s\": %s`, f, v)\n\t\tsep = \",\"\n\t}\n\tpatch += \"}}\"\n\n\t_, err := cli.CoreV1().Nodes().PatchStatus(context.TODO(), nodeName, []byte(patch))\n\n\treturn err\n}\n\n// patchAdjustmentStatus is a helper for patching the status of a Adjustment CRD.\nfunc patchAdjustmentStatus(_ *resmgr.CriresmgrV1alpha1Client, _ *resmgrStatus, _ ...string) error {\n\treturn nil\n}\n\n// watch is a wrapper around the k8s watch.Interface\ntype watch struct {\n\tparent  *watcher\n\tkind    string\n\tns      namespace\n\tname    string\n\topenfn  func(namespace, string) (k8swatch.Interface, error)\n\tqueryfn func(namespace, string) (interface{}, error)\n\tstop    chan struct{}\n\tevents  chan k8swatch.Event\n}\n\n// openFn is the type for functions creating k8s watcher of a particular kind.\ntype openFn func(ns namespace, name string) (k8swatch.Interface, error)\n\n// queryFn is the type for functions querying k8s objects being watched.\ntype queryFn func(ns namespace, name string) (interface{}, error)\n\nconst (\n\t// SyntheticMissing is a synthetic initial event for currently non-existent object.\n\tSyntheticMissing = k8swatch.EventType(\"SyntheticMissing\")\n)\n\nfunc newWatch(parent *watcher, kind string, ns namespace, open openFn, query queryFn) *watch {\n\treturn &watch{\n\t\tparent:  parent,\n\t\tkind:    kind,\n\t\tns:      ns,\n\t\tstop:    make(chan struct{}),\n\t\tevents:  make(chan k8swatch.Event),\n\t\topenfn:  open,\n\t\tqueryfn: query,\n\t}\n}\n\n// newNodeWatch creates a watch for k8s Node\nfunc newNodeWatch(parent *watcher) *watch {\n\tw := newWatch(parent, \"Node\", namespace(\"\"),\n\t\tfunc(ns namespace, name string) (k8swatch.Interface, error) {\n\t\t\tselector := meta_v1.ListOptions{FieldSelector: \"metadata.name=\" + name}\n\t\t\tk8w, err := parent.k8sCli.CoreV1().Nodes().Watch(context.TODO(), selector)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn k8w, nil\n\t\t},\n\t\tfunc(ns namespace, name string) (interface{}, error) {\n\t\t\tnoopts := meta_v1.GetOptions{}\n\t\t\tnode, err := parent.k8sCli.CoreV1().Nodes().Get(context.TODO(), name, noopts)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn node, nil\n\t\t})\n\tw.Start(nodeName)\n\treturn w\n}\n\n// newConfigMapWatch creates a watch for k8s ConfigMap\nfunc newConfigMapWatch(parent *watcher, name string, ns namespace) *watch {\n\tw := newWatch(parent, \"ConfigMap\", ns,\n\t\tfunc(ns namespace, name string) (k8swatch.Interface, error) {\n\t\t\tselector := meta_v1.ListOptions{FieldSelector: \"metadata.name=\" + name}\n\t\t\tk8w, err := parent.k8sCli.CoreV1().ConfigMaps(string(ns)).Watch(context.TODO(), selector)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn k8w, nil\n\t\t},\n\t\tfunc(ns namespace, name string) (interface{}, error) {\n\t\t\tnoopts := meta_v1.GetOptions{}\n\t\t\tcm, err := parent.k8sCli.CoreV1().ConfigMaps(string(ns)).Get(context.TODO(), name, noopts)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn cm, nil\n\t\t})\n\tw.Start(name)\n\treturn w\n}\n\n// newAdustmentCRDWatch creates a watch for k8s Adjustment CRDs\nfunc newAdjustmentCRDWatch(parent *watcher, ns namespace) *watch {\n\tw := newWatch(parent, \"AdjustmentCRD\", ns,\n\t\tfunc(ns namespace, name string) (k8swatch.Interface, error) {\n\t\t\tk8w, err := parent.resmgrCli.Adjustments(string(ns)).Watch(meta_v1.ListOptions{})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn k8w, nil\n\t\t},\n\t\tfunc(ns namespace, name string) (interface{}, error) {\n\t\t\tcrds, err := parent.resmgrCli.Adjustments(string(ns)).List(meta_v1.ListOptions{})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif crds == nil || len(crds.Items) == 0 {\n\t\t\t\tcrds = nil\n\t\t\t}\n\t\t\treturn crds, nil\n\t\t})\n\tw.Start(\"AdjustmentCRD\")\n\treturn w\n}\n\nfunc (w *watch) Name() string {\n\tns, name := w.ns, w.name\n\tif ns != \"\" {\n\t\tns += \"/\"\n\t}\n\tif name == \"\" {\n\t\tname = \"<none>\"\n\t}\n\treturn w.kind + \":\" + string(ns) + name\n}\n\n// Query queries the object being watched.\nfunc (w *watch) Query() (interface{}, error) {\n\tif w.name == \"\" {\n\t\treturn nil, nil\n\t}\n\treturn w.queryfn(w.ns, w.name)\n}\n\n// Start watching an object.\nfunc (w *watch) Start(name string) {\n\tw.Stop()\n\tw.name = name\n\n\tif w.name == \"\" {\n\t\treturn\n\t}\n\n\t// proxy events from a go-routine until we're told to stop.\n\tgo func() {\n\t\tvar k8w k8swatch.Interface\n\t\tvar events <-chan k8swatch.Event\n\t\tvar ratelimit <-chan time.Time\n\t\tvar err error\n\n\t\t// let the watcher know not to expect initial event\n\t\tif objs, _ := w.queryfn(w.ns, w.name); objs == nil {\n\t\t\tw.events <- k8swatch.Event{Type: SyntheticMissing}\n\t\t}\n\n\t\tfor {\n\t\t\tif events == nil {\n\t\t\t\tw.parent.Info(\"creating %s watch\", w.Name())\n\t\t\t\tif k8w, err = w.openfn(w.ns, w.name); err != nil {\n\t\t\t\t\tw.parent.Warn(\"failed to create %s watch: %v\", w.Name(), err)\n\t\t\t\t\tratelimit = time.After(1 * time.Second)\n\t\t\t\t} else {\n\t\t\t\t\tevents = k8w.ResultChan()\n\t\t\t\t\tratelimit = nil\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tselect {\n\t\t\tcase _ = <-w.stop:\n\t\t\t\tif events != nil {\n\t\t\t\t\tk8w.Stop()\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\tcase e, ok := <-events:\n\t\t\t\tif ok {\n\t\t\t\t\tw.events <- e\n\t\t\t\t} else {\n\t\t\t\t\tw.parent.Warn(\"failed to get event from watch %s\", w.Name())\n\t\t\t\t\tk8w.Stop()\n\t\t\t\t\tevents = nil\n\t\t\t\t}\n\t\t\tcase _ = <-ratelimit:\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// Close closes a watch.\nfunc (w *watch) Stop() {\n\tselect {\n\tcase w.stop <- struct{}{}:\n\tdefault:\n\t}\n}\n\n// ResultChan returns the event channel of the watch.\nfunc (w *watch) ResultChan() <-chan k8swatch.Event {\n\treturn w.events\n}\n\nfunc init() {\n\t// Node name is expected to be set in an environment variable\n\tnodeName = os.Getenv(\"NODE_NAME\")\n}\n"
  },
  {
    "path": "pkg/agent/server.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"google.golang.org/grpc\"\n\tcore_v1 \"k8s.io/api/core/v1\"\n\tk8sclient \"k8s.io/client-go/kubernetes\"\n\n\tv1 \"github.com/intel/cri-resource-manager/pkg/agent/api/v1\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/sockets\"\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\n// agentServer is the interface for our gRPC server.\ntype agentServer interface {\n\tStart(string) error\n\tStop()\n}\n\n// server implements agentServer.\ntype server struct {\n\tlog.Logger\n\tcli    *k8sclient.Clientset // client for accessing k8s api\n\tserver *grpc.Server         // gRPC server instance\n\tcfg    configInterface\n}\n\n// newAgentServer creates new agentServer instance.\nfunc newAgentServer(cli *k8sclient.Clientset, cfg configInterface) (agentServer, error) {\n\ts := &server{\n\t\tLogger: log.NewLogger(\"server\"),\n\t\tcli:    cli,\n\t\tcfg:    cfg,\n\t}\n\n\treturn s, nil\n}\n\n// Start runs server instance.\nfunc (s *server) Start(socket string) error {\n\t// Make sure we have a directory for the socket.\n\tif err := os.MkdirAll(filepath.Dir(socket), sockets.DirPermissions); err != nil {\n\t\treturn agentError(\"failed to create directory for socket %s: %v\", socket, err)\n\t}\n\n\t// Remove any leftover sockets.\n\tif err := os.Remove(socket); err != nil && !os.IsNotExist(err) {\n\t\treturn agentError(\"failed to unlink socket file: %s\", err)\n\t}\n\n\t// Create server listening for local unix domain socket\n\tlis, err := net.Listen(\"unix\", socket)\n\tif err != nil {\n\t\treturn agentError(\"failed to listen to socket: %v\", err)\n\t}\n\n\tserverOpts := []grpc.ServerOption{}\n\ts.server = grpc.NewServer(serverOpts...)\n\tgs := &grpcServer{\n\t\tLogger: s.Logger,\n\t\tcli:    s.cli,\n\t\tcfg:    s.cfg,\n\t}\n\tv1.RegisterAgentServer(s.server, gs)\n\n\ts.Info(\"starting gRPC server at socket %s\", socket)\n\tgo func() {\n\t\tdefer lis.Close()\n\t\terr := s.server.Serve(lis)\n\t\tif err != nil {\n\t\t\ts.Fatal(\"grpc server died: %v\", err)\n\t\t}\n\t}()\n\treturn nil\n}\n\n// Stop agentServer instance\nfunc (s *server) Stop() {\n\ts.server.Stop()\n}\n\n// grpcServer implements v1.AgentServer\ntype grpcServer struct {\n\tv1.UnimplementedAgentServer\n\tlog.Logger\n\tcli *k8sclient.Clientset\n\tcfg configInterface\n}\n\n// GetNode gets K8s node object.\nfunc (g *grpcServer) GetNode(_ context.Context, req *v1.GetNodeRequest) (*v1.GetNodeReply, error) {\n\tg.Debug(\"received GetNodeRequest: %v\", req)\n\trpl := &v1.GetNodeReply{}\n\n\tnode, err := getNodeObject(g.cli)\n\tif err != nil {\n\t\treturn rpl, agentError(\"failed to get node object: %v\", err)\n\t}\n\tserialized, err := json.Marshal(node)\n\tif err != nil {\n\t\treturn rpl, agentError(\"failed to serialized node object: %v\", err)\n\t}\n\trpl.Node = string(serialized)\n\n\treturn rpl, nil\n}\n\n// PatchNode patches the K8s node object.\nfunc (g *grpcServer) PatchNode(_ context.Context, req *v1.PatchNodeRequest) (*v1.PatchNodeReply, error) {\n\tg.Debug(\"received PatchNodeRequest: %v\", req)\n\trpl := &v1.PatchNodeReply{}\n\n\t// Apply patches\n\tif len(req.Patches) > 0 {\n\t\terr := patchNode(g.cli, req.Patches)\n\t\tif err != nil {\n\t\t\treturn rpl, agentError(\"failed to patch node object: %v\", err)\n\t\t}\n\t}\n\n\treturn rpl, nil\n}\n\n// UpdateNodeCapacity updates capacity in Node status\nfunc (g *grpcServer) UpdateNodeCapacity(_ context.Context, req *v1.UpdateNodeCapacityRequest) (*v1.UpdateNodeCapacityReply, error) {\n\tg.Debug(\"received UpdateNodeCapacityRequest: %v\", req)\n\n\trpl := &v1.UpdateNodeCapacityReply{}\n\n\tcapacity, sep := \"\", \"\"\n\tfor name, count := range req.Capacities {\n\t\tif isNativeResource(name) {\n\t\t\terr := agentError(\"refusing to update capacity of native resource '%s'\", name)\n\t\t\treturn rpl, err\n\t\t}\n\n\t\tif !strings.Contains(name, \".\") || !strings.Contains(name, \"/\") {\n\t\t\terr := agentError(\"invalid resource '%s' in capacity update\", name)\n\t\t\treturn rpl, err\n\t\t}\n\n\t\tcapacity += sep + fmt.Sprintf(`\"%s\": \"%s\"`, name, count)\n\t\tsep = \", \"\n\t}\n\n\terr := patchNodeStatus(g.cli, map[string]string{\"capacity\": \"{\" + capacity + \"}\"})\n\n\treturn rpl, err\n}\n\n// HealthCheck checks if the agent is in healthy state\nfunc (g *grpcServer) HealthCheck(_ context.Context, req *v1.HealthCheckRequest) (*v1.HealthCheckReply, error) {\n\tg.Debug(\"received HealthCheckRequest: %v\", req)\n\n\treply := &v1.HealthCheckReply{}\n\n\tif req.Query == v1.ConfigStatus {\n\t\tif err := g.cfg.getError(); err != nil {\n\t\t\treply.Error = fmt.Sprintf(\"configuration error: %v\", err)\n\t\t}\n\t}\n\n\treturn reply, nil\n}\n\nfunc isNativeResource(name string) bool {\n\tswitch {\n\tcase name == string(core_v1.ResourceCPU), name == string(core_v1.ResourceMemory):\n\t\treturn true\n\tcase strings.HasPrefix(name, core_v1.ResourceHugePagesPrefix):\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n"
  },
  {
    "path": "pkg/agent/watcher.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\tcore_v1 \"k8s.io/api/core/v1\"\n\tk8swatch \"k8s.io/apimachinery/pkg/watch\"\n\tk8sclient \"k8s.io/client-go/kubernetes\"\n\t\"sync\"\n\t\"time\"\n\n\t\"encoding/json\"\n\tpatch \"github.com/evanphx/json-patch\"\n\tpkgtypes \"k8s.io/apimachinery/pkg/types\"\n\n\tresmgrcli \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1\"\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\ntype cachedConfig struct {\n\tsync.RWMutex\n\tnodeCfg  *resmgrConfig    // node-specific configuration\n\tgroupCfg *resmgrConfig    // group-specific configuration\n\tgroup    string           // group name, \"\" for default\n\tinscope  resmgrAdjustment // external adjustments that apply to this node\n\tignored  resmgrAdjustment // external adjustments that do not apply to this node\n\tstatus   *resmgrStatus    // latest adjustment update status\n}\n\n// k8sWatcher is our interface to K8s control plane watcher\ntype k8sWatcher interface {\n\t// Start the watcher instance\n\tStart() error\n\t// Stop the watcher instance\n\tStop()\n\t// Get a chan through which to receive configuration updates\n\tConfigChan() <-chan resmgrConfig\n\t// Get up-to-date config\n\tGetConfig() resmgrConfig\n\t// Get a chan through which to receive adjustment updates\n\tAdjustmentChan() <-chan resmgrAdjustment\n\t// Update the node Status for adjustment updates.\n\tUpdateStatus(*resmgrStatus) error\n}\n\n// watcher implements k8sWatcher\ntype watcher struct {\n\tlog.Logger\n\tstop           chan struct{}                      // channel to stop watcher goroutine\n\tk8sCli         *k8sclient.Clientset               // k8s client interface\n\tresmgrCli      *resmgrcli.CriresmgrV1alpha1Client // adjustment CRD interface\n\tcurrentConfig  cachedConfig                       // current configuration, cached\n\tconfigChan     chan resmgrConfig                  // channel for config updates\n\tadjustmentChan chan resmgrAdjustment              // channel for adjustment updates\n}\n\n// newK8sWatcher creates a new K8sWatcher instance\nfunc newK8sWatcher(k8sCli *k8sclient.Clientset, resmgrCli *resmgrcli.CriresmgrV1alpha1Client) (k8sWatcher, error) {\n\tw := &watcher{\n\t\tLogger:         log.NewLogger(\"watcher\"),\n\t\tk8sCli:         k8sCli,\n\t\tresmgrCli:      resmgrCli,\n\t\tstop:           make(chan struct{}, 1),\n\t\tcurrentConfig:  newCachedConfig(),\n\t\tconfigChan:     make(chan resmgrConfig, 1),\n\t\tadjustmentChan: make(chan resmgrAdjustment, 1),\n\t}\n\n\treturn w, nil\n}\n\n// Start runs a k8sWatcher instance\nfunc (w *watcher) Start() error {\n\tw.Info(\"starting watcher...\")\n\tif nodeName == \"\" {\n\t\treturn agentError(\"node name not set, NODE_NAME env variable should be set to match the name of this k8s Node\")\n\t}\n\n\tgo func() {\n\t\tw.watch()\n\t}()\n\treturn nil\n}\n\n// Stop stops a running k8sWatcher instance\nfunc (w *watcher) Stop() {\n\tselect {\n\tcase w.stop <- struct{}{}:\n\tdefault:\n\t\tw.Debug(\"stop already sent\")\n\t}\n}\n\n// ConfigChan returns the chan for config updates\nfunc (w *watcher) ConfigChan() <-chan resmgrConfig {\n\treturn w.configChan\n}\n\n// AdjustmentChan returns the chan for adjustment updates\nfunc (w *watcher) AdjustmentChan() <-chan resmgrAdjustment {\n\treturn w.adjustmentChan\n}\n\n// GetConfig returns the current cri-resmgr configuration\nfunc (w *watcher) GetConfig() resmgrConfig {\n\tcfg, kind := w.currentConfig.getConfig()\n\tw.Info(\"giving %s configuration in reply to query\", kind)\n\treturn cfg\n}\n\n// UpdateStatus updates the node status for adjustment updates.\nfunc (w *watcher) UpdateStatus(status *resmgrStatus) error {\n\tw.currentConfig.setStatus(status)\n\treturn w.PatchAdjustmentStatus(status)\n}\n\n// PatchAdjustmentStatus updates the node status for adjustment updates.\nfunc (w *watcher) PatchAdjustmentStatus(status *resmgrStatus) error {\n\terrors := status.errors\n\tif errors == nil {\n\t\terrors = map[string]string{}\n\t}\n\tif status.request != nil {\n\t\terrors[\"request\"] = status.request.Error()\n\t}\n\n\tinscope, ignored := w.currentConfig.getAdjustment()\n\n\tw.currentConfig.Lock()\n\tdefer w.currentConfig.Unlock()\n\n\terrCnt := 0\n\tfor _, adjust := range inscope {\n\t\tif err := w.patchAdjustment(adjust, true, errors); err != nil {\n\t\t\tw.Error(\"%v\", err)\n\t\t\terrCnt++\n\t\t}\n\t}\n\tfor _, adjust := range ignored {\n\t\tif err := w.patchAdjustment(adjust, false, errors); err != nil {\n\t\t\tw.Error(\"%v\", err)\n\t\t\terrCnt++\n\t\t}\n\t}\n\tif errCnt > 0 {\n\t\treturn agentError(\"some adjustment status updates failed\")\n\t}\n\n\treturn nil\n}\n\n// patchAdjustment patches the status of an update to the given adjustment.\nfunc (w *watcher) patchAdjustment(adjust *resmgr.Adjustment, inscope bool, errors map[string]string) error {\n\tvar pdata []byte\n\tvar err error\n\n\told, ok := adjust.Status.Nodes[nodeName]\n\n\tif !inscope {\n\t\tif !ok {\n\t\t\tw.Debug(\"adjustment %s does not need status patching...\", adjust.Name)\n\t\t\treturn nil\n\t\t}\n\t\tcurrent := &resmgr.Adjustment{\n\t\t\tStatus: resmgr.AdjustmentStatus{\n\t\t\t\tNodes: map[string]resmgr.AdjustmentNodeStatus{\n\t\t\t\t\tnodeName: old,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tupdated := &resmgr.Adjustment{\n\t\t\tStatus: resmgr.AdjustmentStatus{\n\t\t\t\tNodes: map[string]resmgr.AdjustmentNodeStatus{},\n\t\t\t},\n\t\t}\n\t\toldData, _ := json.Marshal(current)\n\t\tnewData, _ := json.Marshal(updated)\n\t\tpdata, err = patch.CreateMergePatch(oldData, newData)\n\t\tif err != nil {\n\t\t\treturn agentError(\"failed to adjustment status patch: %v\", err)\n\t\t}\n\t} else {\n\t\tcurrent := &resmgr.Adjustment{\n\t\t\tStatus: resmgr.AdjustmentStatus{\n\t\t\t\tNodes: map[string]resmgr.AdjustmentNodeStatus{},\n\t\t\t},\n\t\t}\n\t\tif ok {\n\t\t\tcurrent.Status.Nodes[nodeName] = old\n\t\t}\n\t\tupdated := &resmgr.Adjustment{\n\t\t\tStatus: resmgr.AdjustmentStatus{\n\t\t\t\tNodes: map[string]resmgr.AdjustmentNodeStatus{\n\t\t\t\t\tnodeName: {Errors: errors},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\toldData, _ := json.Marshal(current)\n\t\tnewData, _ := json.Marshal(updated)\n\t\tpdata, err = patch.CreateMergePatch(oldData, newData)\n\t\tif err != nil {\n\t\t\treturn agentError(\"failed to adjustment status patch: %v\", err)\n\t\t}\n\t}\n\n\tptype := pkgtypes.MergePatchType\n\n\tw.Debug(\"patching status of adjustment %s status with %v...\", adjust.Name, string(pdata))\n\n\tif _, err := w.resmgrCli.Adjustments(opts.configNs).Patch(adjust.Name, ptype, pdata); err != nil {\n\t\treturn agentError(\"failed to patch Adjustment CRD %q: %v\", adjust.Name, err)\n\t}\n\n\tif inscope {\n\t\tif adjust.Status.Nodes == nil {\n\t\t\tadjust.Status.Nodes = make(map[string]resmgr.AdjustmentNodeStatus)\n\t\t}\n\t\tadjust.Status.Nodes[nodeName] = resmgr.AdjustmentNodeStatus{Errors: errors}\n\t} else {\n\t\tdelete(adjust.Status.Nodes, nodeName)\n\t}\n\n\treturn nil\n}\n\n// sendConfig sends the current configuration.\nfunc (w *watcher) sendConfig() {\n\tcfg, kind := w.currentConfig.getConfig()\n\tw.Info(\"pushing %s configuration to client\", kind)\n\tw.configChan <- cfg\n}\n\n// sendAdjustment sends the current overridden policies.\nfunc (w *watcher) sendAdjustment() {\n\tinscope, _ := w.currentConfig.getAdjustment()\n\tw.adjustmentChan <- inscope\n}\n\nfunc (w *watcher) watch() error {\n\tnodew := newNodeWatch(w)\n\tgroup := \"\"\n\n\tif node, err := nodew.Query(); err != nil {\n\t\tw.Warn(\"failed to query node %q: %v\", nodeName, err)\n\t} else if node == nil {\n\t\tw.Warn(\"failed to query node %q, make sure that NODE_NAME is correctly set\", nodeName)\n\t} else {\n\t\tgroup = node.(*core_v1.Node).Labels[opts.labelName]\n\t\tw.Info(\"configuration group is set to '%s'\", group)\n\t}\n\n\tcfgw := newConfigMapWatch(w, opts.configMapName+\".node.\"+nodeName, namespace(opts.configNs))\n\tgrpw := newConfigMapWatch(w, groupMapName(group), namespace(opts.configNs))\n\tcrdw := newAdjustmentCRDWatch(w, namespace(opts.configNs))\n\n\tw.Info(\"watcher running\")\n\tw.sendConfig()\n\n\tfor {\n\t\tselect {\n\t\tcase _ = <-w.stop:\n\t\t\tw.Info(\"stopping configuration watcher\")\n\t\t\tnodew.Stop()\n\t\t\tcfgw.Stop()\n\t\t\tgrpw.Stop()\n\t\t\tcrdw.Stop()\n\t\t\treturn nil\n\n\t\tcase e, ok := <-nodew.ResultChan():\n\t\t\tif ok {\n\t\t\t\tswitch e.Type {\n\t\t\t\tcase k8swatch.Added, k8swatch.Modified:\n\t\t\t\t\tw.Info(\"node (%s) configuration updated\", nodeName)\n\t\t\t\t\tlabel, _ := e.Object.(*core_v1.Node).Labels[opts.labelName]\n\t\t\t\t\tif group != label {\n\t\t\t\t\t\tgroup = label\n\t\t\t\t\t\tw.Info(\"configuration group is set to '%s'\", group)\n\t\t\t\t\t\tgrpw.Start(groupMapName(group))\n\t\t\t\t\t}\n\t\t\t\tcase k8swatch.Deleted:\n\t\t\t\t\tw.Warn(\"Hmm, our node got removed...\")\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\tcase e, ok := <-cfgw.ResultChan():\n\t\t\tif ok {\n\t\t\t\tswitch e.Type {\n\t\t\t\tcase k8swatch.Added, k8swatch.Modified:\n\t\t\t\t\tw.Info(\"node ConfigMap updated\")\n\t\t\t\t\tcm := e.Object.(*core_v1.ConfigMap)\n\t\t\t\t\tw.currentConfig.setNode(&cm.Data)\n\t\t\t\t\tw.sendConfig()\n\n\t\t\t\tcase k8swatch.Deleted, SyntheticMissing:\n\t\t\t\t\tw.Info(\"node ConfigMap deleted\")\n\t\t\t\t\tw.currentConfig.setNode(nil)\n\t\t\t\t\tw.sendConfig()\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\tcase e, ok := <-grpw.ResultChan():\n\t\t\tif ok {\n\t\t\t\tswitch e.Type {\n\t\t\t\tcase k8swatch.Added, k8swatch.Modified:\n\t\t\t\t\tw.Info(\"group/default ConfigMap updated\")\n\t\t\t\t\tcm := e.Object.(*core_v1.ConfigMap)\n\t\t\t\t\tif w.currentConfig.setGroup(group, &cm.Data) {\n\t\t\t\t\t\tw.sendConfig()\n\t\t\t\t\t}\n\t\t\t\tcase k8swatch.Deleted, SyntheticMissing:\n\t\t\t\t\tw.Info(\"group/default ConfigMap deleted\")\n\t\t\t\t\tif w.currentConfig.setGroup(group, nil) {\n\t\t\t\t\t\tw.sendConfig()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\tcase e, ok := <-crdw.ResultChan():\n\t\t\tif ok {\n\t\t\t\tswitch e.Type {\n\t\t\t\tcase k8swatch.Added, k8swatch.Modified:\n\t\t\t\t\tw.Info(\"Adjustment CRD(s) updated: %T, %+v\", e.Object, e.Object)\n\t\t\t\t\tw.Info(\"Adjustment CRD(s): %+v\", e.Object.(*resmgr.Adjustment).Spec)\n\t\t\t\t\tif w.currentConfig.setAdjustment(e.Object.(*resmgr.Adjustment)) {\n\t\t\t\t\t\tw.sendAdjustment()\n\t\t\t\t\t}\n\n\t\t\t\tcase k8swatch.Deleted:\n\t\t\t\t\tw.Info(\"Adjustment CRD(s) (%T) deleted\", e.Object)\n\t\t\t\t\tif w.currentConfig.deleteAdjustment(e.Object.(*resmgr.Adjustment)) {\n\t\t\t\t\t\tw.sendAdjustment()\n\t\t\t\t\t}\n\n\t\t\t\tcase SyntheticMissing:\n\t\t\t\t\tw.Info(\"No Adjustment CRD(s)\")\n\t\t\t\t\tw.sendAdjustment()\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// shouln't be necessary, but just in case avoid spinning on a closed channel\n\t\ttime.Sleep(1 * time.Second)\n\t}\n}\n\n// groupMapName returns the our group ConfigMap, or the default one is we have no group.\nfunc groupMapName(group string) string {\n\tif group == \"\" {\n\t\treturn opts.configMapName + \".default\"\n\t}\n\treturn opts.configMapName + \".group.\" + group\n}\n\n// newCacheConfig creates a new cachedConfig instance.\nfunc newCachedConfig() cachedConfig {\n\treturn cachedConfig{\n\t\tinscope: resmgrAdjustment{},\n\t\tignored: resmgrAdjustment{},\n\t}\n}\n\n// getConfig is a helper method for getting the config data\nfunc (c *cachedConfig) getConfig() (resmgrConfig, string) {\n\tc.RLock()\n\tdefer c.RUnlock()\n\n\tvar cfg *resmgrConfig\n\tvar kind string\n\n\tswitch {\n\tcase c.nodeCfg != nil:\n\t\tkind = \"node\"\n\t\tcfg = c.nodeCfg\n\tcase c.group != \"\":\n\t\tkind = \"group \" + c.group\n\t\tcfg = c.groupCfg\n\tcase c.groupCfg != nil:\n\t\tkind = \"default\"\n\t\tcfg = c.groupCfg\n\tdefault:\n\t\tkind = \"fallback\"\n\t}\n\n\tif cfg == nil {\n\t\tkind = \"empty \" + kind\n\t\tcfg = &resmgrConfig{}\n\t}\n\n\treturn *cfg, kind\n}\n\n// getAdjustment is a helper method for getting a copy of external adjustments\nfunc (c *cachedConfig) getAdjustment() (resmgrAdjustment, resmgrAdjustment) {\n\tc.RLock()\n\tdefer c.RUnlock()\n\n\tinscope := resmgrAdjustment{}\n\tfor name, value := range c.inscope {\n\t\tinscope[name] = value\n\t}\n\tignored := resmgrAdjustment{}\n\tfor name, value := range c.ignored {\n\t\tignored[name] = value\n\t}\n\n\treturn inscope, ignored\n}\n\n// set node-specific configuration\nfunc (c *cachedConfig) setNode(data *map[string]string) bool {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\tc.nodeCfg = (*resmgrConfig)(data)\n\treturn true\n}\n\n// set group-specific or default configuration\nfunc (c *cachedConfig) setGroup(group string, data *map[string]string) bool {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\tc.groupCfg = (*resmgrConfig)(data)\n\tc.group = group\n\treturn c.nodeCfg == nil\n}\n\n// setAdjustment is a helper method for updating external adjustments\nfunc (c *cachedConfig) setAdjustment(adjust *resmgr.Adjustment) bool {\n\tvar inscope, ignored bool\n\tvar updated *resmgr.Adjustment\n\n\tc.Lock()\n\tdefer c.Unlock()\n\n\t//\n\t// filter out updates\n\t//   - for expired watches being recreated\n\t//   - without any Spec changes (Status updates)\n\t//\n\n\tif updated, inscope = c.inscope[adjust.Name]; inscope {\n\t\tif adjust.HasSameVersion(updated) || adjust.Spec.Compare(&updated.Spec) {\n\t\t\tc.inscope[adjust.Name] = adjust\n\t\t\treturn false\n\t\t}\n\t} else if updated, ignored = c.ignored[adjust.Name]; ignored {\n\t\tif adjust.HasSameVersion(updated) || adjust.Spec.Compare(&updated.Spec) {\n\t\t\tc.ignored[adjust.Name] = adjust\n\t\t\treturn false\n\t\t}\n\t}\n\n\t//\n\t// we need to notify cri-resmgr if\n\t//   - the adjustment applies to this node\n\t//   - the adjustment used to apply to this node before the update\n\t//\n\n\tnotify := false\n\tif adjust.Spec.IsNodeInScope(nodeName) {\n\t\tc.inscope[adjust.Name] = adjust\n\t\tif ignored {\n\t\t\tdelete(c.ignored, adjust.Name)\n\t\t}\n\t\tnotify = true\n\t} else {\n\t\tc.ignored[adjust.Name] = adjust\n\t\tif inscope {\n\t\t\tdelete(c.inscope, adjust.Name)\n\t\t\tnotify = true\n\t\t}\n\t}\n\n\treturn notify\n}\n\n// deleteAdjustment is a helper method for updating external adjustments\nfunc (c *cachedConfig) deleteAdjustment(o *resmgr.Adjustment) bool {\n\tc.Lock()\n\tdefer c.Unlock()\n\n\t// we need to notify cri-resmgr if the deleted adjustment used to apply to this node\n\tif _, ok := c.inscope[o.Name]; ok {\n\t\tdelete(c.inscope, o.Name)\n\t\treturn true\n\t}\n\n\tdelete(c.ignored, o.Name)\n\treturn false\n}\n\n// getAdjustmentNames returns the names of in scope and ignored adjustments.\nfunc (c *cachedConfig) getAdjustmentNames() ([]string, []string) {\n\tc.RLock()\n\tdefer c.RUnlock()\n\n\tinscope := make([]string, 0, len(c.inscope))\n\tignored := make([]string, 0, len(c.ignored))\n\tfor name := range c.inscope {\n\t\tinscope = append(inscope, name)\n\t}\n\tfor name := range c.ignored {\n\t\tignored = append(ignored, name)\n\t}\n\treturn inscope, ignored\n}\n\n// cache the status of the last adjustment update\nfunc (c *cachedConfig) setStatus(status *resmgrStatus) {\n\tc.Lock()\n\tdefer c.Unlock()\n\tc.status = status\n}\n\n// get the last cached adjustment update status\nfunc (c *cachedConfig) getStatus() *resmgrStatus {\n\tc.RLock()\n\tdefer c.RUnlock()\n\treturn c.status\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/expression.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"fmt\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\n// Evaluable is the interface objects need to implement to be evaluable against Expressions.\ntype Evaluable interface {\n\tEval(string) interface{}\n}\n\n// Expression is used to describe a criteria to select objects within a domain.\ntype Expression struct {\n\tKey    string   `json:\"key\"`              // key to check values of/against\n\tOp     Operator `json:\"operator\"`         // operator to apply to value of Key and Values\n\tValues []string `json:\"values,omitempty\"` // value(s) for domain key\n}\n\nconst (\n\tKeyPod       = \"pod\"\n\tKeyID        = \"id\"\n\tKeyUID       = \"uid\"\n\tKeyName      = \"name\"\n\tKeyNamespace = \"namespace\"\n\tKeyQOSClass  = \"qosclass\"\n\tKeyLabels    = \"labels\"\n\tKeyTags      = \"tags\"\n)\n\n// Operator defines the possible operators for an Expression.\ntype Operator string\n\nconst (\n\t// Equals tests for equality with a single value.\n\tEquals Operator = \"Equals\"\n\t// NotEqual test for inequality with a single value.\n\tNotEqual Operator = \"NotEqual\"\n\t// In tests if the key's value is one of the specified set.\n\tIn Operator = \"In\"\n\t// NotIn tests if the key's value is not one of the specified set.\n\tNotIn Operator = \"NotIn\"\n\t// Exists evalutes to true if the named key exists.\n\tExists Operator = \"Exists\"\n\t// NotExist evalutes to true if the named key does not exist.\n\tNotExist Operator = \"NotExist\"\n\t// AlwaysTrue always evaluates to true.\n\tAlwaysTrue Operator = \"AlwaysTrue\"\n\t// Matches tests if the key value matches the only given globbing pattern.\n\tMatches Operator = \"Matches\"\n\t// MatchesNot is true if Matches would be false for the same key and pattern.\n\tMatchesNot Operator = \"MatchesNot\"\n\t// MatchesAny tests if the key value matches any of the given globbing patterns.\n\tMatchesAny Operator = \"MatchesAny\"\n\t// MatchesNone is true if MatchesAny would be false for the same key and patterns.\n\tMatchesNone Operator = \"MatchesNone\"\n)\n\n// Our logger instance.\nvar log = logger.NewLogger(\"expression\")\n\n// Validate checks the expression for (obvious) invalidity.\nfunc (e *Expression) Validate() error {\n\tif e == nil {\n\t\treturn exprError(\"nil expression\")\n\t}\n\n\tswitch e.Op {\n\tcase Equals, NotEqual:\n\t\tif len(e.Values) != 1 {\n\t\t\treturn exprError(\"invalid expression, '%s' requires a single value\", e.Op)\n\t\t}\n\tcase Matches, MatchesNot:\n\t\tif len(e.Values) != 1 {\n\t\t\treturn exprError(\"invalid expression, '%s' requires a single value\", e.Op)\n\t\t}\n\tcase Exists, NotExist:\n\t\tif e.Values != nil && len(e.Values) != 0 {\n\t\t\treturn exprError(\"invalid expression, '%s' does not take any values\", e.Op)\n\t\t}\n\n\tcase In, NotIn:\n\tcase MatchesAny, MatchesNone:\n\tcase AlwaysTrue:\n\n\tdefault:\n\t\treturn exprError(\"invalid expression, unknown operator: %q\", e.Op)\n\t}\n\treturn nil\n}\n\n// Evaluate evaluates an expression against a container.\nfunc (e *Expression) Evaluate(subject Evaluable) bool {\n\tlog.Debug(\"evaluating %q @ %s...\", *e, subject)\n\n\tif e.Op == AlwaysTrue {\n\t\treturn true\n\t}\n\n\tvalue, ok := e.KeyValue(subject)\n\tresult := false\n\n\tswitch e.Op {\n\tcase Equals:\n\t\tresult = ok && (value == e.Values[0] || e.Values[0] == \"*\")\n\tcase NotEqual:\n\t\tresult = !ok || value != e.Values[0]\n\tcase Matches, MatchesNot:\n\t\tmatch := false\n\t\tif ok {\n\t\t\tmatch, _ = filepath.Match(e.Values[0], value)\n\t\t}\n\t\tresult = ok && match\n\t\tif e.Op == MatchesNot {\n\t\t\tresult = !result\n\t\t}\n\tcase In, NotIn:\n\t\tif ok {\n\t\t\tfor _, v := range e.Values {\n\t\t\t\tif value == v || v == \"*\" {\n\t\t\t\t\tresult = true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif e.Op == NotIn {\n\t\t\tresult = !result\n\t\t}\n\tcase MatchesAny, MatchesNone:\n\t\tif ok {\n\t\t\tfor _, pattern := range e.Values {\n\t\t\t\tif match, _ := filepath.Match(pattern, value); match {\n\t\t\t\t\tresult = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif e.Op == MatchesNone {\n\t\t\tresult = !result\n\t\t}\n\tcase Exists:\n\t\tresult = ok\n\tcase NotExist:\n\t\tresult = !ok\n\t}\n\n\tlog.Debug(\"%q @ %s => %v\", *e, subject, result)\n\n\treturn result\n}\n\n// KeyValue extracts the value of the expresssion key from a container.\nfunc (e *Expression) KeyValue(subject Evaluable) (string, bool) {\n\tlog.Debug(\"looking up %q @ %s...\", e.Key, subject)\n\n\tvalue := \"\"\n\tok := false\n\n\tkeys, vsep := splitKeys(e.Key)\n\tif len(keys) == 1 {\n\t\tvalue, ok, _ = ResolveRef(subject, keys[0])\n\t} else {\n\t\tvals := make([]string, 0, len(keys))\n\t\tfor _, key := range keys {\n\t\t\tv, found, _ := ResolveRef(subject, key)\n\t\t\tvals = append(vals, v)\n\t\t\tok = ok || found\n\t\t}\n\t\tvalue = strings.Join(vals, vsep)\n\t}\n\n\tlog.Debug(\"%q @ %s => %q, %v\", e.Key, subject, value, ok)\n\n\treturn value, ok\n}\n\nfunc splitKeys(keys string) ([]string, string) {\n\t// joint key specs have two valid forms:\n\t//   - \":keylist\" (equivalent to \":::<colon-separated-keylist>\")\n\t//   - \":<ksep><vsep><ksep-separated-keylist>\"\n\n\tif len(keys) < 4 || keys[0] != ':' {\n\t\treturn []string{keys}, \"\"\n\t}\n\n\tkeys = keys[1:]\n\tksep := keys[0:1]\n\tvsep := keys[1:2]\n\n\tif validSeparator(ksep[0]) && validSeparator(vsep[0]) {\n\t\tkeys = keys[2:]\n\t} else {\n\t\tksep = \":\"\n\t\tvsep = \":\"\n\t}\n\n\treturn strings.Split(keys, ksep), vsep\n}\n\nfunc validSeparator(b byte) bool {\n\tswitch {\n\tcase '0' <= b && b <= '9':\n\t\treturn false\n\tcase 'a' <= b && b <= 'z':\n\t\treturn false\n\tcase 'A' <= b && b <= 'Z':\n\t\treturn false\n\tcase b == '/', b == '.':\n\t\treturn false\n\t}\n\treturn true\n}\n\n// ResolveRef walks an object trying to resolve a reference to a value.\nfunc ResolveRef(subject Evaluable, spec string) (string, bool, error) {\n\tvar obj interface{}\n\n\tlog.Debug(\"resolving %q @ %s...\", spec, subject)\n\n\tspec = path.Clean(spec)\n\tref := strings.Split(spec, \"/\")\n\tif len(ref) == 1 {\n\t\tif strings.Index(spec, \".\") != -1 {\n\t\t\tref = []string{\"labels\", spec}\n\t\t}\n\t}\n\n\tobj = subject\n\tfor len(ref) > 0 {\n\t\tkey := ref[0]\n\n\t\tlog.Debug(\"resolve walking %q @ %s...\", key, obj)\n\t\tswitch v := obj.(type) {\n\t\tcase string:\n\t\t\tobj = v\n\t\tcase map[string]string:\n\t\t\tvalue, ok := v[key]\n\t\t\tif !ok {\n\t\t\t\treturn \"\", false, nil\n\t\t\t}\n\t\t\tobj = value\n\t\tcase error:\n\t\t\treturn \"\", false, exprError(\"%s: failed to resolve %q: %v\", subject, spec, v)\n\t\tdefault:\n\t\t\te, ok := obj.(Evaluable)\n\t\t\tif !ok {\n\t\t\t\treturn \"\", false, exprError(\"%s: failed to resolve %q, unexpected type %T\",\n\t\t\t\t\tsubject, spec, obj)\n\t\t\t}\n\t\t\tobj = e.Eval(key)\n\t\t}\n\n\t\tref = ref[1:]\n\t}\n\n\tstr, ok := obj.(string)\n\tif !ok {\n\t\treturn \"\", false, exprError(\"%s: reference %q resolved to non-string: %T\",\n\t\t\tsubject, spec, obj)\n\t}\n\n\tlog.Debug(\"resolved %q @ %s => %s\", spec, subject, str)\n\n\treturn str, true, nil\n}\n\n// String returns the expression as a string.\nfunc (e *Expression) String() string {\n\treturn fmt.Sprintf(\"<%s %s %s>\", e.Key, e.Op, strings.Join(e.Values, \",\"))\n}\n\n// DeepCopy creates a deep copy of the expression.\nfunc (e *Expression) DeepCopy() *Expression {\n\tout := &Expression{}\n\te.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto copies the expression into another one.\nfunc (e *Expression) DeepCopyInto(out *Expression) {\n\tout.Key = e.Key\n\tout.Op = e.Op\n\tout.Values = make([]string, len(e.Values))\n\tcopy(out.Values, e.Values)\n}\n\n// exprError returns a formatted error specific to expressions.\nfunc exprError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"expression: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/expression_test.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\ntype evaluable struct {\n\tname      string\n\tnamespace string\n\tqosclass  string\n\tlabels    map[string]string\n\ttags      map[string]string\n\tparent    Evaluable\n}\n\nfunc newEvaluable(name, ns, qos string, labels, tags map[string]string, p Evaluable) *evaluable {\n\treturn &evaluable{\n\t\tname:      name,\n\t\tnamespace: ns,\n\t\tqosclass:  qos,\n\t\tlabels:    labels,\n\t\ttags:      tags,\n\t\tparent:    p,\n\t}\n}\n\nfunc (e *evaluable) Eval(key string) interface{} {\n\tswitch key {\n\tcase KeyName:\n\t\treturn e.name\n\tcase KeyNamespace:\n\t\treturn e.namespace\n\tcase KeyQOSClass:\n\t\treturn e.qosclass\n\tcase KeyLabels:\n\t\treturn e.labels\n\tcase KeyTags:\n\t\treturn e.tags\n\tcase KeyPod:\n\t\tif e.parent != nil {\n\t\t\treturn e.parent\n\t\t}\n\t\tfallthrough\n\tdefault:\n\t\treturn fmt.Errorf(\"evaluable: cannot evaluate %q\", key)\n\t}\n}\n\nfunc (e *evaluable) String() string {\n\ts := fmt.Sprintf(\"{ name: %q, namespace: %q, qosclass: %q, \", e.name, e.namespace, e.qosclass)\n\tlabels, t := \"{\", \"\"\n\tfor k, v := range e.labels {\n\t\tlabels += t + fmt.Sprintf(\"%q:%q\", k, v)\n\t\tt = \", \"\n\t}\n\tlabels += \"}\"\n\ttags, t := \"{\", \"\"\n\tfor k, v := range e.tags {\n\t\ttags += t + fmt.Sprintf(\"%q:%q\", k, v)\n\t\tt = \", \"\n\t}\n\ttags += \"}\"\n\ts = fmt.Sprintf(\"%s, labels: %s, tags: %s }\", s, labels, tags)\n\treturn s\n}\n\nfunc TestResolveRefAndKeyValue(t *testing.T) {\n\tdefer logger.Flush()\n\n\tpod := newEvaluable(\"P1\", \"pns\", \"pqos\",\n\t\tmap[string]string{\"l1\": \"plone\", \"l2\": \"pltwo\", \"l5\": \"plfive\"}, nil, nil)\n\n\ttcases := []struct {\n\t\tname      string\n\t\tsubject   Evaluable\n\t\tkeys      []string\n\t\tvalues    []string\n\t\tok        []bool\n\t\terror     []bool\n\t\tkeyvalues []string\n\t}{\n\t\t{\n\t\t\tname: \"test resolving references\",\n\t\t\tsubject: newEvaluable(\"C1\", \"cns\", \"cqos\",\n\t\t\t\tmap[string]string{\"l1\": \"clone\", \"l2\": \"cltwo\", \"l3\": \"clthree\"},\n\t\t\t\tmap[string]string{\"t1\": \"ctone\", \"t2\": \"cttwo\", \"t3\": \"ctthree\"}, pod),\n\t\t\tkeys: []string{\n\t\t\t\t\"name\", \"namespace\", \"qosclass\",\n\t\t\t\t\"labels/l1\", \"labels/l2\", \"labels/l3\", \"labels/l4\",\n\t\t\t\t\"tags/t1\", \"tags/t2\", \"tags/t3\", \"tags/t4\",\n\t\t\t\t\"pod/labels/l1\",\n\t\t\t\t\"pod/labels/l2\",\n\t\t\t\t\"pod/labels/l3\",\n\t\t\t\t\"pod/labels/l4\",\n\t\t\t\t\"pod/labels/l5\",\n\t\t\t\t\":,-pod/qosclass,pod/namespace,pod/name,name\",\n\t\t\t},\n\t\t\tvalues: []string{\n\t\t\t\t\"C1\", \"cns\", \"cqos\",\n\t\t\t\t\"clone\", \"cltwo\", \"clthree\", \"\",\n\t\t\t\t\"ctone\", \"cttwo\", \"ctthree\", \"\",\n\t\t\t\t\"plone\", \"pltwo\", \"\", \"\", \"plfive\",\n\t\t\t\t\"\",\n\t\t\t},\n\t\t\tkeyvalues: []string{\n\t\t\t\t\"C1\", \"cns\", \"cqos\",\n\t\t\t\t\"clone\", \"cltwo\", \"clthree\", \"\",\n\t\t\t\t\"ctone\", \"cttwo\", \"ctthree\", \"\",\n\t\t\t\t\"plone\", \"pltwo\", \"\", \"\", \"plfive\",\n\t\t\t\t\"pqos-pns-P1-C1\",\n\t\t\t},\n\t\t\tok: []bool{\n\t\t\t\ttrue, true, true,\n\t\t\t\ttrue, true, true, false,\n\t\t\t\ttrue, true, true, false,\n\t\t\t\ttrue, true, false, false, true,\n\t\t\t\tfalse,\n\t\t\t},\n\t\t\terror: []bool{\n\t\t\t\tfalse, false, false,\n\t\t\t\tfalse, false, false, false,\n\t\t\t\tfalse, false, false, false,\n\t\t\t\tfalse, false, false, false, false,\n\t\t\t\ttrue,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfor i := range tc.keys {\n\t\t\t\tvalue, ok, err := ResolveRef(tc.subject, tc.keys[i])\n\t\t\t\tif err != nil && !tc.error[i] {\n\t\t\t\t\tt.Errorf(\"ResolveRef %s/%q should have given %q, but failed: %v\",\n\t\t\t\t\t\ttc.subject, tc.keys[i], tc.values[i], err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif value != tc.values[i] || ok != tc.ok[i] {\n\t\t\t\t\tt.Errorf(\"ResolveRef %s@%q: expected %v, %v got %v, %v\",\n\t\t\t\t\t\ttc.subject, tc.keys[i], tc.values[i], tc.ok[i], value, ok)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\texpr := &Expression{\n\t\t\t\t\tKey:    tc.keys[i],\n\t\t\t\t\tOp:     Equals,\n\t\t\t\t\tValues: []string{},\n\t\t\t\t}\n\t\t\t\tvalue, _ = expr.KeyValue(tc.subject)\n\t\t\t\tif value != tc.keyvalues[i] {\n\t\t\t\t\tt.Errorf(\"KeyValue %s@%q: expected %v, got %v\",\n\t\t\t\t\t\ttc.subject, tc.keys[i], tc.keyvalues[i], value)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestSimpleOperators(t *testing.T) {\n\tdefer logger.Flush()\n\n\tpod := newEvaluable(\"P1\", \"pns\", \"pqos\",\n\t\tmap[string]string{\"l1\": \"plone\", \"l2\": \"pltwo\", \"l5\": \"plfive\"},\n\t\tnil,\n\t\tnil)\n\tsub := newEvaluable(\"C1\", \"cns\", \"cqos\",\n\t\tmap[string]string{\"l1\": \"clone\", \"l2\": \"cltwo\", \"l3\": \"clthree\"},\n\t\tmap[string]string{\"t1\": \"ctone\", \"t2\": \"cttwo\", \"t4\": \"ctfour\"},\n\t\tpod)\n\n\ttcases := []struct {\n\t\tname    string\n\t\tsubject Evaluable\n\t\tkeys    []string\n\t\tops     []Operator\n\t\tvalues  [][][]string\n\t\tresults [][]bool\n\t}{\n\t\t{\n\t\t\tname:    \"test Equals, NotEqual, In, NotIn operators\",\n\t\t\tsubject: sub,\n\t\t\tkeys: []string{\n\t\t\t\t\"name\",\n\t\t\t\t\"pod/name\",\n\t\t\t\t\"namespace\",\n\t\t\t\t\"pod/namespace\",\n\t\t\t\t\"qosclass\",\n\t\t\t\t\"pod/qosclass\",\n\t\t\t\t\"labels/l1\",\n\t\t\t\t\"labels/l2\",\n\t\t\t\t\"labels/l3\",\n\t\t\t\t\"labels/l4\",\n\t\t\t\t\"tags/t1\",\n\t\t\t\t\"tags/t2\",\n\t\t\t\t\"tags/t3\",\n\t\t\t\t\"tags/t4\",\n\t\t\t\t\"pod/labels/l1\",\n\t\t\t\t\"pod/labels/l2\",\n\t\t\t\t\"pod/labels/l3\",\n\t\t\t\t\"pod/labels/l4\",\n\t\t\t\t\"pod/labels/l5\",\n\t\t\t},\n\t\t\tops: []Operator{Equals, NotEqual, In, NotIn},\n\t\t\tvalues: [][][]string{\n\t\t\t\t{{\"C1\"}, {\"C1\"}, {\"foo\", \"C1\"}, {\"foo\"}},                    // name\n\t\t\t\t{{\"P1\"}, {\"P1\"}, {\"foo\", \"P1\"}, {\"foo\"}},                    // pod/name\n\t\t\t\t{{\"cns\"}, {\"cns\"}, {\"foo\", \"cns\"}, {\"foo\"}},                 // namespace\n\t\t\t\t{{\"pns\"}, {\"pns\"}, {\"foo\", \"pns\"}, {\"pns\"}},                 // pod/namespace\n\t\t\t\t{{\"cqos\"}, {\"cqos\"}, {\"foo\", \"cqos\"}, {\"foo\"}},              // qosclass\n\t\t\t\t{{\"pqos\"}, {\"pqos\"}, {\"foo\", \"pqos\"}, {\"pqos\"}},             // pod/qosclass\n\t\t\t\t{{\"clone\"}, {\"clone\"}, {\"foo\", \"clone\"}, {\"foo\"}},           // labels/l1\n\t\t\t\t{{\"cltwo\"}, {\"cltwo\"}, {\"foo\", \"cltwo\"}, {\"foo\"}},           // labels/l2\n\t\t\t\t{{\"clthree\"}, {\"clthree\"}, {\"foo\", \"clthree\"}, {\"clthree\"}}, // labels/l3\n\t\t\t\t{{\"clfour\"}, {\"clfour\"}, {\"foo\", \"clfour\"}, {\"foo\"}},        // labels/l4\n\t\t\t\t{{\"ctone\"}, {\"ctone\"}, {\"foo\", \"ctone\"}, {\"foo\"}},           // tags/t1\n\t\t\t\t{{\"cttwo\"}, {\"cttwo\"}, {\"foo\", \"cttwo\"}, {\"foo\"}},           // tags/t2\n\t\t\t\t{{\"ctthree\"}, {\"ctthree\"}, {\"foo\", \"ctthree\"}, {\"foo\"}},     // tags/t3\n\t\t\t\t{{\"ctfour\"}, {\"ctfour\"}, {\"foo\", \"ctfour\"}, {\"ctfour\"}},     // tags/t4\n\t\t\t\t{{\"plone\"}, {\"plone\"}, {\"foo\", \"plone\"}, {\"foo\"}},           // pod/labels/l1\n\t\t\t\t{{\"pltwo\"}, {\"pltwo\"}, {\"foo\", \"pltwo\"}, {\"foo\"}},           // pod/labels/l2\n\t\t\t\t{{\"plthree\"}, {\"plthree\"}, {\"foo\", \"plthree\"}, {\"foo\"}},     // pod/labels/l3\n\t\t\t\t{{\"plfour\"}, {\"plfour\"}, {\"foo\", \"plfour\"}, {\"foo\"}},        // pod/labels/l4\n\t\t\t\t{{\"plfive\"}, {\"plfive\"}, {\"foo\", \"plfive\"}, {\"foo\"}},        // pod/labels/l5\n\t\t\t},\n\t\t\tresults: [][]bool{\n\t\t\t\t{true, false, true, true},  // name\n\t\t\t\t{true, false, true, true},  // pod/name\n\t\t\t\t{true, false, true, true},  // namespace\n\t\t\t\t{true, false, true, false}, // pod/namespace\n\t\t\t\t{true, false, true, true},  // qosclass\n\t\t\t\t{true, false, true, false}, // pod/qosclass\n\t\t\t\t{true, false, true, true},  // labels/l1\n\t\t\t\t{true, false, true, true},  // labels/l2\n\t\t\t\t{true, false, true, false}, // labels/l3\n\t\t\t\t{false, true, false, true}, // labels/l4\n\t\t\t\t{true, false, true, true},  // tags/t1\n\t\t\t\t{true, false, true, true},  // tags/t2\n\t\t\t\t{false, true, false, true}, // tags/t3\n\t\t\t\t{true, false, true, false}, // tags/t4\n\t\t\t\t{true, false, true, true},  // pod/labels/l1\n\t\t\t\t{true, false, true, true},  // pod/labels/l2\n\t\t\t\t{false, true, false, true}, // pod/labels/l3\n\t\t\t\t{false, true, false, true}, // pod/labels/l4\n\t\t\t\t{true, false, true, true},  // pod/labels/l5\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfor k := range tc.keys {\n\t\t\t\tfor o := range tc.ops {\n\t\t\t\t\texpr := &Expression{\n\t\t\t\t\t\tKey:    tc.keys[k],\n\t\t\t\t\t\tOp:     tc.ops[o],\n\t\t\t\t\t\tValues: tc.values[k][o],\n\t\t\t\t\t}\n\t\t\t\t\texpect := tc.results[k][o]\n\t\t\t\t\tresult := expr.Evaluate(tc.subject)\n\t\t\t\t\tif result != expect {\n\t\t\t\t\t\tt.Errorf(\"%s for %s: expected %v, got %v\", expr, tc.subject, expect, result)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMatching(t *testing.T) {\n\tdefer logger.Flush()\n\n\tp1 := newEvaluable(\"P1\", \"pns1\", \"pqos1\",\n\t\tmap[string]string{\"l1\": \"plv1\", \"l2\": \"plv2\", \"l5\": \"plv5\"},\n\t\tnil,\n\t\tnil)\n\tc11 := newEvaluable(\"C11\", \"cns1\", \"cqos11\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"tag2\", \"t4\": \"ctv4\"},\n\t\tp1)\n\tc12 := newEvaluable(\"C12\", \"cns1\", \"cqos12\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"foo\", \"t4\": \"ctv4\"},\n\t\tp1)\n\tc13 := newEvaluable(\"C12\", \"cns1\", \"cqos13\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"ctv2\", \"t4\": \"ctv4\"},\n\t\tp1)\n\n\tp2 := newEvaluable(\"P2\", \"pns2\", \"pqos2\",\n\t\tmap[string]string{\"l1\": \"plv1\", \"l2\": \"plv2\", \"l5\": \"plv5\"},\n\t\tnil,\n\t\tnil)\n\tc21 := newEvaluable(\"C21\", \"cns1\", \"cqos21\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"tag2\", \"t4\": \"ctv4\"},\n\t\tp2)\n\tc22 := newEvaluable(\"C22\", \"cns1\", \"cqos22\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"ctv2\", \"t4\": \"ctv4\"},\n\t\tp2)\n\tc23 := newEvaluable(\"C23\", \"cns1\", \"cqos23\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"foo\", \"t4\": \"ctv4\"},\n\t\tp2)\n\n\tp3 := newEvaluable(\"P3\", \"pns3\", \"pqos3\",\n\t\tmap[string]string{\"l1\": \"plv1\", \"l2\": \"plv2\", \"l5\": \"plv5\"},\n\t\tnil,\n\t\tnil)\n\tc3 := newEvaluable(\"C3\", \"cns3\", \"cqos3\",\n\t\tmap[string]string{\"l1\": \"clv1\", \"l2\": \"clv2\", \"l3\": \"clv3\"},\n\t\tmap[string]string{\"t1\": \"ctv1\", \"t2\": \"tag2\", \"t4\": \"ctv4\"},\n\t\tp3)\n\n\ttcases := []struct {\n\t\tname      string\n\t\tsubjects  []Evaluable\n\t\tselectors []*Expression\n\t\texpected  [][]string\n\t}{\n\t\t{\n\t\t\tname:     \"test inverted membership operator\",\n\t\t\tsubjects: []Evaluable{c11, c12, c13, c21, c22, c23, c3},\n\t\t\tselectors: []*Expression{\n\t\t\t\t{\n\t\t\t\t\tKey: \":,:pod/qosclass,pod/namespace,pod/name,qosclass,name\",\n\t\t\t\t\tOp:  Matches,\n\t\t\t\t\tValues: []string{\n\t\t\t\t\t\t\"pqos2:*:*:*:*\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tKey:    \"tags/t2\",\n\t\t\t\t\tOp:     Matches,\n\t\t\t\t\tValues: []string{\"[tf][ao][go]*\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: [][]string{\n\t\t\t\t{\"C21\", \"C22\", \"C23\"},\n\t\t\t\t{\"C11\", \"C12\", \"C21\", \"C23\", \"C3\"},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tfor i, expr := range tc.selectors {\n\t\t\t\tresults := []string{}\n\t\t\t\tfor _, s := range tc.subjects {\n\t\t\t\t\tif expr.Evaluate(s) {\n\t\t\t\t\t\tresults = append(results, s.Eval(\"name\").(string))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpected := strings.Join(tc.expected[i], \",\")\n\t\t\t\tgot := strings.Join(results, \",\")\n\t\t\t\tif expected != got {\n\t\t\t\t\tt.Errorf(\"%s: expected %s, got %s\", expr, expected, got)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/clientset.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage versioned\n\nimport (\n\t\"fmt\"\n\n\tcriresmgrv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1\"\n\tdiscovery \"k8s.io/client-go/discovery\"\n\trest \"k8s.io/client-go/rest\"\n\tflowcontrol \"k8s.io/client-go/util/flowcontrol\"\n)\n\ntype Interface interface {\n\tDiscovery() discovery.DiscoveryInterface\n\tCriresmgrV1alpha1() criresmgrv1alpha1.CriresmgrV1alpha1Interface\n}\n\n// Clientset contains the clients for groups. Each group has exactly one\n// version included in a Clientset.\ntype Clientset struct {\n\t*discovery.DiscoveryClient\n\tcriresmgrV1alpha1 *criresmgrv1alpha1.CriresmgrV1alpha1Client\n}\n\n// CriresmgrV1alpha1 retrieves the CriresmgrV1alpha1Client\nfunc (c *Clientset) CriresmgrV1alpha1() criresmgrv1alpha1.CriresmgrV1alpha1Interface {\n\treturn c.criresmgrV1alpha1\n}\n\n// Discovery retrieves the DiscoveryClient\nfunc (c *Clientset) Discovery() discovery.DiscoveryInterface {\n\tif c == nil {\n\t\treturn nil\n\t}\n\treturn c.DiscoveryClient\n}\n\n// NewForConfig creates a new Clientset for the given config.\n// If config's RateLimiter is not set and QPS and Burst are acceptable,\n// NewForConfig will generate a rate-limiter in configShallowCopy.\nfunc NewForConfig(c *rest.Config) (*Clientset, error) {\n\tconfigShallowCopy := *c\n\tif configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {\n\t\tif configShallowCopy.Burst <= 0 {\n\t\t\treturn nil, fmt.Errorf(\"Burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0\")\n\t\t}\n\t\tconfigShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)\n\t}\n\tvar cs Clientset\n\tvar err error\n\tcs.criresmgrV1alpha1, err = criresmgrv1alpha1.NewForConfig(&configShallowCopy)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &cs, nil\n}\n\n// NewForConfigOrDie creates a new Clientset for the given config and\n// panics if there is an error in the config.\nfunc NewForConfigOrDie(c *rest.Config) *Clientset {\n\tvar cs Clientset\n\tcs.criresmgrV1alpha1 = criresmgrv1alpha1.NewForConfigOrDie(c)\n\n\tcs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c)\n\treturn &cs\n}\n\n// New creates a new Clientset for the given RESTClient.\nfunc New(c rest.Interface) *Clientset {\n\tvar cs Clientset\n\tcs.criresmgrV1alpha1 = criresmgrv1alpha1.New(c)\n\n\tcs.DiscoveryClient = discovery.NewDiscoveryClient(c)\n\treturn &cs\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\n// This package has the automatically generated clientset.\npackage versioned\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/fake/clientset_generated.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage fake\n\nimport (\n\tclientset \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned\"\n\tcriresmgrv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1\"\n\tfakecriresmgrv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/fake\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\t\"k8s.io/apimachinery/pkg/watch\"\n\t\"k8s.io/client-go/discovery\"\n\tfakediscovery \"k8s.io/client-go/discovery/fake\"\n\t\"k8s.io/client-go/testing\"\n)\n\n// NewSimpleClientset returns a clientset that will respond with the provided objects.\n// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,\n// without applying any validations and/or defaults. It shouldn't be considered a replacement\n// for a real clientset and is mostly useful in simple unit tests.\nfunc NewSimpleClientset(objects ...runtime.Object) *Clientset {\n\to := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())\n\tfor _, obj := range objects {\n\t\tif err := o.Add(obj); err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\n\tcs := &Clientset{tracker: o}\n\tcs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}\n\tcs.AddReactor(\"*\", \"*\", testing.ObjectReaction(o))\n\tcs.AddWatchReactor(\"*\", func(action testing.Action) (handled bool, ret watch.Interface, err error) {\n\t\tgvr := action.GetResource()\n\t\tns := action.GetNamespace()\n\t\twatch, err := o.Watch(gvr, ns)\n\t\tif err != nil {\n\t\t\treturn false, nil, err\n\t\t}\n\t\treturn true, watch, nil\n\t})\n\n\treturn cs\n}\n\n// Clientset implements clientset.Interface. Meant to be embedded into a\n// struct to get a default implementation. This makes faking out just the method\n// you want to test easier.\ntype Clientset struct {\n\ttesting.Fake\n\tdiscovery *fakediscovery.FakeDiscovery\n\ttracker   testing.ObjectTracker\n}\n\nfunc (c *Clientset) Discovery() discovery.DiscoveryInterface {\n\treturn c.discovery\n}\n\nfunc (c *Clientset) Tracker() testing.ObjectTracker {\n\treturn c.tracker\n}\n\nvar _ clientset.Interface = &Clientset{}\n\n// CriresmgrV1alpha1 retrieves the CriresmgrV1alpha1Client\nfunc (c *Clientset) CriresmgrV1alpha1() criresmgrv1alpha1.CriresmgrV1alpha1Interface {\n\treturn &fakecriresmgrv1alpha1.FakeCriresmgrV1alpha1{Fake: &c.Fake}\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/fake/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\n// This package has the automatically generated fake clientset.\npackage fake\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/fake/register.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage fake\n\nimport (\n\tcriresmgrv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n\tschema \"k8s.io/apimachinery/pkg/runtime/schema\"\n\tserializer \"k8s.io/apimachinery/pkg/runtime/serializer\"\n\tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n)\n\nvar scheme = runtime.NewScheme()\nvar codecs = serializer.NewCodecFactory(scheme)\nvar parameterCodec = runtime.NewParameterCodec(scheme)\nvar localSchemeBuilder = runtime.SchemeBuilder{\n\tcriresmgrv1alpha1.AddToScheme,\n}\n\n// AddToScheme adds all types of this clientset into the given scheme. This allows composition\n// of clientsets, like in:\n//\n//\timport (\n//\t  \"k8s.io/client-go/kubernetes\"\n//\t  clientsetscheme \"k8s.io/client-go/kubernetes/scheme\"\n//\t  aggregatorclientsetscheme \"k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme\"\n//\t)\n//\n//\tkclientset, _ := kubernetes.NewForConfig(c)\n//\t_ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)\n//\n// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types\n// correctly.\nvar AddToScheme = localSchemeBuilder.AddToScheme\n\nfunc init() {\n\tv1.AddToGroupVersion(scheme, schema.GroupVersion{Version: \"v1\"})\n\tutilruntime.Must(AddToScheme(scheme))\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/scheme/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\n// This package contains the scheme of the automatically generated clientset.\npackage scheme\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/scheme/register.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage scheme\n\nimport (\n\tcriresmgrv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n\tschema \"k8s.io/apimachinery/pkg/runtime/schema\"\n\tserializer \"k8s.io/apimachinery/pkg/runtime/serializer\"\n\tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n)\n\nvar Scheme = runtime.NewScheme()\nvar Codecs = serializer.NewCodecFactory(Scheme)\nvar ParameterCodec = runtime.NewParameterCodec(Scheme)\nvar localSchemeBuilder = runtime.SchemeBuilder{\n\tcriresmgrv1alpha1.AddToScheme,\n}\n\n// AddToScheme adds all types of this clientset into the given scheme. This allows composition\n// of clientsets, like in:\n//\n//\timport (\n//\t  \"k8s.io/client-go/kubernetes\"\n//\t  clientsetscheme \"k8s.io/client-go/kubernetes/scheme\"\n//\t  aggregatorclientsetscheme \"k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme\"\n//\t)\n//\n//\tkclientset, _ := kubernetes.NewForConfig(c)\n//\t_ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)\n//\n// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types\n// correctly.\nvar AddToScheme = localSchemeBuilder.AddToScheme\n\nfunc init() {\n\tv1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: \"v1\"})\n\tutilruntime.Must(AddToScheme(Scheme))\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/adjustment.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\tscheme \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/scheme\"\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\ttypes \"k8s.io/apimachinery/pkg/types\"\n\twatch \"k8s.io/apimachinery/pkg/watch\"\n\trest \"k8s.io/client-go/rest\"\n)\n\n// AdjustmentsGetter has a method to return a AdjustmentInterface.\n// A group's client should implement this interface.\ntype AdjustmentsGetter interface {\n\tAdjustments(namespace string) AdjustmentInterface\n}\n\n// AdjustmentInterface has methods to work with Adjustment resources.\ntype AdjustmentInterface interface {\n\tCreate(*v1alpha1.Adjustment) (*v1alpha1.Adjustment, error)\n\tUpdate(*v1alpha1.Adjustment) (*v1alpha1.Adjustment, error)\n\tUpdateStatus(*v1alpha1.Adjustment) (*v1alpha1.Adjustment, error)\n\tDelete(name string, options *v1.DeleteOptions) error\n\tDeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error\n\tGet(name string, options v1.GetOptions) (*v1alpha1.Adjustment, error)\n\tList(opts v1.ListOptions) (*v1alpha1.AdjustmentList, error)\n\tWatch(opts v1.ListOptions) (watch.Interface, error)\n\tPatch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Adjustment, err error)\n\tAdjustmentExpansion\n}\n\n// adjustments implements AdjustmentInterface\ntype adjustments struct {\n\tclient rest.Interface\n\tns     string\n}\n\n// newAdjustments returns a Adjustments\nfunc newAdjustments(c *CriresmgrV1alpha1Client, namespace string) *adjustments {\n\treturn &adjustments{\n\t\tclient: c.RESTClient(),\n\t\tns:     namespace,\n\t}\n}\n\n// Get takes name of the adjustment, and returns the corresponding adjustment object, and an error if there is any.\nfunc (c *adjustments) Get(name string, options v1.GetOptions) (result *v1alpha1.Adjustment, err error) {\n\tresult = &v1alpha1.Adjustment{}\n\terr = c.client.Get().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tName(name).\n\t\tVersionedParams(&options, scheme.ParameterCodec).\n\t\tDo(context.TODO()).\n\t\tInto(result)\n\treturn\n}\n\n// List takes label and field selectors, and returns the list of Adjustments that match those selectors.\nfunc (c *adjustments) List(opts v1.ListOptions) (result *v1alpha1.AdjustmentList, err error) {\n\tvar timeout time.Duration\n\tif opts.TimeoutSeconds != nil {\n\t\ttimeout = time.Duration(*opts.TimeoutSeconds) * time.Second\n\t}\n\tresult = &v1alpha1.AdjustmentList{}\n\terr = c.client.Get().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tVersionedParams(&opts, scheme.ParameterCodec).\n\t\tTimeout(timeout).\n\t\tDo(context.TODO()).\n\t\tInto(result)\n\treturn\n}\n\n// Watch returns a watch.Interface that watches the requested adjustments.\nfunc (c *adjustments) Watch(opts v1.ListOptions) (watch.Interface, error) {\n\tvar timeout time.Duration\n\tif opts.TimeoutSeconds != nil {\n\t\ttimeout = time.Duration(*opts.TimeoutSeconds) * time.Second\n\t}\n\topts.Watch = true\n\treturn c.client.Get().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tVersionedParams(&opts, scheme.ParameterCodec).\n\t\tTimeout(timeout).\n\t\tWatch(context.TODO())\n}\n\n// Create takes the representation of a adjustment and creates it.  Returns the server's representation of the adjustment, and an error, if there is any.\nfunc (c *adjustments) Create(adjustment *v1alpha1.Adjustment) (result *v1alpha1.Adjustment, err error) {\n\tresult = &v1alpha1.Adjustment{}\n\terr = c.client.Post().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tBody(adjustment).\n\t\tDo(context.TODO()).\n\t\tInto(result)\n\treturn\n}\n\n// Update takes the representation of a adjustment and updates it. Returns the server's representation of the adjustment, and an error, if there is any.\nfunc (c *adjustments) Update(adjustment *v1alpha1.Adjustment) (result *v1alpha1.Adjustment, err error) {\n\tresult = &v1alpha1.Adjustment{}\n\terr = c.client.Put().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tName(adjustment.Name).\n\t\tBody(adjustment).\n\t\tDo(context.TODO()).\n\t\tInto(result)\n\treturn\n}\n\n// UpdateStatus was generated because the type contains a Status member.\n// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().\n\nfunc (c *adjustments) UpdateStatus(adjustment *v1alpha1.Adjustment) (result *v1alpha1.Adjustment, err error) {\n\tresult = &v1alpha1.Adjustment{}\n\terr = c.client.Put().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tName(adjustment.Name).\n\t\tSubResource(\"status\").\n\t\tBody(adjustment).\n\t\tDo(context.TODO()).\n\t\tInto(result)\n\treturn\n}\n\n// Delete takes name of the adjustment and deletes it. Returns an error if one occurs.\nfunc (c *adjustments) Delete(name string, options *v1.DeleteOptions) error {\n\treturn c.client.Delete().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tName(name).\n\t\tBody(options).\n\t\tDo(context.TODO()).\n\t\tError()\n}\n\n// DeleteCollection deletes a collection of objects.\nfunc (c *adjustments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {\n\tvar timeout time.Duration\n\tif listOptions.TimeoutSeconds != nil {\n\t\ttimeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second\n\t}\n\treturn c.client.Delete().\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tVersionedParams(&listOptions, scheme.ParameterCodec).\n\t\tTimeout(timeout).\n\t\tBody(options).\n\t\tDo(context.TODO()).\n\t\tError()\n}\n\n// Patch applies the patch and returns the patched adjustment.\nfunc (c *adjustments) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Adjustment, err error) {\n\tresult = &v1alpha1.Adjustment{}\n\terr = c.client.Patch(pt).\n\t\tNamespace(c.ns).\n\t\tResource(\"adjustments\").\n\t\tSubResource(subresources...).\n\t\tName(name).\n\t\tBody(data).\n\t\tDo(context.TODO()).\n\t\tInto(result)\n\treturn\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\n// This package has the automatically generated typed clients.\npackage v1alpha1\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/fake/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\n// Package fake has the automatically generated clients.\npackage fake\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/fake/fake_adjustment.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage fake\n\nimport (\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tlabels \"k8s.io/apimachinery/pkg/labels\"\n\tschema \"k8s.io/apimachinery/pkg/runtime/schema\"\n\ttypes \"k8s.io/apimachinery/pkg/types\"\n\twatch \"k8s.io/apimachinery/pkg/watch\"\n\ttesting \"k8s.io/client-go/testing\"\n)\n\n// FakeAdjustments implements AdjustmentInterface\ntype FakeAdjustments struct {\n\tFake *FakeCriresmgrV1alpha1\n\tns   string\n}\n\nvar adjustmentsResource = schema.GroupVersionResource{Group: \"criresmgr.intel.com\", Version: \"v1alpha1\", Resource: \"adjustments\"}\n\nvar adjustmentsKind = schema.GroupVersionKind{Group: \"criresmgr.intel.com\", Version: \"v1alpha1\", Kind: \"Adjustment\"}\n\n// Get takes name of the adjustment, and returns the corresponding adjustment object, and an error if there is any.\nfunc (c *FakeAdjustments) Get(name string, options v1.GetOptions) (result *v1alpha1.Adjustment, err error) {\n\tobj, err := c.Fake.\n\t\tInvokes(testing.NewGetAction(adjustmentsResource, c.ns, name), &v1alpha1.Adjustment{})\n\n\tif obj == nil {\n\t\treturn nil, err\n\t}\n\treturn obj.(*v1alpha1.Adjustment), err\n}\n\n// List takes label and field selectors, and returns the list of Adjustments that match those selectors.\nfunc (c *FakeAdjustments) List(opts v1.ListOptions) (result *v1alpha1.AdjustmentList, err error) {\n\tobj, err := c.Fake.\n\t\tInvokes(testing.NewListAction(adjustmentsResource, adjustmentsKind, c.ns, opts), &v1alpha1.AdjustmentList{})\n\n\tif obj == nil {\n\t\treturn nil, err\n\t}\n\n\tlabel, _, _ := testing.ExtractFromListOptions(opts)\n\tif label == nil {\n\t\tlabel = labels.Everything()\n\t}\n\tlist := &v1alpha1.AdjustmentList{ListMeta: obj.(*v1alpha1.AdjustmentList).ListMeta}\n\tfor _, item := range obj.(*v1alpha1.AdjustmentList).Items {\n\t\tif label.Matches(labels.Set(item.Labels)) {\n\t\t\tlist.Items = append(list.Items, item)\n\t\t}\n\t}\n\treturn list, err\n}\n\n// Watch returns a watch.Interface that watches the requested adjustments.\nfunc (c *FakeAdjustments) Watch(opts v1.ListOptions) (watch.Interface, error) {\n\treturn c.Fake.\n\t\tInvokesWatch(testing.NewWatchAction(adjustmentsResource, c.ns, opts))\n\n}\n\n// Create takes the representation of a adjustment and creates it.  Returns the server's representation of the adjustment, and an error, if there is any.\nfunc (c *FakeAdjustments) Create(adjustment *v1alpha1.Adjustment) (result *v1alpha1.Adjustment, err error) {\n\tobj, err := c.Fake.\n\t\tInvokes(testing.NewCreateAction(adjustmentsResource, c.ns, adjustment), &v1alpha1.Adjustment{})\n\n\tif obj == nil {\n\t\treturn nil, err\n\t}\n\treturn obj.(*v1alpha1.Adjustment), err\n}\n\n// Update takes the representation of a adjustment and updates it. Returns the server's representation of the adjustment, and an error, if there is any.\nfunc (c *FakeAdjustments) Update(adjustment *v1alpha1.Adjustment) (result *v1alpha1.Adjustment, err error) {\n\tobj, err := c.Fake.\n\t\tInvokes(testing.NewUpdateAction(adjustmentsResource, c.ns, adjustment), &v1alpha1.Adjustment{})\n\n\tif obj == nil {\n\t\treturn nil, err\n\t}\n\treturn obj.(*v1alpha1.Adjustment), err\n}\n\n// UpdateStatus was generated because the type contains a Status member.\n// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().\nfunc (c *FakeAdjustments) UpdateStatus(adjustment *v1alpha1.Adjustment) (*v1alpha1.Adjustment, error) {\n\tobj, err := c.Fake.\n\t\tInvokes(testing.NewUpdateSubresourceAction(adjustmentsResource, \"status\", c.ns, adjustment), &v1alpha1.Adjustment{})\n\n\tif obj == nil {\n\t\treturn nil, err\n\t}\n\treturn obj.(*v1alpha1.Adjustment), err\n}\n\n// Delete takes name of the adjustment and deletes it. Returns an error if one occurs.\nfunc (c *FakeAdjustments) Delete(name string, options *v1.DeleteOptions) error {\n\t_, err := c.Fake.\n\t\tInvokes(testing.NewDeleteAction(adjustmentsResource, c.ns, name), &v1alpha1.Adjustment{})\n\n\treturn err\n}\n\n// DeleteCollection deletes a collection of objects.\nfunc (c *FakeAdjustments) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {\n\taction := testing.NewDeleteCollectionAction(adjustmentsResource, c.ns, listOptions)\n\n\t_, err := c.Fake.Invokes(action, &v1alpha1.AdjustmentList{})\n\treturn err\n}\n\n// Patch applies the patch and returns the patched adjustment.\nfunc (c *FakeAdjustments) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Adjustment, err error) {\n\tobj, err := c.Fake.\n\t\tInvokes(testing.NewPatchSubresourceAction(adjustmentsResource, c.ns, name, pt, data, subresources...), &v1alpha1.Adjustment{})\n\n\tif obj == nil {\n\t\treturn nil, err\n\t}\n\treturn obj.(*v1alpha1.Adjustment), err\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/fake/fake_resmgr_client.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage fake\n\nimport (\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1\"\n\trest \"k8s.io/client-go/rest\"\n\ttesting \"k8s.io/client-go/testing\"\n)\n\ntype FakeCriresmgrV1alpha1 struct {\n\t*testing.Fake\n}\n\nfunc (c *FakeCriresmgrV1alpha1) Adjustments(namespace string) v1alpha1.AdjustmentInterface {\n\treturn &FakeAdjustments{c, namespace}\n}\n\n// RESTClient returns a RESTClient that is used to communicate\n// with API server by this client implementation.\nfunc (c *FakeCriresmgrV1alpha1) RESTClient() rest.Interface {\n\tvar ret *rest.RESTClient\n\treturn ret\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/generated_expansion.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage v1alpha1\n\ntype AdjustmentExpansion interface{}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/clientset/versioned/typed/resmgr/v1alpha1/resmgr_client.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by client-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned/scheme\"\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\trest \"k8s.io/client-go/rest\"\n)\n\ntype CriresmgrV1alpha1Interface interface {\n\tRESTClient() rest.Interface\n\tAdjustmentsGetter\n}\n\n// CriresmgrV1alpha1Client is used to interact with features provided by the criresmgr.intel.com group.\ntype CriresmgrV1alpha1Client struct {\n\trestClient rest.Interface\n}\n\nfunc (c *CriresmgrV1alpha1Client) Adjustments(namespace string) AdjustmentInterface {\n\treturn newAdjustments(c, namespace)\n}\n\n// NewForConfig creates a new CriresmgrV1alpha1Client for the given config.\nfunc NewForConfig(c *rest.Config) (*CriresmgrV1alpha1Client, error) {\n\tconfig := *c\n\tif err := setConfigDefaults(&config); err != nil {\n\t\treturn nil, err\n\t}\n\tclient, err := rest.RESTClientFor(&config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &CriresmgrV1alpha1Client{client}, nil\n}\n\n// NewForConfigOrDie creates a new CriresmgrV1alpha1Client for the given config and\n// panics if there is an error in the config.\nfunc NewForConfigOrDie(c *rest.Config) *CriresmgrV1alpha1Client {\n\tclient, err := NewForConfig(c)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn client\n}\n\n// New creates a new CriresmgrV1alpha1Client for the given RESTClient.\nfunc New(c rest.Interface) *CriresmgrV1alpha1Client {\n\treturn &CriresmgrV1alpha1Client{c}\n}\n\nfunc setConfigDefaults(config *rest.Config) error {\n\tgv := v1alpha1.SchemeGroupVersion\n\tconfig.GroupVersion = &gv\n\tconfig.APIPath = \"/apis\"\n\tconfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion()\n\n\tif config.UserAgent == \"\" {\n\t\tconfig.UserAgent = rest.DefaultKubernetesUserAgent()\n\t}\n\n\treturn nil\n}\n\n// RESTClient returns a RESTClient that is used to communicate\n// with API server by this client implementation.\nfunc (c *CriresmgrV1alpha1Client) RESTClient() rest.Interface {\n\tif c == nil {\n\t\treturn nil\n\t}\n\treturn c.restClient\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/informers/externalversions/factory.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by informer-gen. DO NOT EDIT.\n\npackage externalversions\n\nimport (\n\treflect \"reflect\"\n\tsync \"sync\"\n\ttime \"time\"\n\n\tversioned \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned\"\n\tinternalinterfaces \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/informers/externalversions/internalinterfaces\"\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/informers/externalversions/resmgr\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n\tschema \"k8s.io/apimachinery/pkg/runtime/schema\"\n\tcache \"k8s.io/client-go/tools/cache\"\n)\n\n// SharedInformerOption defines the functional option type for SharedInformerFactory.\ntype SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory\n\ntype sharedInformerFactory struct {\n\tclient           versioned.Interface\n\tnamespace        string\n\ttweakListOptions internalinterfaces.TweakListOptionsFunc\n\tlock             sync.Mutex\n\tdefaultResync    time.Duration\n\tcustomResync     map[reflect.Type]time.Duration\n\n\tinformers map[reflect.Type]cache.SharedIndexInformer\n\t// startedInformers is used for tracking which informers have been started.\n\t// This allows Start() to be called multiple times safely.\n\tstartedInformers map[reflect.Type]bool\n}\n\n// WithCustomResyncConfig sets a custom resync period for the specified informer types.\nfunc WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption {\n\treturn func(factory *sharedInformerFactory) *sharedInformerFactory {\n\t\tfor k, v := range resyncConfig {\n\t\t\tfactory.customResync[reflect.TypeOf(k)] = v\n\t\t}\n\t\treturn factory\n\t}\n}\n\n// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory.\nfunc WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption {\n\treturn func(factory *sharedInformerFactory) *sharedInformerFactory {\n\t\tfactory.tweakListOptions = tweakListOptions\n\t\treturn factory\n\t}\n}\n\n// WithNamespace limits the SharedInformerFactory to the specified namespace.\nfunc WithNamespace(namespace string) SharedInformerOption {\n\treturn func(factory *sharedInformerFactory) *sharedInformerFactory {\n\t\tfactory.namespace = namespace\n\t\treturn factory\n\t}\n}\n\n// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces.\nfunc NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory {\n\treturn NewSharedInformerFactoryWithOptions(client, defaultResync)\n}\n\n// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.\n// Listers obtained via this SharedInformerFactory will be subject to the same filters\n// as specified here.\n// Deprecated: Please use NewSharedInformerFactoryWithOptions instead\nfunc NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {\n\treturn NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions))\n}\n\n// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options.\nfunc NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {\n\tfactory := &sharedInformerFactory{\n\t\tclient:           client,\n\t\tnamespace:        v1.NamespaceAll,\n\t\tdefaultResync:    defaultResync,\n\t\tinformers:        make(map[reflect.Type]cache.SharedIndexInformer),\n\t\tstartedInformers: make(map[reflect.Type]bool),\n\t\tcustomResync:     make(map[reflect.Type]time.Duration),\n\t}\n\n\t// Apply all options\n\tfor _, opt := range options {\n\t\tfactory = opt(factory)\n\t}\n\n\treturn factory\n}\n\n// Start initializes all requested informers.\nfunc (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {\n\tf.lock.Lock()\n\tdefer f.lock.Unlock()\n\n\tfor informerType, informer := range f.informers {\n\t\tif !f.startedInformers[informerType] {\n\t\t\tgo informer.Run(stopCh)\n\t\t\tf.startedInformers[informerType] = true\n\t\t}\n\t}\n}\n\n// WaitForCacheSync waits for all started informers' cache were synced.\nfunc (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {\n\tinformers := func() map[reflect.Type]cache.SharedIndexInformer {\n\t\tf.lock.Lock()\n\t\tdefer f.lock.Unlock()\n\n\t\tinformers := map[reflect.Type]cache.SharedIndexInformer{}\n\t\tfor informerType, informer := range f.informers {\n\t\t\tif f.startedInformers[informerType] {\n\t\t\t\tinformers[informerType] = informer\n\t\t\t}\n\t\t}\n\t\treturn informers\n\t}()\n\n\tres := map[reflect.Type]bool{}\n\tfor informType, informer := range informers {\n\t\tres[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced)\n\t}\n\treturn res\n}\n\n// InternalInformerFor returns the SharedIndexInformer for obj using an internal\n// client.\nfunc (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer {\n\tf.lock.Lock()\n\tdefer f.lock.Unlock()\n\n\tinformerType := reflect.TypeOf(obj)\n\tinformer, exists := f.informers[informerType]\n\tif exists {\n\t\treturn informer\n\t}\n\n\tresyncPeriod, exists := f.customResync[informerType]\n\tif !exists {\n\t\tresyncPeriod = f.defaultResync\n\t}\n\n\tinformer = newFunc(f.client, resyncPeriod)\n\tf.informers[informerType] = informer\n\n\treturn informer\n}\n\n// SharedInformerFactory provides shared informers for resources in all known\n// API group versions.\ntype SharedInformerFactory interface {\n\tinternalinterfaces.SharedInformerFactory\n\tForResource(resource schema.GroupVersionResource) (GenericInformer, error)\n\tWaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool\n\n\tCriresmgr() resmgr.Interface\n}\n\nfunc (f *sharedInformerFactory) Criresmgr() resmgr.Interface {\n\treturn resmgr.New(f, f.namespace, f.tweakListOptions)\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/informers/externalversions/generic.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by informer-gen. DO NOT EDIT.\n\npackage externalversions\n\nimport (\n\t\"fmt\"\n\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tschema \"k8s.io/apimachinery/pkg/runtime/schema\"\n\tcache \"k8s.io/client-go/tools/cache\"\n)\n\n// GenericInformer is type of SharedIndexInformer which will locate and delegate to other\n// sharedInformers based on type\ntype GenericInformer interface {\n\tInformer() cache.SharedIndexInformer\n\tLister() cache.GenericLister\n}\n\ntype genericInformer struct {\n\tinformer cache.SharedIndexInformer\n\tresource schema.GroupResource\n}\n\n// Informer returns the SharedIndexInformer.\nfunc (f *genericInformer) Informer() cache.SharedIndexInformer {\n\treturn f.informer\n}\n\n// Lister returns the GenericLister.\nfunc (f *genericInformer) Lister() cache.GenericLister {\n\treturn cache.NewGenericLister(f.Informer().GetIndexer(), f.resource)\n}\n\n// ForResource gives generic access to a shared informer of the matching type\n// TODO extend this to unknown resources with a client pool\nfunc (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {\n\tswitch resource {\n\t// Group=criresmgr.intel.com, Version=v1alpha1\n\tcase v1alpha1.SchemeGroupVersion.WithResource(\"adjustments\"):\n\t\treturn &genericInformer{resource: resource.GroupResource(), informer: f.Criresmgr().V1alpha1().Adjustments().Informer()}, nil\n\n\t}\n\n\treturn nil, fmt.Errorf(\"no informer found for %v\", resource)\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/informers/externalversions/internalinterfaces/factory_interfaces.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by informer-gen. DO NOT EDIT.\n\npackage internalinterfaces\n\nimport (\n\ttime \"time\"\n\n\tversioned \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n\tcache \"k8s.io/client-go/tools/cache\"\n)\n\n// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer.\ntype NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer\n\n// SharedInformerFactory a small interface to allow for adding an informer without an import cycle\ntype SharedInformerFactory interface {\n\tStart(stopCh <-chan struct{})\n\tInformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer\n}\n\n// TweakListOptionsFunc is a function that transforms a v1.ListOptions.\ntype TweakListOptionsFunc func(*v1.ListOptions)\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/informers/externalversions/resmgr/interface.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by informer-gen. DO NOT EDIT.\n\npackage resmgr\n\nimport (\n\tinternalinterfaces \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/informers/externalversions/internalinterfaces\"\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/informers/externalversions/resmgr/v1alpha1\"\n)\n\n// Interface provides access to each of this group's versions.\ntype Interface interface {\n\t// V1alpha1 provides access to shared informers for resources in V1alpha1.\n\tV1alpha1() v1alpha1.Interface\n}\n\ntype group struct {\n\tfactory          internalinterfaces.SharedInformerFactory\n\tnamespace        string\n\ttweakListOptions internalinterfaces.TweakListOptionsFunc\n}\n\n// New returns a new Interface.\nfunc New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {\n\treturn &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}\n}\n\n// V1alpha1 returns a new v1alpha1.Interface.\nfunc (g *group) V1alpha1() v1alpha1.Interface {\n\treturn v1alpha1.New(g.factory, g.namespace, g.tweakListOptions)\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/informers/externalversions/resmgr/v1alpha1/adjustment.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by informer-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\ttime \"time\"\n\n\tversioned \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/clientset/versioned\"\n\tinternalinterfaces \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/informers/externalversions/internalinterfaces\"\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/listers/resmgr/v1alpha1\"\n\tresmgrv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n\twatch \"k8s.io/apimachinery/pkg/watch\"\n\tcache \"k8s.io/client-go/tools/cache\"\n)\n\n// AdjustmentInformer provides access to a shared informer and lister for\n// Adjustments.\ntype AdjustmentInformer interface {\n\tInformer() cache.SharedIndexInformer\n\tLister() v1alpha1.AdjustmentLister\n}\n\ntype adjustmentInformer struct {\n\tfactory          internalinterfaces.SharedInformerFactory\n\ttweakListOptions internalinterfaces.TweakListOptionsFunc\n\tnamespace        string\n}\n\n// NewAdjustmentInformer constructs a new informer for Adjustment type.\n// Always prefer using an informer factory to get a shared informer instead of getting an independent\n// one. This reduces memory footprint and number of connections to the server.\nfunc NewAdjustmentInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {\n\treturn NewFilteredAdjustmentInformer(client, namespace, resyncPeriod, indexers, nil)\n}\n\n// NewFilteredAdjustmentInformer constructs a new informer for Adjustment type.\n// Always prefer using an informer factory to get a shared informer instead of getting an independent\n// one. This reduces memory footprint and number of connections to the server.\nfunc NewFilteredAdjustmentInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {\n\treturn cache.NewSharedIndexInformer(\n\t\t&cache.ListWatch{\n\t\t\tListFunc: func(options v1.ListOptions) (runtime.Object, error) {\n\t\t\t\tif tweakListOptions != nil {\n\t\t\t\t\ttweakListOptions(&options)\n\t\t\t\t}\n\t\t\t\treturn client.CriresmgrV1alpha1().Adjustments(namespace).List(options)\n\t\t\t},\n\t\t\tWatchFunc: func(options v1.ListOptions) (watch.Interface, error) {\n\t\t\t\tif tweakListOptions != nil {\n\t\t\t\t\ttweakListOptions(&options)\n\t\t\t\t}\n\t\t\t\treturn client.CriresmgrV1alpha1().Adjustments(namespace).Watch(options)\n\t\t\t},\n\t\t},\n\t\t&resmgrv1alpha1.Adjustment{},\n\t\tresyncPeriod,\n\t\tindexers,\n\t)\n}\n\nfunc (f *adjustmentInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {\n\treturn NewFilteredAdjustmentInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)\n}\n\nfunc (f *adjustmentInformer) Informer() cache.SharedIndexInformer {\n\treturn f.factory.InformerFor(&resmgrv1alpha1.Adjustment{}, f.defaultInformer)\n}\n\nfunc (f *adjustmentInformer) Lister() v1alpha1.AdjustmentLister {\n\treturn v1alpha1.NewAdjustmentLister(f.Informer().GetIndexer())\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/informers/externalversions/resmgr/v1alpha1/interface.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by informer-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\tinternalinterfaces \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/generated/informers/externalversions/internalinterfaces\"\n)\n\n// Interface provides access to all the informers in this group version.\ntype Interface interface {\n\t// Adjustments returns a AdjustmentInformer.\n\tAdjustments() AdjustmentInformer\n}\n\ntype version struct {\n\tfactory          internalinterfaces.SharedInformerFactory\n\tnamespace        string\n\ttweakListOptions internalinterfaces.TweakListOptionsFunc\n}\n\n// New returns a new Interface.\nfunc New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {\n\treturn &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}\n}\n\n// Adjustments returns a AdjustmentInformer.\nfunc (v *version) Adjustments() AdjustmentInformer {\n\treturn &adjustmentInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/listers/resmgr/v1alpha1/adjustment.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by lister-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\tv1alpha1 \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n\t\"k8s.io/apimachinery/pkg/api/errors\"\n\t\"k8s.io/apimachinery/pkg/labels\"\n\t\"k8s.io/client-go/tools/cache\"\n)\n\n// AdjustmentLister helps list Adjustments.\ntype AdjustmentLister interface {\n\t// List lists all Adjustments in the indexer.\n\tList(selector labels.Selector) (ret []*v1alpha1.Adjustment, err error)\n\t// Adjustments returns an object that can list and get Adjustments.\n\tAdjustments(namespace string) AdjustmentNamespaceLister\n\tAdjustmentListerExpansion\n}\n\n// adjustmentLister implements the AdjustmentLister interface.\ntype adjustmentLister struct {\n\tindexer cache.Indexer\n}\n\n// NewAdjustmentLister returns a new AdjustmentLister.\nfunc NewAdjustmentLister(indexer cache.Indexer) AdjustmentLister {\n\treturn &adjustmentLister{indexer: indexer}\n}\n\n// List lists all Adjustments in the indexer.\nfunc (s *adjustmentLister) List(selector labels.Selector) (ret []*v1alpha1.Adjustment, err error) {\n\terr = cache.ListAll(s.indexer, selector, func(m interface{}) {\n\t\tret = append(ret, m.(*v1alpha1.Adjustment))\n\t})\n\treturn ret, err\n}\n\n// Adjustments returns an object that can list and get Adjustments.\nfunc (s *adjustmentLister) Adjustments(namespace string) AdjustmentNamespaceLister {\n\treturn adjustmentNamespaceLister{indexer: s.indexer, namespace: namespace}\n}\n\n// AdjustmentNamespaceLister helps list and get Adjustments.\ntype AdjustmentNamespaceLister interface {\n\t// List lists all Adjustments in the indexer for a given namespace.\n\tList(selector labels.Selector) (ret []*v1alpha1.Adjustment, err error)\n\t// Get retrieves the Adjustment from the indexer for a given namespace and name.\n\tGet(name string) (*v1alpha1.Adjustment, error)\n\tAdjustmentNamespaceListerExpansion\n}\n\n// adjustmentNamespaceLister implements the AdjustmentNamespaceLister\n// interface.\ntype adjustmentNamespaceLister struct {\n\tindexer   cache.Indexer\n\tnamespace string\n}\n\n// List lists all Adjustments in the indexer for a given namespace.\nfunc (s adjustmentNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.Adjustment, err error) {\n\terr = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {\n\t\tret = append(ret, m.(*v1alpha1.Adjustment))\n\t})\n\treturn ret, err\n}\n\n// Get retrieves the Adjustment from the indexer for a given namespace and name.\nfunc (s adjustmentNamespaceLister) Get(name string) (*v1alpha1.Adjustment, error) {\n\tobj, exists, err := s.indexer.GetByKey(s.namespace + \"/\" + name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif !exists {\n\t\treturn nil, errors.NewNotFound(v1alpha1.Resource(\"adjustment\"), name)\n\t}\n\treturn obj.(*v1alpha1.Adjustment), nil\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/generated/listers/resmgr/v1alpha1/expansion_generated.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by lister-gen. DO NOT EDIT.\n\npackage v1alpha1\n\n// AdjustmentListerExpansion allows custom methods to be added to\n// AdjustmentLister.\ntype AdjustmentListerExpansion interface{}\n\n// AdjustmentNamespaceListerExpansion allows custom methods to be added to\n// AdjustmentNamespaceLister.\ntype AdjustmentNamespaceListerExpansion interface{}\n"
  },
  {
    "path": "pkg/apis/resmgr/v1alpha1/adjustment-schema.yaml",
    "content": "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: adjustments.criresmgr.intel.com\nspec:\n  group: criresmgr.intel.com\n  names:\n    kind: Adjustment\n    singular: adjustment\n    plural: adjustments\n  scope: Namespaced\n  versions:\n    - name: v1alpha1\n      served: true\n      storage: true\n      schema:\n        # openAPI V3 Schema for validating adjustments\n        openAPIV3Schema:\n          type: object\n          required: [ spec ]\n          properties:\n            spec:\n              type: object\n              required: [ scope ]\n              properties:\n                scope:\n                  type: array\n                  items:\n                    type: object\n                    properties:\n                      nodes:\n                        type: array\n                        items:\n                          type: string\n                      containers:\n                        type: array\n                        items:\n                          type: object\n                          properties:\n                            key:\n                              type: string\n                            operator:\n                              type: string\n                            values:\n                              type: array\n                              items:\n                                type: string\n                resources:\n                  type: object\n                  properties:\n                    requests:\n                      type: object\n                      properties:\n                        cpu:\n                          type: string\n                        memory:\n                          type: string\n                    limits:\n                      type: object\n                      properties:\n                        cpu:\n                          type: string\n                        memory:\n                          type: string\n                classes:\n                  type: object\n                  properties:\n                    rdt:\n                      type: string\n                    blockio:\n                      type: string\n                toptierLimit:\n                  type: string\n            status:\n              type: object\n              properties:\n                nodes:\n                  type: object\n                  additionalProperties:\n                    type: object\n                    properties:\n                      errors:\n                        type: object\n                        additionalProperties:\n                          type: string\n"
  },
  {
    "path": "pkg/apis/resmgr/v1alpha1/adjustment.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1alpha1\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\tcorev1 \"k8s.io/api/core/v1\"\n)\n\n// HasSameVersion checks if the policy has the same version as the other.\nfunc (a *Adjustment) HasSameVersion(o *Adjustment) bool {\n\tif a.ResourceVersion != o.ResourceVersion {\n\t\treturn false\n\t}\n\tif a.Generation != o.Generation {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// NodeScope returns the sub-slice of scopes that apply to the given node.\nfunc (spec *AdjustmentSpec) NodeScope(node string) []AdjustmentScope {\n\tfiltered := []AdjustmentScope{}\n\tfor _, scope := range spec.Scope {\n\t\tif scope.IsNodeInScope(node) {\n\t\t\tfiltered = append(filtered, scope)\n\t\t}\n\t}\n\treturn filtered\n}\n\n// GetResourceRequirements returns the k8s resource requirements for this adjustment.\nfunc (spec *AdjustmentSpec) GetResourceRequirements() (corev1.ResourceRequirements, bool) {\n\tif spec.Resources != nil {\n\t\treturn *spec.Resources, true\n\t}\n\treturn corev1.ResourceRequirements{}, false\n}\n\n// GetRDTClass returns the RDT class for this adjustment.\nfunc (spec *AdjustmentSpec) GetRDTClass() (string, bool) {\n\tif spec.Classes == nil || spec.Classes.RDT == nil {\n\t\treturn \"\", false\n\t}\n\treturn *spec.Classes.RDT, true\n}\n\n// GetBlockIOClass returns the Block I/O class for this adjustment.\nfunc (spec *AdjustmentSpec) GetBlockIOClass() (string, bool) {\n\tif spec.Classes == nil || spec.Classes.BlockIO == nil {\n\t\treturn \"\", false\n\t}\n\treturn *spec.Classes.BlockIO, true\n}\n\n// IsNodeInScope tests if the node is within the scope of this spec.\nfunc (spec *AdjustmentSpec) IsNodeInScope(node string) bool {\n\tif len(spec.Scope) == 0 {\n\t\treturn true\n\t}\n\tfor _, s := range spec.Scope {\n\t\tif s.IsNodeInScope(node) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// IsContainerInScope tests if the container is within the scope of this spec.\nfunc (spec *AdjustmentSpec) IsContainerInScope(container resmgr.Evaluable) bool {\n\tif len(spec.Scope) == 0 {\n\t\treturn true\n\t}\n\tfor _, s := range spec.Scope {\n\t\tif s.IsContainerInScope(container) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Compare checks if this spec is identical to another.\nfunc (spec *AdjustmentSpec) Compare(other *AdjustmentSpec) bool {\n\tswitch {\n\tcase !CompareScopes(spec.Scope, other.Scope):\n\t\treturn false\n\tcase !spec.compareResources(other):\n\t\treturn false\n\tcase !spec.Classes.Compare(other.Classes):\n\t\treturn false\n\tcase spec.ToptierLimit == nil && other.ToptierLimit != nil:\n\t\treturn false\n\tcase spec.ToptierLimit != nil && other.ToptierLimit == nil:\n\t\treturn false\n\tcase spec.ToptierLimit != nil && spec.ToptierLimit.Value() != other.ToptierLimit.Value():\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Verify checks the given spec for obvious errors.\nfunc (spec *AdjustmentSpec) Verify() error {\n\tif err := spec.verifyResources(); err != nil {\n\t\treturn err\n\t}\n\tif err := spec.verifyToptierLimit(); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Check if the resources in this spec are identical to another one.\nfunc (spec *AdjustmentSpec) compareResources(other *AdjustmentSpec) bool {\n\tswitch {\n\tcase spec == nil && other == nil:\n\t\treturn true\n\tcase spec != nil && other == nil:\n\t\treturn true\n\tcase spec == nil && other != nil:\n\t\treturn true\n\tcase spec.Resources == nil && other.Resources == nil:\n\t\treturn true\n\tcase spec.Resources != nil && other.Resources == nil:\n\t\treturn false\n\tcase spec.Resources == nil && other.Resources != nil:\n\t\treturn false\n\t}\n\n\tr := *spec.Resources\n\to := *other.Resources\n\n\tif len(r.Requests) != len(o.Requests) {\n\t\treturn false\n\t}\n\tif len(r.Limits) != len(o.Limits) {\n\t\treturn false\n\t}\n\tfor name, qty := range r.Requests {\n\t\toqty, ok := o.Requests[name]\n\t\tif !ok || qty.Cmp(oqty) != 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\tfor name, qty := range r.Limits {\n\t\toqty, ok := o.Limits[name]\n\t\tif !ok || qty.Cmp(oqty) != 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// verifyResources verifies the resource requirements of this spec.\nfunc (spec *AdjustmentSpec) verifyResources() error {\n\tif spec.Resources == nil {\n\t\treturn nil\n\t}\n\n\tr := *spec.Resources\n\tif r.Requests == nil {\n\t\tr.Requests = corev1.ResourceList{}\n\t}\n\tif r.Limits == nil {\n\t\tr.Limits = corev1.ResourceList{}\n\t}\n\n\treq, rok := r.Requests[corev1.ResourceCPU]\n\tlim, lok := r.Limits[corev1.ResourceCPU]\n\tswitch {\n\tcase !rok && lok:\n\t\tr.Requests[corev1.ResourceCPU] = lim\n\tcase rok && lok:\n\t\tif lim.Cmp(req) < 0 {\n\t\t\treturn apiError(\"invalid CPU limit %q < request %q\", lim, req)\n\t\t}\n\t}\n\n\treq, rok = r.Requests[corev1.ResourceMemory]\n\tlim, lok = r.Limits[corev1.ResourceMemory]\n\tswitch {\n\tcase !rok && lok:\n\t\tr.Requests[corev1.ResourceMemory] = lim\n\tcase rok && lok:\n\t\tif lim.Cmp(req) < 0 {\n\t\t\treturn apiError(\"invalid memory limit %q < request %q\", lim, req)\n\t\t}\n\t}\n\n\tfor name := range r.Requests {\n\t\tswitch name {\n\t\tcase corev1.ResourceCPU, corev1.ResourceMemory:\n\t\tdefault:\n\t\t\treturn apiError(\"invalid resource requests: unsupported resource %v\", name)\n\t\t}\n\t}\n\n\tfor name := range r.Limits {\n\t\tswitch name {\n\t\tcase corev1.ResourceCPU, corev1.ResourceMemory:\n\t\tdefault:\n\t\t\treturn apiError(\"invalid resource limits: unsupported resource %v\", name)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// verifyToptierLimit verifies the top tier memory limit settings of this spec.\nfunc (spec *AdjustmentSpec) verifyToptierLimit() error {\n\tif spec.ToptierLimit == nil {\n\t\treturn nil\n\t}\n\n\tl := spec.ToptierLimit.Value()\n\tif l < 0 {\n\t\treturn apiError(\"invalid ToptierLimit %v\", l)\n\t}\n\n\treturn nil\n}\n\n// IsNodeInScope tests if the node is within this scope.\nfunc (scope *AdjustmentScope) IsNodeInScope(node string) bool {\n\tif len(scope.Nodes) == 0 {\n\t\treturn true\n\t}\n\tfor _, n := range scope.Nodes {\n\t\tif matches(n, node) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// IsContainerInScope tests if the container is within this scope.\nfunc (scope *AdjustmentScope) IsContainerInScope(container resmgr.Evaluable) bool {\n\tif len(scope.Containers) == 0 {\n\t\treturn true\n\t}\n\tfor _, expr := range scope.Containers {\n\t\tif expr.Evaluate(container) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// match a string against a primitive pattern with a single optional trailing '*'.\nfunc matches(pattern, name string) bool {\n\tif pattern == \"\" {\n\t\treturn true\n\t}\n\tif !strings.HasSuffix(pattern, \"*\") {\n\t\treturn pattern == name\n\t}\n\treturn strings.HasPrefix(name, pattern[0:len(pattern)-1])\n}\n\n// CompareScopes checks if two slices of scopes are (syntactically) identical.\nfunc CompareScopes(scopes []AdjustmentScope, others []AdjustmentScope) bool {\n\tif len(scopes) != len(others) {\n\t\treturn false\n\t}\n\tfor idx, s := range scopes {\n\t\to := others[idx]\n\t\tif !s.Compare(&o) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Compare check if the scope is identical to another one.\nfunc (scope *AdjustmentScope) Compare(other *AdjustmentScope) bool {\n\tif len(scope.Nodes) != len(other.Nodes) || len(scope.Containers) != len(other.Containers) {\n\t\treturn false\n\t}\n\tfor idx, n := range scope.Nodes {\n\t\tif other.Nodes[idx] != n {\n\t\t\treturn false\n\t\t}\n\t}\n\tfor idx, c := range scope.Containers {\n\t\tif other.Containers[idx] != c {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Compare checks if the classes are identical to another set.\nfunc (c *Classes) Compare(o *Classes) bool {\n\tswitch {\n\tcase c == nil && o == nil:\n\t\treturn true\n\tcase c != nil && o == nil, c == nil && o != nil:\n\t\treturn false\n\tcase c.RDT != nil && o.RDT == nil, c.RDT == nil && o.RDT != nil:\n\t\treturn false\n\tcase c.BlockIO != nil && o.BlockIO == nil, c.BlockIO == nil && o.BlockIO != nil:\n\t\treturn false\n\tcase c.RDT == nil && c.BlockIO == nil:\n\t\treturn true\n\t}\n\treturn *c.RDT == *o.RDT && *c.BlockIO == *o.BlockIO\n}\n\n// apiError returns a format error specific to this API.\nfunc apiError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"adjustment API error: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/v1alpha1/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// +k8s:deepcopy-gen=package\n// +groupName=criresmgr.intel.com\n\npackage v1alpha1\n"
  },
  {
    "path": "pkg/apis/resmgr/v1alpha1/register.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1alpha1\n\nimport (\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\t\"k8s.io/apimachinery/pkg/runtime/schema\"\n)\n\nvar (\n\t// SchemeBuilder initializes a scheme builder\n\tSchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)\n\t// AddToScheme is a global function that registers this API group & version to a scheme\n\tAddToScheme = SchemeBuilder.AddToScheme\n)\n\n// SchemeGroupVersion is group version used to register these objects.\nvar SchemeGroupVersion = schema.GroupVersion{\n\tGroup:   GroupName,\n\tVersion: Version,\n}\n\nfunc Resource(resource string) schema.GroupResource {\n\treturn SchemeGroupVersion.WithResource(resource).GroupResource()\n}\n\n// Adds the list of known types to api.Scheme.\nfunc addKnownTypes(scheme *runtime.Scheme) error {\n\tscheme.AddKnownTypes(SchemeGroupVersion,\n\t\t&Adjustment{},\n\t\t&AdjustmentList{},\n\t)\n\tmetav1.AddToGroupVersion(scheme, SchemeGroupVersion)\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/v1alpha1/types.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1alpha1\n\nimport (\n\tcorev1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n)\n\nconst (\n\tGroupName string = \"criresmgr.intel.com\"    // GroupName is the group of our CRD.\n\tVersion   string = \"v1alpha1\"               // Version is the API version of our CRD.\n\tKind      string = \"Adjustment\"             // Kind is the object kind of our CRD.\n\tPlural    string = \"adjustments\"            // Plural is Kind in plural form.\n\tSingular  string = \"adjustment\"             // Singular is Kind in singular form.\n\tName      string = Plural + \".\" + GroupName // Name is the full name of our CRD.\n)\n\n// +genclient\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n\n// Adjustment is a CRD used to externally adjust containers resource assignments.\ntype Adjustment struct {\n\tmetav1.TypeMeta   `json:\",inline\"`\n\tmetav1.ObjectMeta `json:\"metadata,omitempty\"`\n\n\tSpec   AdjustmentSpec   `json:\"spec\"`\n\tStatus AdjustmentStatus `json:\"status\"`\n}\n\n// AdjustmentSpec specifies the scope for an external adjustment.\ntype AdjustmentSpec struct {\n\tScope        []AdjustmentScope            `json:\"scope\"`\n\tResources    *corev1.ResourceRequirements `json:\"resources\"`\n\tClasses      *Classes                     `json:\"classes\"`\n\tToptierLimit *resapi.Quantity             `json:\"toptierLimit\"`\n}\n\n// AdjustmentStatus represents the status of applying an adjustment.\ntype AdjustmentStatus struct {\n\tNodes map[string]AdjustmentNodeStatus `json:\"nodes\"`\n}\n\n// AdjustmentNodeStatus represents the status of an adjustment on a node.\ntype AdjustmentNodeStatus struct {\n\tErrors map[string]string `json:\"errors\"`\n}\n\n// AdjustmentScope defines the scope for an adjustment.\ntype AdjustmentScope struct {\n\tNodes      []string             `json:\"nodes\"`\n\tContainers []*resmgr.Expression `json:\"containers\"`\n}\n\n// Classes defines RDT and BlockIO class assignments.\ntype Classes struct {\n\tBlockIO *string `json:\"blockio\"`\n\tRDT     *string `json:\"rdt\"`\n}\n\n// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n\n// AdjustmentList is a list of Adjustments.\ntype AdjustmentList struct {\n\tmetav1.TypeMeta `json:\",inline\"`\n\tmetav1.ListMeta `json:\"metadata\"`\n\n\tItems []Adjustment `json:\"items\"`\n}\n"
  },
  {
    "path": "pkg/apis/resmgr/v1alpha1/zz_generated.deepcopy.go",
    "content": "//go:build !ignore_autogenerated\n// +build !ignore_autogenerated\n\n// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Code generated by deepcopy-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\tv1 \"k8s.io/api/core/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n)\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *Adjustment) DeepCopyInto(out *Adjustment) {\n\t*out = *in\n\tout.TypeMeta = in.TypeMeta\n\tin.ObjectMeta.DeepCopyInto(&out.ObjectMeta)\n\tin.Spec.DeepCopyInto(&out.Spec)\n\tin.Status.DeepCopyInto(&out.Status)\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Adjustment.\nfunc (in *Adjustment) DeepCopy() *Adjustment {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(Adjustment)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.\nfunc (in *Adjustment) DeepCopyObject() runtime.Object {\n\tif c := in.DeepCopy(); c != nil {\n\t\treturn c\n\t}\n\treturn nil\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *AdjustmentList) DeepCopyInto(out *AdjustmentList) {\n\t*out = *in\n\tout.TypeMeta = in.TypeMeta\n\tin.ListMeta.DeepCopyInto(&out.ListMeta)\n\tif in.Items != nil {\n\t\tin, out := &in.Items, &out.Items\n\t\t*out = make([]Adjustment, len(*in))\n\t\tfor i := range *in {\n\t\t\t(*in)[i].DeepCopyInto(&(*out)[i])\n\t\t}\n\t}\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdjustmentList.\nfunc (in *AdjustmentList) DeepCopy() *AdjustmentList {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(AdjustmentList)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.\nfunc (in *AdjustmentList) DeepCopyObject() runtime.Object {\n\tif c := in.DeepCopy(); c != nil {\n\t\treturn c\n\t}\n\treturn nil\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *AdjustmentNodeStatus) DeepCopyInto(out *AdjustmentNodeStatus) {\n\t*out = *in\n\tif in.Errors != nil {\n\t\tin, out := &in.Errors, &out.Errors\n\t\t*out = make(map[string]string, len(*in))\n\t\tfor key, val := range *in {\n\t\t\t(*out)[key] = val\n\t\t}\n\t}\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdjustmentNodeStatus.\nfunc (in *AdjustmentNodeStatus) DeepCopy() *AdjustmentNodeStatus {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(AdjustmentNodeStatus)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *AdjustmentScope) DeepCopyInto(out *AdjustmentScope) {\n\t*out = *in\n\tif in.Nodes != nil {\n\t\tin, out := &in.Nodes, &out.Nodes\n\t\t*out = make([]string, len(*in))\n\t\tcopy(*out, *in)\n\t}\n\tif in.Containers != nil {\n\t\tin, out := &in.Containers, &out.Containers\n\t\t*out = make([]*resmgr.Expression, len(*in))\n\t\tfor i := range *in {\n\t\t\tif (*in)[i] != nil {\n\t\t\t\tin, out := &(*in)[i], &(*out)[i]\n\t\t\t\t*out = (*in).DeepCopy()\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdjustmentScope.\nfunc (in *AdjustmentScope) DeepCopy() *AdjustmentScope {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(AdjustmentScope)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *AdjustmentSpec) DeepCopyInto(out *AdjustmentSpec) {\n\t*out = *in\n\tif in.Scope != nil {\n\t\tin, out := &in.Scope, &out.Scope\n\t\t*out = make([]AdjustmentScope, len(*in))\n\t\tfor i := range *in {\n\t\t\t(*in)[i].DeepCopyInto(&(*out)[i])\n\t\t}\n\t}\n\tif in.Resources != nil {\n\t\tin, out := &in.Resources, &out.Resources\n\t\t*out = new(v1.ResourceRequirements)\n\t\t(*in).DeepCopyInto(*out)\n\t}\n\tif in.Classes != nil {\n\t\tin, out := &in.Classes, &out.Classes\n\t\t*out = new(Classes)\n\t\t(*in).DeepCopyInto(*out)\n\t}\n\tif in.ToptierLimit != nil {\n\t\tin, out := &in.ToptierLimit, &out.ToptierLimit\n\t\tx := (*in).DeepCopy()\n\t\t*out = &x\n\t}\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdjustmentSpec.\nfunc (in *AdjustmentSpec) DeepCopy() *AdjustmentSpec {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(AdjustmentSpec)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *AdjustmentStatus) DeepCopyInto(out *AdjustmentStatus) {\n\t*out = *in\n\tif in.Nodes != nil {\n\t\tin, out := &in.Nodes, &out.Nodes\n\t\t*out = make(map[string]AdjustmentNodeStatus, len(*in))\n\t\tfor key, val := range *in {\n\t\t\t(*out)[key] = *val.DeepCopy()\n\t\t}\n\t}\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdjustmentStatus.\nfunc (in *AdjustmentStatus) DeepCopy() *AdjustmentStatus {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(AdjustmentStatus)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *Classes) DeepCopyInto(out *Classes) {\n\t*out = *in\n\tif in.BlockIO != nil {\n\t\tin, out := &in.BlockIO, &out.BlockIO\n\t\t*out = new(string)\n\t\t**out = **in\n\t}\n\tif in.RDT != nil {\n\t\tin, out := &in.RDT, &out.RDT\n\t\t*out = new(string)\n\t\t**out = **in\n\t}\n\treturn\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Classes.\nfunc (in *Classes) DeepCopy() *Classes {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(Classes)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n"
  },
  {
    "path": "pkg/avx/collector.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage avx\n\n//go:generate go run elfdump.go\n\nimport (\n\t\"bytes\"\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"unsafe\"\n\n\tbpf \"github.com/cilium/ebpf\"\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"golang.org/x/sys/unix\"\n)\n\nconst (\n\t// LastCPUName is the Prometheuse Gauge name for last CPU with AVX512 instructions.\n\tLastCPUName = \"last_cpu_avx_task_switches\"\n\t// AVXSwitchCountName is the Prometheuse Gauge name for AVX switch count per cgroup.\n\tAVXSwitchCountName = \"avx_switch_count_per_cgroup\"\n\t// AllSwitchCountName is the Prometheuse Gauge name for all switch count per cgroup.\n\tAllSwitchCountName = \"all_switch_count_per_cgroup\"\n\t// LastUpdateNs is the Prometheuse Gauge name for per cgroup AVX512 activity timestamp.\n\tLastUpdateNs = \"last_update_ns\"\n\t// Path to kernel tracepoints\n\tkernelTracepointPath = \"/sys/kernel/debug/tracing/events\"\n\t// rlimit value (512k) needed to lock map data in memory\n\tmapMemLockLimit = 524288\n)\n\n// Prometheus Metric descriptor indices and descriptor table\nconst (\n\tlastCPUDesc = iota\n\tavxSwitchCountDesc\n\tallSwitchCountDesc\n\tlastUpdateNsDesc\n\tnumDescriptors\n)\n\nvar descriptors = [numDescriptors]*prometheus.Desc{\n\tlastCPUDesc: prometheus.NewDesc(\n\t\tLastCPUName,\n\t\t\"Number of task switches on the CPU where AVX512 instructions were used.\",\n\t\t[]string{\n\t\t\t\"cpu_id\",\n\t\t}, nil,\n\t),\n\tavxSwitchCountDesc: prometheus.NewDesc(\n\t\tAVXSwitchCountName,\n\t\t\"Number of task switches where AVX512 instructions were used in a particular cgroup.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t}, nil,\n\t),\n\tallSwitchCountDesc: prometheus.NewDesc(\n\t\tAllSwitchCountName,\n\t\t\"Total number of task switches in a particular cgroup.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t}, nil,\n\t),\n\tlastUpdateNsDesc: prometheus.NewDesc(\n\t\t\"last_update_ns\",\n\t\t\"Time since last AVX512 activity in a particular cgroup.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t}, nil,\n\t),\n}\n\nvar (\n\tbpfBinaryName  = \"avx512.o\"\n\tbpfInstallpath = \"/usr/libexec/bpf\"\n\n\t// our logger instance\n\tlog = logger.NewLogger(\"avx\")\n)\n\ntype collector struct {\n\troot string\n\tebpf *bpf.Collection\n\tfds  []int\n}\n\nfunc enablePerfTracepoint(prog *bpf.Program, tracepoint string) (int, error) {\n\n\tid, err := os.ReadFile(filepath.Join(kernelTracepointPath, tracepoint, \"id\"))\n\tif err != nil {\n\t\treturn -1, errors.Wrap(err, \"unable to read tracepoint ID\")\n\t}\n\n\ttid, err := strconv.Atoi(strings.TrimSpace(string(id)))\n\tif err != nil {\n\t\treturn -1, errors.New(\"unable to convert tracepoint ID\")\n\t}\n\n\tattr := unix.PerfEventAttr{\n\t\tType:        unix.PERF_TYPE_TRACEPOINT,\n\t\tConfig:      uint64(tid), // tracepoint id\n\t\tSample_type: unix.PERF_SAMPLE_RAW,\n\t\tSample:      1,\n\t\tWakeup:      1,\n\t}\n\n\tpfd, err := unix.PerfEventOpen(&attr, -1, 0, -1, unix.PERF_FLAG_FD_CLOEXEC)\n\tif err != nil {\n\t\treturn -1, errors.Wrap(err, \"unable to open perf events\")\n\t}\n\n\tif _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(pfd), unix.PERF_EVENT_IOC_ENABLE, 0); errno != 0 {\n\t\treturn -1, errors.Errorf(\"unable to set up perf events: %s\", errno)\n\t}\n\n\tif _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(pfd), unix.PERF_EVENT_IOC_SET_BPF, uintptr(prog.FD())); errno != 0 {\n\t\treturn -1, errors.Errorf(\"unable to attach bpf program to perf events: %s\", errno)\n\t}\n\n\treturn pfd, nil\n}\n\nfunc getKernelVersion() uint32 {\n\n\tvar uts unix.Utsname\n\n\terr := unix.Uname(&uts)\n\tif err != nil {\n\t\treturn 0\n\t}\n\n\tstr := string(bytes.SplitN(uts.Release[:], []byte{0}, 2)[0])\n\n\tver := strings.SplitN(str, \".\", 3)\n\n\tmajor, err := strconv.ParseUint(ver[0], 10, 8)\n\tif err != nil {\n\t\treturn 0\n\t}\n\tminor, err := strconv.ParseUint(ver[1], 10, 8)\n\tif err != nil {\n\t\treturn uint32(major << 16)\n\t}\n\n\t// ignore patch version\n\treturn uint32(major<<16 + minor<<8)\n}\n\nfunc kernelVersionStr(v uint32) string {\n\treturn fmt.Sprintf(\"%d.%d.0\", v>>16, (v>>8)&0xff)\n}\n\n// NewCollector creates new Prometheus collector for AVX metrics\nfunc NewCollector() (prometheus.Collector, error) {\n\n\t// Set rlimit to be able to lock map values in memory\n\tmemlockLimit := &unix.Rlimit{\n\t\tCur: mapMemLockLimit,\n\t\tMax: mapMemLockLimit,\n\t}\n\terr := unix.Setrlimit(unix.RLIMIT_MEMLOCK, memlockLimit)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to set rlimit\")\n\t}\n\n\tspec, err := bpf.LoadCollectionSpec(filepath.Join(bpfInstallpath, bpfBinaryName))\n\tif err != nil {\n\t\tlog.Info(\"Unable to load user eBPF (%v). Using default CollectionSpec from ELF program bytes\", err)\n\t\tspec, err = bpf.LoadCollectionSpecFromReader(bytes.NewReader(program[:]))\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"unable to load default CollectionSpec from ELF program bytes\")\n\t\t}\n\t}\n\n\thostVer := getKernelVersion()\n\tprogVer := spec.Programs[\"tracepoint__x86_fpu_regs_deactivated\"].KernelVersion\n\n\tif hostVer < progVer {\n\t\treturn nil, errors.Wrapf(err, \"The host kernel version (v%s) is too old to run the AVX512 collector program. Minimum version is v%s\", kernelVersionStr(hostVer), kernelVersionStr(progVer))\n\t}\n\n\tcollection, err := bpf.NewCollection(spec)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to create new Collection\")\n\t}\n\n\tffd, err := enablePerfTracepoint(collection.Programs[\"tracepoint__x86_fpu_regs_deactivated\"], \"x86_fpu/x86_fpu_regs_deactivated\")\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to enable fpu tracepoint\")\n\t}\n\n\tsfd, err := enablePerfTracepoint(collection.Programs[\"tracepoint__sched_switch\"], \"sched/sched_switch\")\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to enable sched tracepoint\")\n\t}\n\n\treturn &collector{\n\t\troot: cgroups.GetV2Dir(),\n\t\tebpf: collection,\n\t\tfds:  []int{ffd, sfd},\n\t}, nil\n}\n\n// Describe implements prometheus.Collector interface\nfunc (c *collector) Describe(ch chan<- *prometheus.Desc) {\n\tfor _, d := range descriptors {\n\t\tch <- d\n\t}\n}\n\n// from iovisor/gobpf: bpf.NowNanoseconds()\n// nowNanoseconds returns a time that can be compared to bpf_ktime_get_ns()\nfunc nowNanoseconds() uint64 {\n\tvar ts syscall.Timespec\n\tsyscall.Syscall(syscall.SYS_CLOCK_GETTIME, 1 /* CLOCK_MONOTONIC */, uintptr(unsafe.Pointer(&ts)), 0)\n\tsec, nsec := ts.Unix()\n\treturn 1000*1000*1000*uint64(sec) + uint64(nsec)\n}\n\n// Collect implements prometheus.Collector interface\nfunc (c collector) Collect(ch chan<- prometheus.Metric) {\n\tvar (\n\t\twg sync.WaitGroup\n\n\t\tkey       uint64\n\t\tperCPUVal []uint32\n\t)\n\n\tcgroupids := make(map[uint64]uint32)\n\tlastCPUs := make(map[string]uint32)\n\n\tcg := cgroups.NewCgroupID(c.root)\n\n\tm := c.ebpf.Maps[\"avx_context_switch_count_hash\"]\n\titer := m.Iterate()\n\n\tfor iter.Next(&key, &perCPUVal) {\n\t\tvar sum uint32\n\t\tfor cpuID, count := range perCPUVal {\n\t\t\tif count == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsum = sum + count\n\n\t\t\tcpuX := fmt.Sprintf(\"CPU%d\", cpuID)\n\t\t\tlastCPUs[cpuX] = lastCPUs[cpuX] + count\n\n\t\t}\n\t\tcgroupids[key] = sum\n\t\tlog.Debug(\"cgroupid %d => counter %d\", key, sum)\n\n\t\t// reset the counter by deleting the key\n\t\terr := m.Delete(key)\n\t\tif err != nil {\n\t\t\tlog.Error(\"%+v\", err)\n\t\t}\n\t}\n\tif iter.Err() != nil {\n\t\tlog.Error(\"unable to iterate all elements of avx_context_switch_count: %+v\", iter.Err())\n\t}\n\n\tfor lastCPU, count := range lastCPUs {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[lastCPUDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(count),\n\t\t\tlastCPU)\n\t}\n\n\tfor cgroupid, counter := range cgroupids {\n\t\twg.Add(1)\n\t\tgo func(cgroupid_ uint64, counter_ uint32) {\n\t\t\tvar allCount uint32\n\t\t\tvar lastUpdate uint64\n\n\t\t\tdefer wg.Done()\n\n\t\t\tpath, err := cg.Find(cgroupid_)\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(\"failed to find cgroup by id: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tre := regexp.MustCompile(`[a-z0-9]{64}`)\n\t\t\tmatches := re.FindStringSubmatch(filepath.Base(path))\n\t\t\tif len(matches) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tch <- prometheus.MustNewConstMetric(\n\t\t\t\tdescriptors[avxSwitchCountDesc],\n\t\t\t\tprometheus.GaugeValue,\n\t\t\t\tfloat64(counter_),\n\t\t\t\tmatches[0])\n\n\t\t\tif err := c.ebpf.Maps[\"all_context_switch_count_hash\"].Lookup(uint64(cgroupid_), &allCount); err != nil {\n\t\t\t\tlog.Error(\"unable to find 'all' context switch count: %+v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tlog.Debug(\"all: %d\", allCount)\n\n\t\t\tif err := c.ebpf.Maps[\"last_update_ns_hash\"].Lookup(uint64(cgroupid_), &lastUpdate); err != nil {\n\t\t\t\tlog.Error(\"unable to find last update timestamp: %+v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tlog.Debug(\"last: %d\", lastUpdate)\n\n\t\t\tch <- prometheus.MustNewConstMetric(\n\t\t\t\tdescriptors[allSwitchCountDesc],\n\t\t\t\tprometheus.GaugeValue,\n\t\t\t\tfloat64(allCount),\n\t\t\t\tre.FindStringSubmatch(filepath.Base(path))[0])\n\n\t\t\tch <- prometheus.MustNewConstMetric(\n\t\t\t\tdescriptors[lastUpdateNsDesc],\n\t\t\t\tprometheus.GaugeValue,\n\t\t\t\tfloat64(nowNanoseconds()-lastUpdate),\n\t\t\t\tre.FindStringSubmatch(filepath.Base(path))[0])\n\n\t\t}(cgroupid, counter)\n\t}\n\n\t// We need to wait so that the response channel doesn't get closed.\n\twg.Wait()\n\n\tm = c.ebpf.Maps[\"all_context_switch_count_hash\"]\n\titer = m.Iterate()\n\n\tvar val uint32\n\tfor iter.Next(&key, &val) {\n\t\t// reset the counter by deleting the key\n\t\terr := m.Delete(key)\n\t\tif err != nil {\n\t\t\tlog.Error(\"%+v\", err)\n\t\t}\n\t}\n\n\tif iter.Err() != nil {\n\t\tlog.Error(\"unable to reset all elements of all_context_switch_count: %+v\", iter.Err())\n\t}\n}\n\nfunc init() {\n\tflag.StringVar(&bpfInstallpath, \"bpf-install-path\", bpfInstallpath,\n\t\t\"Path to eBPF install directory\")\n}\n"
  },
  {
    "path": "pkg/avx/elfdump.go",
    "content": "//go:build ignore\n// +build ignore\n\n/*\nCopyright 2020 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"text/template\"\n)\n\nconst (\n\tblocksPerRow = 12\n)\n\ntype Program struct {\n\tProgramLines []string\n}\n\nfunc main() {\n\tf, err := os.ReadFile(\"../../libexec/avx512.o\")\n\tif err != nil {\n\t\tfmt.Println(\"Note: AVX512 eBPF ELF not available.\")\n\t}\n\tenc := make([]byte, hex.EncodedLen(len(f)))\n\tenclen := hex.Encode(enc, f)\n\n\tvar j int\n\tvar row strings.Builder\n\tprogram := make([]string, 0)\n\n\tfor i := 0; i < enclen-1; i = i + 2 {\n\t\tfmt.Fprintf(&row, \"0x%s, \", enc[i:i+2])\n\t\tj++\n\t\tif j%blocksPerRow == 0 {\n\t\t\tprogram = append(program, row.String())\n\t\t\trow.Reset()\n\t\t}\n\t}\n\t// flush last row\n\tprogram = append(program, row.String())\n\n\tp := Program{\n\t\tProgramLines: program,\n\t}\n\n\ttemplate := template.Must(template.New(\"\").Parse(`// Code generated by go generate; DO NOT EDIT.\n\npackage avx\n\nvar program = [...]byte{\n{{- range .ProgramLines }}\n\t{{ printf \"%s\" . }}\n{{- end }}\n}\n`))\n\n\toutfile, err := os.Create(\"programbytes_gendata.go\")\n\tif err != nil {\n\t\tfmt.Println(\"elfdump:\", err)\n\t\tos.Exit(1)\n\t}\n\tdefer outfile.Close()\n\n\terr = template.Execute(outfile, p)\n\tif err != nil {\n\t\tfmt.Println(\"elfdump:\", err)\n\t}\n}\n"
  },
  {
    "path": "pkg/avx/register.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !noavx\n// +build !noavx\n\npackage avx\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/metrics\"\n)\n\nfunc init() {\n\terr := metrics.RegisterCollector(\"avx\", NewCollector)\n\tif err != nil {\n\t\tlog.Error(\"Failed to register AVX collector: %v\", err)\n\t}\n}\n"
  },
  {
    "path": "pkg/blockio/blockio.go",
    "content": "/*\nCopyright 2020 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage blockio\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"golang.org/x/sys/unix\"\n\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// ConfigModuleName is the configuration section of blockio class definitions\n\tConfigModuleName = \"blockio\"\n\n\t// sysfsBlockDeviceIOSchedulerPaths expands (with glob) to block device scheduler files.\n\t// If modified, check how to parse device node from expanded paths.\n\tsysfsBlockDeviceIOSchedulerPaths = \"/sys/block/*/queue/scheduler\"\n)\n\n// Class represents a block I/O class, a class name together with its associated\n// parameters, essentially a single key/value pair from staticOciBlockIO below.\n// This type is only used for querying all (static) block I/O classes in a sorting-\n// form.\ntype Class struct {\n\tName       string\n\tParameters cgroups.OciBlockIOParameters\n}\n\n// BlockDeviceInfo holds information on a block device to be configured.\n// As users can specify block devices using wildcards (\"/dev/disk/by-id/*SSD*\")\n// BlockDeviceInfo.Origin is maintained for traceability: why this\n// block device is included in configuration.\n// BlockDeviceInfo.DevNode contains resolved device node, like \"/dev/sda\".\ntype BlockDeviceInfo struct {\n\tMajor   int64\n\tMinor   int64\n\tDevNode string\n\tOrigin  string\n}\n\n// Our logger instance.\nvar log logger.Logger = logger.NewLogger(\"blockio\")\n\n// staticOciBlockIO connects user-defined block I/O classes to\n// corresponding OCI BlockIO parameters. \"Static\" means that\n// new/current block devices matching device wildcards in these\n// classes are not expanded every time new containers are assigned to\n// these classes. Devices are scanned on only at the beginning and on\n// blockio configuration changes.\nvar staticOciBlockIO = map[string]cgroups.OciBlockIOParameters{}\n\n// currentIOSchedulers contains io-schedulers (found in\n// sysfsBlockDeviceIOSchedulerPaths) of device nodes:\n// {\"/dev/sda\": \"bfq\"}\nvar currentIOSchedulers map[string]string\n\n// GetClasses returns block I/O class names and associated parameters in sorted slice.\nfunc GetClasses() []*Class {\n\tclasses := make([]*Class, 0, len(staticOciBlockIO))\n\tfor name, params := range staticOciBlockIO {\n\t\tclasses = append(classes, &Class{Name: name, Parameters: params})\n\t}\n\tsort.Slice(classes, func(i, j int) bool {\n\t\treturn strings.Compare(classes[i].Name, classes[j].Name) < 0\n\t})\n\treturn classes\n}\n\n// UpdateOciConfig converts the configuration in the opt variable into staticOciBlockIO\nfunc UpdateOciConfig(ignoreErrors bool) error {\n\tcurrentIOSchedulers, ioSchedulerDetectionError := getCurrentIOSchedulers()\n\tif ioSchedulerDetectionError != nil {\n\t\tlog.Warn(\"configuration validation partly disabled due to IO scheduler detection error %#v\", ioSchedulerDetectionError.Error())\n\t}\n\n\tstaticOciBlockIO = map[string]cgroups.OciBlockIOParameters{}\n\t// Create static OCI BlockIO structures for each blockio class\n\tfor class := range opt.Classes {\n\t\tociBlockIO, err := devicesParametersToOci(opt.Classes[class], currentIOSchedulers)\n\t\tif err != nil {\n\t\t\tif ignoreErrors {\n\t\t\t\tlog.Error(\"ignoring: %v\", err)\n\t\t\t} else {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\t// Handle all configurations as static for now. That\n\t\t// is, the list of block devices matching Devices\n\t\t// wildcards will not be updated without new\n\t\t// configNotify(). class.DynamicDevices not supported\n\t\t// yet.\n\t\tstaticOciBlockIO[class] = ociBlockIO\n\t}\n\treturn nil\n}\n\n// SetContainerClass assigns the pod in a container to a blockio class.\nfunc SetContainerClass(c cache.Container, class string) error {\n\tociBlockIO, classIsStatic := staticOciBlockIO[class]\n\tif !classIsStatic {\n\t\treturn blockioError(\"no OCI BlockIO parameters for class %#v\", class)\n\t}\n\n\tblkioCgroupRoot := cgroups.Blkio.Path()\n\tcontainerCgroupDir := c.GetCgroupDir()\n\tif containerCgroupDir == \"\" {\n\t\treturn blockioError(\"failed to find cgroup directory for container %s under %#v, container id %#v\", c.PrettyName(), blkioCgroupRoot, c.GetID())\n\t}\n\tcontainerCgroupPath := filepath.Join(blkioCgroupRoot, containerCgroupDir)\n\n\terr := cgroups.ResetBlkioParameters(containerCgroupPath, ociBlockIO)\n\tif err != nil {\n\t\treturn blockioError(\"assigning container %v to class %#v failed: %w\", c.PrettyName(), class, err)\n\t}\n\n\treturn nil\n}\n\n// getCurrentIOSchedulers returns currently active io-scheduler used for each block device in the system.\nfunc getCurrentIOSchedulers() (map[string]string, error) {\n\tvar ios = map[string]string{}\n\tschedulerFiles, err := filepath.Glob(sysfsBlockDeviceIOSchedulerPaths)\n\tif err != nil {\n\t\treturn ios, blockioError(\"error in IO scheduler wildcards %#v: %w\", sysfsBlockDeviceIOSchedulerPaths, err)\n\t}\n\tfor _, schedulerFile := range schedulerFiles {\n\t\tdevName := strings.SplitN(schedulerFile, \"/\", 5)[3]\n\t\tschedulerDataB, err := os.ReadFile(schedulerFile)\n\t\tif err != nil {\n\t\t\t// A block device may be disconnected. Continue without error.\n\t\t\tlog.Error(\"failed to read current IO scheduler %#v: %v\\n\", schedulerFile, err)\n\t\t\tcontinue\n\t\t}\n\t\tschedulerData := strings.Trim(string(schedulerDataB), \"\\n\")\n\t\tcurrentScheduler := \"\"\n\t\tif strings.IndexByte(schedulerData, ' ') == -1 {\n\t\t\tcurrentScheduler = schedulerData\n\t\t} else {\n\t\t\topenB := strings.Index(schedulerData, \"[\")\n\t\t\tcloseB := strings.Index(schedulerData, \"]\")\n\t\t\tif -1 < openB && openB < closeB {\n\t\t\t\tcurrentScheduler = schedulerData[openB+1 : closeB]\n\t\t\t}\n\t\t}\n\t\tif currentScheduler == \"\" {\n\t\t\treturn ios, blockioError(\"could not parse current scheduler in %#v\\n\", schedulerFile)\n\t\t}\n\n\t\tios[\"/dev/\"+devName] = currentScheduler\n\t}\n\treturn ios, nil\n}\n\n// deviceParametersToOci converts single blockio class parameters into OCI BlockIO structure.\nfunc devicesParametersToOci(dps []DevicesParameters, currentIOSchedulers map[string]string) (cgroups.OciBlockIOParameters, error) {\n\terrs := []error{}\n\toci := cgroups.NewOciBlockIOParameters()\n\tfor _, dp := range dps {\n\t\tvar err error\n\t\tvar weight, throttleReadBps, throttleWriteBps, throttleReadIOPS, throttleWriteIOPS int64\n\t\tweight, err = parseAndValidateInt64(\"Weight\", dp.Weight, -1, 10, 1000)\n\t\terrs = append(errs, err)\n\t\tthrottleReadBps, err = parseAndValidateInt64(\"ThrottleReadBps\", dp.ThrottleReadBps, -1, 0, -1)\n\t\terrs = append(errs, err)\n\t\tthrottleWriteBps, err = parseAndValidateInt64(\"ThrottleWriteBps\", dp.ThrottleWriteBps, -1, 0, -1)\n\t\terrs = append(errs, err)\n\t\tthrottleReadIOPS, err = parseAndValidateInt64(\"ThrottleReadIOPS\", dp.ThrottleReadIOPS, -1, 0, -1)\n\t\terrs = append(errs, err)\n\t\tthrottleWriteIOPS, err = parseAndValidateInt64(\"ThrottleWriteIOPS\", dp.ThrottleWriteIOPS, -1, 0, -1)\n\t\terrs = append(errs, err)\n\t\tif dp.Devices == nil {\n\t\t\tif weight > -1 {\n\t\t\t\toci.Weight = weight\n\t\t\t}\n\t\t\tif throttleReadBps > -1 || throttleWriteBps > -1 || throttleReadIOPS > -1 || throttleWriteIOPS > -1 {\n\t\t\t\terrs = append(errs, fmt.Errorf(\"ignoring throttling (rbps=%#v wbps=%#v riops=%#v wiops=%#v): Devices not listed\",\n\t\t\t\t\tdp.ThrottleReadBps, dp.ThrottleWriteBps, dp.ThrottleReadIOPS, dp.ThrottleWriteIOPS))\n\t\t\t}\n\t\t} else {\n\t\t\tblockDevices, err := currentPlatform.configurableBlockDevices(dp.Devices)\n\t\t\tif err != nil {\n\t\t\t\t// Problems in matching block device wildcards and resolving symlinks\n\t\t\t\t// are worth reporting, but must not block configuring blkio where possible.\n\t\t\t\tlog.Error(err.Error())\n\t\t\t}\n\t\t\tif len(blockDevices) == 0 {\n\t\t\t\tlog.Warn(\"no matches on any of Devices: %v, parameters ignored\", dp.Devices)\n\t\t\t}\n\t\t\tfor _, blockDeviceInfo := range blockDevices {\n\t\t\t\tif weight != -1 {\n\t\t\t\t\tif ios, found := currentIOSchedulers[blockDeviceInfo.DevNode]; found {\n\t\t\t\t\t\tif ios != \"bfq\" && ios != \"cfq\" {\n\t\t\t\t\t\t\tlog.Warn(\"weight has no effect on device %#v due to \"+\n\t\t\t\t\t\t\t\t\"incompatible io-scheduler %#v (bfq or cfq required)\", blockDeviceInfo.DevNode, ios)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\toci.WeightDevice.Update(blockDeviceInfo.Major, blockDeviceInfo.Minor, weight)\n\t\t\t\t}\n\t\t\t\tif throttleReadBps != -1 {\n\t\t\t\t\toci.ThrottleReadBpsDevice.Update(blockDeviceInfo.Major, blockDeviceInfo.Minor, throttleReadBps)\n\t\t\t\t}\n\t\t\t\tif throttleWriteBps != -1 {\n\t\t\t\t\toci.ThrottleWriteBpsDevice.Update(blockDeviceInfo.Major, blockDeviceInfo.Minor, throttleWriteBps)\n\t\t\t\t}\n\t\t\t\tif throttleReadIOPS != -1 {\n\t\t\t\t\toci.ThrottleReadIOPSDevice.Update(blockDeviceInfo.Major, blockDeviceInfo.Minor, throttleReadIOPS)\n\t\t\t\t}\n\t\t\t\tif throttleWriteIOPS != -1 {\n\t\t\t\t\toci.ThrottleWriteIOPSDevice.Update(blockDeviceInfo.Major, blockDeviceInfo.Minor, throttleWriteIOPS)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn oci, errors.Join(errs...)\n}\n\n// parseAndValidateInt64 parses quantities, like \"64 M\", and validates that they are in given range.\nfunc parseAndValidateInt64(fieldName string, fieldContent string,\n\tdefaultValue int64, min int64, max int64) (int64, error) {\n\t// Returns field content\n\tif fieldContent == \"\" {\n\t\treturn defaultValue, nil\n\t}\n\tqty, err := resource.ParseQuantity(fieldContent)\n\tif err != nil {\n\t\treturn defaultValue, fmt.Errorf(\"syntax error in %#v (%#v)\", fieldName, fieldContent)\n\t}\n\tvalue := qty.Value()\n\tif min != -1 && min > value {\n\t\treturn defaultValue, fmt.Errorf(\"value of %#v (%#v) smaller than minimum (%#v)\", fieldName, value, min)\n\t}\n\tif max != -1 && value > max {\n\t\treturn defaultValue, fmt.Errorf(\"value of %#v (%#v) bigger than maximum (%#v)\", fieldName, value, max)\n\t}\n\treturn value, nil\n}\n\n// platformInterface includes functions that access the system. Enables mocking the system.\ntype platformInterface interface {\n\tconfigurableBlockDevices(devWildcards []string) ([]BlockDeviceInfo, error)\n}\n\n// defaultPlatform versions of platformInterface functions access the underlying system.\ntype defaultPlatform struct{}\n\n// currentPlatform defines which platformInterface is used: defaultPlatform or a mock, for instance.\nvar currentPlatform platformInterface = defaultPlatform{}\n\n// configurableBlockDevices finds major:minor numbers for device filenames (wildcards allowed)\nfunc (dpm defaultPlatform) configurableBlockDevices(devWildcards []string) ([]BlockDeviceInfo, error) {\n\t// Return map {devNode: BlockDeviceInfo}\n\t// Example: {\"/dev/sda\": {Major:8, Minor:0, Origin:\"from symlink /dev/disk/by-id/ata-VendorXSSD from wildcard /dev/disk/by-id/*SSD*\"}}\n\terrs := []error{}\n\tblockDevices := []BlockDeviceInfo{}\n\tvar origin string\n\n\t// 1. Expand wildcards to device filenames (may be symlinks)\n\t// Example: devMatches[\"/dev/disk/by-id/ata-VendorSSD\"] == \"from wildcard \\\"dev/disk/by-id/*SSD*\\\"\"\n\tdevMatches := map[string]string{} // {devNodeOrSymlink: origin}\n\tfor _, devWildcard := range devWildcards {\n\t\tdevWildcardMatches, err := filepath.Glob(devWildcard)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"bad device wildcard %#v: %w\", devWildcard, err))\n\t\t\tcontinue\n\t\t}\n\t\tif len(devWildcardMatches) == 0 {\n\t\t\terrs = append(errs, fmt.Errorf(\"device wildcard %#v does not match any device nodes\", devWildcard))\n\t\t\tcontinue\n\t\t}\n\t\tfor _, devMatch := range devWildcardMatches {\n\t\t\tif devMatch != devWildcard {\n\t\t\t\torigin = fmt.Sprintf(\"from wildcard %#v\", devWildcard)\n\t\t\t} else {\n\t\t\t\torigin = \"\"\n\t\t\t}\n\t\t\tdevMatches[devMatch] = strings.TrimSpace(fmt.Sprintf(\"%v %v\", devMatches[devMatch], origin))\n\t\t}\n\t}\n\n\t// 2. Find out real device nodes behind symlinks\n\t// Example: devRealPaths[\"/dev/sda\"] == \"from symlink \\\"/dev/disk/by-id/ata-VendorSSD\\\"\"\n\tdevRealpaths := map[string]string{} // {devNode: origin}\n\tfor devMatch, devOrigin := range devMatches {\n\t\trealDevNode, err := filepath.EvalSymlinks(devMatch)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"cannot filepath.EvalSymlinks(%#v): %w\", devMatch, err))\n\t\t\tcontinue\n\t\t}\n\t\tif realDevNode != devMatch {\n\t\t\torigin = fmt.Sprintf(\"from symlink %#v %v\", devMatch, devOrigin)\n\t\t} else {\n\t\t\torigin = devOrigin\n\t\t}\n\t\tdevRealpaths[realDevNode] = strings.TrimSpace(fmt.Sprintf(\"%v %v\", devRealpaths[realDevNode], origin))\n\t}\n\n\t// 3. Filter out everything but block devices that are not partitions\n\t// Example: blockDevices[0] == {Major: 8, Minor: 0, DevNode: \"/dev/sda\", Origin: \"...\"}\n\tfor devRealpath, devOrigin := range devRealpaths {\n\t\torigin := \"\"\n\t\tif devOrigin != \"\" {\n\t\t\torigin = fmt.Sprintf(\" (origin: %s)\", devOrigin)\n\t\t}\n\t\tfileInfo, err := os.Stat(devRealpath)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"cannot os.Stat(%#v): %w%s\", devRealpath, err, origin))\n\t\t\tcontinue\n\t\t}\n\t\tfileMode := fileInfo.Mode()\n\t\tif fileMode&os.ModeDevice == 0 {\n\t\t\terrs = append(errs, fmt.Errorf(\"file %#v is not a device%s\", devRealpath, origin))\n\t\t\tcontinue\n\t\t}\n\t\tif fileMode&os.ModeCharDevice != 0 {\n\t\t\terrs = append(errs, fmt.Errorf(\"file %#v is a character device%s\", devRealpath, origin))\n\t\t\tcontinue\n\t\t}\n\t\tsys, ok := fileInfo.Sys().(*syscall.Stat_t)\n\t\tmajor := unix.Major(sys.Rdev)\n\t\tminor := unix.Minor(sys.Rdev)\n\t\tif !ok {\n\t\t\terrs = append(errs, fmt.Errorf(\"cannot get syscall stat_t from %#v: %w%s\", devRealpath, err, origin))\n\t\t\tcontinue\n\t\t}\n\t\tif minor&0xf != 0 {\n\t\t\terrs = append(errs, fmt.Errorf(\"skipping %#v: cannot weight/throttle partitions%s\", devRealpath, origin))\n\t\t\tcontinue\n\t\t}\n\t\tblockDevices = append(blockDevices, BlockDeviceInfo{\n\t\t\tMajor:   int64(major),\n\t\t\tMinor:   int64(minor),\n\t\t\tDevNode: devRealpath,\n\t\t\tOrigin:  devOrigin,\n\t\t})\n\t}\n\treturn blockDevices, errors.Join(errs...)\n}\n\n// blockioError creates a formatted error message.\nfunc blockioError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(format, args...)\n}\n"
  },
  {
    "path": "pkg/blockio/blockio_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage blockio\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/testutils\"\n)\n\nvar knownIOSchedulers = map[string]bool{\n\t\"bfq\":         true,\n\t\"cfq\":         true,\n\t\"deadline\":    true,\n\t\"kyber\":       true,\n\t\"mq-deadline\": true,\n\t\"none\":        true,\n\t\"noop\":        true,\n}\n\n// TestGetCurrentIOSchedulers: unit test for getCurrentIOSchedulers()\nfunc TestGetCurrentIOSchedulers(t *testing.T) {\n\tcurrentIOSchedulers, err := getCurrentIOSchedulers()\n\ttestutils.VerifyError(t, err, 0, nil)\n\tfor blockDev, ioScheduler := range currentIOSchedulers {\n\t\ts, ok := knownIOSchedulers[ioScheduler]\n\t\tif !ok || !s {\n\t\t\tt.Errorf(\"unknown io scheduler %#v on block device %#v\", ioScheduler, blockDev)\n\t\t}\n\t}\n}\n\n// TestConfigurableBlockDevices: unit tests for configurableBlockDevices()\nfunc TestConfigurableBlockDevices(t *testing.T) {\n\tsysfsBlockDevs, err := filepath.Glob(\"/sys/block/*\")\n\tif err != nil {\n\t\tsysfsBlockDevs = []string{}\n\t}\n\tdevBlockDevs := []string{}\n\tfor _, sysfsBlockDev := range sysfsBlockDevs {\n\t\tif strings.HasPrefix(sysfsBlockDev, \"/sys/block/sd\") || strings.HasPrefix(sysfsBlockDev, \"/sys/block/vd\") {\n\t\t\tdevBlockDevs = append(devBlockDevs, strings.Replace(sysfsBlockDev, \"/sys/block/\", \"/dev/\", 1))\n\t\t}\n\t}\n\tdevPartitions := []string{}\n\tfor _, devBlockDev := range devBlockDevs {\n\t\tdevPartitions, _ = filepath.Glob(devBlockDev + \"[0-9]\")\n\t\tif len(devPartitions) > 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\tt.Logf(\"test real block devices: %v\", devBlockDevs)\n\tt.Logf(\"test partitions: %v\", devPartitions)\n\ttcases := []struct {\n\t\tname                    string\n\t\tdevWildcards            []string\n\t\texpectedErrorCount      int\n\t\texpectedErrorSubstrings []string\n\t\texpectedMatches         int\n\t\tdisabled                bool\n\t\tdisabledReason          string\n\t}{\n\t\t{\n\t\t\tname:               \"no device wildcards\",\n\t\t\tdevWildcards:       nil,\n\t\t\texpectedErrorCount: 0,\n\t\t},\n\t\t{\n\t\t\tname:                    \"bad wildcard\",\n\t\t\tdevWildcards:            []string{\"/[-/verybadwildcard]\"},\n\t\t\texpectedErrorCount:      1,\n\t\t\texpectedErrorSubstrings: []string{\"verybadwildcard\", \"syntax error\"},\n\t\t},\n\t\t{\n\t\t\tname:                    \"not matching wildcard\",\n\t\t\tdevWildcards:            []string{\"/dev/path that should not exist/*\"},\n\t\t\texpectedErrorCount:      1,\n\t\t\texpectedErrorSubstrings: []string{\"does not match any\"},\n\t\t},\n\t\t{\n\t\t\tname:                    \"two wildcards: empty string and a character device\",\n\t\t\tdevWildcards:            []string{\"/dev/null\", \"\"},\n\t\t\texpectedErrorCount:      2,\n\t\t\texpectedErrorSubstrings: []string{\"\\\"/dev/null\\\" is a character device\", \"\\\"\\\" does not match any\"},\n\t\t},\n\t\t{\n\t\t\tname:                    \"not a device or even a file\",\n\t\t\tdevWildcards:            []string{\"/proc\", \"/proc/meminfo\", \"/proc/notexistingfile\"},\n\t\t\texpectedErrorCount:      3,\n\t\t\texpectedErrorSubstrings: []string{\"\\\"/proc\\\" is not a device\", \"\\\"/proc/meminfo\\\" is not a device\"},\n\t\t},\n\t\t{\n\t\t\tname:            \"real block devices\",\n\t\t\tdevWildcards:    devBlockDevs,\n\t\t\texpectedMatches: len(devBlockDevs),\n\t\t},\n\t\t{\n\t\t\tname:                    \"partition\",\n\t\t\tdevWildcards:            devPartitions,\n\t\t\texpectedErrorCount:      len(devPartitions),\n\t\t\texpectedErrorSubstrings: []string{\"cannot weight/throttle partitions\"},\n\t\t\tdisabled:                len(devPartitions) == 0,\n\t\t\tdisabledReason:          \"no block device partitions found\",\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tif tc.disabled {\n\t\t\t\tt.Skip(tc.disabledReason)\n\t\t\t}\n\t\t\trealPlatform := defaultPlatform{}\n\t\t\tbdis, err := realPlatform.configurableBlockDevices(tc.devWildcards)\n\t\t\ttestutils.VerifyError(t, err, tc.expectedErrorCount, tc.expectedErrorSubstrings)\n\t\t\tif len(bdis) != tc.expectedMatches {\n\t\t\t\tt.Errorf(\"expected %d matching block devices, got %d\", tc.expectedMatches, len(bdis))\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestDevicesParametersToOci: unit tests for devicesParametersToOci\nfunc TestDevicesParametersToOci(t *testing.T) {\n\t// switch real devicesParametersToOci to call mockPlatform.configurableBlockDevices\n\tcurrentPlatform = mockPlatform{}\n\ttcases := []struct {\n\t\tname                    string\n\t\tdps                     []DevicesParameters\n\t\tiosched                 map[string]string\n\t\texpectedOci             *cgroups.OciBlockIOParameters\n\t\texpectedErrorCount      int\n\t\texpectedErrorSubstrings []string\n\t}{\n\t\t{\n\t\t\tname: \"all OCI fields\",\n\t\t\tdps: []DevicesParameters{\n\t\t\t\t{\n\t\t\t\t\tWeight: \"144\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDevices:           []string{\"/dev/sda\"},\n\t\t\t\t\tThrottleReadBps:   \"1G\",\n\t\t\t\t\tThrottleWriteBps:  \"2M\",\n\t\t\t\t\tThrottleReadIOPS:  \"3k\",\n\t\t\t\t\tThrottleWriteIOPS: \"4\",\n\t\t\t\t\tWeight:            \"50\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tiosched: map[string]string{\"/dev/sda\": \"bfq\"},\n\t\t\texpectedOci: &cgroups.OciBlockIOParameters{\n\t\t\t\tWeight: 144,\n\t\t\t\tWeightDevice: cgroups.OciDeviceWeights{\n\t\t\t\t\t{Major: 11, Minor: 12, Weight: 50},\n\t\t\t\t},\n\t\t\t\tThrottleReadBpsDevice: cgroups.OciDeviceRates{\n\t\t\t\t\t{Major: 11, Minor: 12, Rate: 1000000000},\n\t\t\t\t},\n\t\t\t\tThrottleWriteBpsDevice: cgroups.OciDeviceRates{\n\t\t\t\t\t{Major: 11, Minor: 12, Rate: 2000000},\n\t\t\t\t},\n\t\t\t\tThrottleReadIOPSDevice: cgroups.OciDeviceRates{\n\t\t\t\t\t{Major: 11, Minor: 12, Rate: 3000},\n\t\t\t\t},\n\t\t\t\tThrottleWriteIOPSDevice: cgroups.OciDeviceRates{\n\t\t\t\t\t{Major: 11, Minor: 12, Rate: 4},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"later match overrides value\",\n\t\t\tdps: []DevicesParameters{\n\t\t\t\t{\n\t\t\t\t\tDevices:         []string{\"/dev/sda\", \"/dev/sdb\", \"/dev/sdc\"},\n\t\t\t\t\tThrottleReadBps: \"100\",\n\t\t\t\t\tWeight:          \"110\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDevices:         []string{\"/dev/sdb\", \"/dev/sdc\"},\n\t\t\t\t\tThrottleReadBps: \"300\",\n\t\t\t\t\tWeight:          \"330\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDevices:         []string{\"/dev/sdb\"},\n\t\t\t\t\tThrottleReadBps: \"200\",\n\t\t\t\t\tWeight:          \"220\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tiosched: map[string]string{\"/dev/sda\": \"bfq\", \"/dev/sdb\": \"bfq\", \"/dev/sdc\": \"cfq\"},\n\t\t\texpectedOci: &cgroups.OciBlockIOParameters{\n\t\t\t\tWeight: -1,\n\t\t\t\tWeightDevice: cgroups.OciDeviceWeights{\n\t\t\t\t\t{Major: 11, Minor: 12, Weight: 110},\n\t\t\t\t\t{Major: 21, Minor: 22, Weight: 220},\n\t\t\t\t\t{Major: 31, Minor: 32, Weight: 330},\n\t\t\t\t},\n\t\t\t\tThrottleReadBpsDevice: cgroups.OciDeviceRates{\n\t\t\t\t\t{Major: 11, Minor: 12, Rate: 100},\n\t\t\t\t\t{Major: 21, Minor: 22, Rate: 200},\n\t\t\t\t\t{Major: 31, Minor: 32, Rate: 300},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"invalid weights, many errors in different parameter sets\",\n\t\t\tdps: []DevicesParameters{\n\t\t\t\t{\n\t\t\t\t\tWeight: \"99999\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDevices: []string{\"/dev/sda\"},\n\t\t\t\t\tWeight:  \"1\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDevices: []string{\"/dev/sdb\"},\n\t\t\t\t\tWeight:  \"-2\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedErrorCount: 3,\n\t\t\texpectedErrorSubstrings: []string{\n\t\t\t\t\"(99999) bigger than maximum\",\n\t\t\t\t\"(1) smaller than minimum\",\n\t\t\t\t\"(-2) smaller than minimum\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"throttling without listing Devices\",\n\t\t\tdps: []DevicesParameters{\n\t\t\t\t{\n\t\t\t\t\tThrottleReadBps:   \"100M\",\n\t\t\t\t\tThrottleWriteIOPS: \"20k\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedErrorCount: 1,\n\t\t\texpectedErrorSubstrings: []string{\n\t\t\t\t\"Devices not listed\",\n\t\t\t\t\"\\\"100M\\\"\",\n\t\t\t\t\"\\\"20k\\\"\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\toci, err := devicesParametersToOci(tc.dps, tc.iosched)\n\t\t\ttestutils.VerifyError(t, err, tc.expectedErrorCount, tc.expectedErrorSubstrings)\n\t\t\tif tc.expectedOci != nil {\n\t\t\t\ttestutils.VerifyDeepEqual(t, \"OCI parameters\", *tc.expectedOci, oci)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// mockPlatform implements mock versions of platformInterface functions.\ntype mockPlatform struct{}\n\n// configurableBlockDevices mock always returns a set of block devices.\nfunc (mpf mockPlatform) configurableBlockDevices(devWildcards []string) ([]BlockDeviceInfo, error) {\n\tblockDevices := []BlockDeviceInfo{}\n\tfor _, devWildcard := range devWildcards {\n\t\tif devWildcard == \"/dev/sda\" {\n\t\t\tblockDevices = append(blockDevices, BlockDeviceInfo{\n\t\t\t\tMajor:   11,\n\t\t\t\tMinor:   12,\n\t\t\t\tDevNode: devWildcard,\n\t\t\t\tOrigin:  fmt.Sprintf(\"from wildcards %v\", devWildcard),\n\t\t\t})\n\t\t} else if devWildcard == \"/dev/sdb\" {\n\t\t\tblockDevices = append(blockDevices, BlockDeviceInfo{\n\t\t\t\tMajor:   21,\n\t\t\t\tMinor:   22,\n\t\t\t\tDevNode: devWildcard,\n\t\t\t\tOrigin:  fmt.Sprintf(\"from wildcards %v\", devWildcard),\n\t\t\t})\n\t\t} else if devWildcard == \"/dev/sdc\" {\n\t\t\tblockDevices = append(blockDevices, BlockDeviceInfo{\n\t\t\t\tMajor:   31,\n\t\t\t\tMinor:   32,\n\t\t\t\tDevNode: devWildcard,\n\t\t\t\tOrigin:  fmt.Sprintf(\"from wildcards %v\", devWildcard),\n\t\t\t})\n\t\t}\n\t}\n\treturn blockDevices, nil\n}\n"
  },
  {
    "path": "pkg/blockio/config.go",
    "content": "/*\nCopyright 2020 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage blockio\n\nimport (\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// options captures our configurable parameters.\ntype options struct {\n\t// Classes define weights and throttling parameters for sets of devices.\n\tClasses map[string][]DevicesParameters `json:\",omitempty\"`\n}\n\n// DevicesParameters defines Block IO parameters for a set of devices.\ntype DevicesParameters struct {\n\tDevices           []string `json:\",omitempty\"`\n\tThrottleReadBps   string   `json:\",omitempty\"`\n\tThrottleWriteBps  string   `json:\",omitempty\"`\n\tThrottleReadIOPS  string   `json:\",omitempty\"`\n\tThrottleWriteIOPS string   `json:\",omitempty\"`\n\tWeight            string   `json:\",omitempty\"`\n}\n\n// Currently active set of \"raw\" options\nvar opt = defaultOptions().(*options)\n\n// defaultOptions returns a new instance of \"raw\" options set to their defaults\nfunc defaultOptions() interface{} {\n\treturn &options{}\n}\n\nfunc init() {\n\tpkgcfg.Register(ConfigModuleName, \"Block I/O class control\", opt, defaultOptions)\n}\n"
  },
  {
    "path": "pkg/cgroups/cgroupblkio.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cgroups\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\tblkioCgroupDir = \"/sys/fs/cgroup/blkio/\"\n)\n\n// logger\nvar log logger.Logger = logger.NewLogger(\"cgroupblkio\")\n\n// cgroups blkio parameter filenames.\nvar blkioWeightFiles = []string{\"blkio.bfq.weight\", \"blkio.weight\"}\nvar blkioWeightDeviceFiles = []string{\"blkio.bfq.weight_device\", \"blkio.weight_device\"}\nvar blkioThrottleReadBpsFiles = []string{\"blkio.throttle.read_bps_device\"}\nvar blkioThrottleWriteBpsFiles = []string{\"blkio.throttle.write_bps_device\"}\nvar blkioThrottleReadIOPSFiles = []string{\"blkio.throttle.read_iops_device\"}\nvar blkioThrottleWriteIOPSFiles = []string{\"blkio.throttle.write_iops_device\"}\n\n// OciBlockIOParameters contains OCI standard configuration of cgroups blkio parameters.\n//\n// Effects of Weight and Rate values in SetBlkioParameters():\n// Value  |  Effect\n// -------+-------------------------------------------------------------------\n//\n//\t  -1  |  Do not write to cgroups, value is missing\n//\t   0  |  Write to cgroups, will remove the setting as specified in cgroups blkio interface\n//\tother |  Write to cgroups, sets the value\ntype OciBlockIOParameters struct {\n\tWeight                  int64\n\tWeightDevice            OciDeviceWeights\n\tThrottleReadBpsDevice   OciDeviceRates\n\tThrottleWriteBpsDevice  OciDeviceRates\n\tThrottleReadIOPSDevice  OciDeviceRates\n\tThrottleWriteIOPSDevice OciDeviceRates\n}\n\n// OciDeviceWeight contains values for\n// - blkio.[io-scheduler].weight\ntype OciDeviceWeight struct {\n\tMajor  int64\n\tMinor  int64\n\tWeight int64\n}\n\n// OciDeviceRate contains values for\n// - blkio.throttle.read_bps_device\n// - blkio.throttle.write_bps_device\n// - blkio.throttle.read_iops_device\n// - blkio.throttle.write_iops_device\ntype OciDeviceRate struct {\n\tMajor int64\n\tMinor int64\n\tRate  int64\n}\n\n// OciDeviceWeights contains weights for devices\ntype OciDeviceWeights []OciDeviceWeight\n\n// OciDeviceRates contains throttling rates for devices\ntype OciDeviceRates []OciDeviceRate\n\n// OciDeviceParameters interface provides functions common to OciDeviceWeights and OciDeviceRates\ntype OciDeviceParameters interface {\n\tAppend(maj, min, val int64)\n\tUpdate(maj, min, val int64)\n}\n\n// Append appends (major, minor, value) to OciDeviceWeights slice.\nfunc (w *OciDeviceWeights) Append(maj, min, val int64) {\n\t*w = append(*w, OciDeviceWeight{Major: maj, Minor: min, Weight: val})\n}\n\n// Append appends (major, minor, value) to OciDeviceRates slice.\nfunc (r *OciDeviceRates) Append(maj, min, val int64) {\n\t*r = append(*r, OciDeviceRate{Major: maj, Minor: min, Rate: val})\n}\n\n// Update updates device weight in OciDeviceWeights slice, or appends it if not found.\nfunc (w *OciDeviceWeights) Update(maj, min, val int64) {\n\tfor index, devWeight := range *w {\n\t\tif devWeight.Major == maj && devWeight.Minor == min {\n\t\t\t(*w)[index].Weight = val\n\t\t\treturn\n\t\t}\n\t}\n\tw.Append(maj, min, val)\n}\n\n// Update updates device rate in OciDeviceRates slice, or appends it if not found.\nfunc (r *OciDeviceRates) Update(maj, min, val int64) {\n\tfor index, devRate := range *r {\n\t\tif devRate.Major == maj && devRate.Minor == min {\n\t\t\t(*r)[index].Rate = val\n\t\t\treturn\n\t\t}\n\t}\n\tr.Append(maj, min, val)\n}\n\n// NewOciBlockIOParameters creates new OciBlockIOParameters instance.\nfunc NewOciBlockIOParameters() OciBlockIOParameters {\n\treturn OciBlockIOParameters{\n\t\tWeight: -1,\n\t}\n}\n\n// NewOciDeviceWeight creates new OciDeviceWeight instance.\nfunc NewOciDeviceWeight() OciDeviceWeight {\n\treturn OciDeviceWeight{\n\t\tMajor:  -1,\n\t\tMinor:  -1,\n\t\tWeight: -1,\n\t}\n}\n\n// NewOciDeviceRate creates new OciDeviceRate instance.\nfunc NewOciDeviceRate() OciDeviceRate {\n\treturn OciDeviceRate{\n\t\tMajor: -1,\n\t\tMinor: -1,\n\t\tRate:  -1,\n\t}\n}\n\n// GetBlkioDir returns the cgroups blkio controller directory.\nfunc GetBlkioDir() string {\n\treturn blkioCgroupDir\n}\n\ntype devMajMin struct {\n\tMajor int64\n\tMinor int64\n}\n\n// ResetBlkioParameters adds new, changes existing and removes missing blockIO parameters in cgroupsDir\nfunc ResetBlkioParameters(cgroupsDir string, blockIO OciBlockIOParameters) error {\n\terrs := []error{}\n\toldBlockIO, getErr := GetBlkioParameters(cgroupsDir)\n\terrs = append(errs, getErr)\n\tnewBlockIO := NewOciBlockIOParameters()\n\tnewBlockIO.Weight = blockIO.Weight\n\t// Set new device weights\n\tseenDev := map[devMajMin]bool{}\n\tfor _, ociWDP := range blockIO.WeightDevice {\n\t\tseenDev[devMajMin{ociWDP.Major, ociWDP.Minor}] = true\n\t\tnewBlockIO.WeightDevice = append(newBlockIO.WeightDevice, ociWDP)\n\t}\n\t// Reset old device weights that were missing from blockIO.WeightDevice\n\tfor _, ociWDP := range oldBlockIO.WeightDevice {\n\t\tif !seenDev[devMajMin{ociWDP.Major, ociWDP.Minor}] {\n\t\t\tnewBlockIO.WeightDevice = append(newBlockIO.WeightDevice, OciDeviceWeight{ociWDP.Major, ociWDP.Minor, 0})\n\t\t}\n\t}\n\tnewBlockIO.ThrottleReadBpsDevice = resetDevRates(oldBlockIO.ThrottleReadBpsDevice, blockIO.ThrottleReadBpsDevice)\n\tnewBlockIO.ThrottleWriteBpsDevice = resetDevRates(oldBlockIO.ThrottleWriteBpsDevice, blockIO.ThrottleWriteBpsDevice)\n\tnewBlockIO.ThrottleReadIOPSDevice = resetDevRates(oldBlockIO.ThrottleReadIOPSDevice, blockIO.ThrottleReadIOPSDevice)\n\tnewBlockIO.ThrottleWriteIOPSDevice = resetDevRates(oldBlockIO.ThrottleWriteIOPSDevice, blockIO.ThrottleWriteIOPSDevice)\n\terrs = append(errs, SetBlkioParameters(cgroupsDir, newBlockIO))\n\treturn errors.Join(errs...)\n}\n\n// resetDevRates adds wanted rate parameters to new and resets unwated rates\nfunc resetDevRates(old, wanted []OciDeviceRate) []OciDeviceRate {\n\trates := []OciDeviceRate{}\n\tseenDev := map[devMajMin]bool{}\n\tfor _, rdp := range wanted {\n\t\trates = append(rates, rdp)\n\t\tseenDev[devMajMin{rdp.Major, rdp.Minor}] = true\n\t}\n\tfor _, rdp := range old {\n\t\tif !seenDev[devMajMin{rdp.Major, rdp.Minor}] {\n\t\t\trates = append(rates, OciDeviceRate{rdp.Major, rdp.Minor, 0})\n\t\t}\n\t}\n\treturn rates\n}\n\n// GetBlkioParameters returns OCI BlockIO parameters from files in cgroups blkio controller directory.\nfunc GetBlkioParameters(cgroupsDir string) (OciBlockIOParameters, error) {\n\terrs := []error{}\n\tblockIO := NewOciBlockIOParameters()\n\tcontent, err := readFromFileInDir(cgroupsDir, blkioWeightFiles)\n\tif err == nil {\n\t\tweight, err := strconv.ParseInt(strings.TrimSuffix(content, \"\\n\"), 10, 64)\n\t\tif err == nil {\n\t\t\tblockIO.Weight = weight\n\t\t} else {\n\t\t\terrs = append(errs, fmt.Errorf(\"parsing weight from %#v failed: %w\", content, err))\n\t\t}\n\t} else {\n\t\terrs = append(errs, err)\n\t}\n\terrs = append(errs, readOciDeviceParameters(cgroupsDir, blkioWeightDeviceFiles, &blockIO.WeightDevice))\n\terrs = append(errs, readOciDeviceParameters(cgroupsDir, blkioThrottleReadBpsFiles, &blockIO.ThrottleReadBpsDevice))\n\terrs = append(errs, readOciDeviceParameters(cgroupsDir, blkioThrottleWriteBpsFiles, &blockIO.ThrottleWriteBpsDevice))\n\terrs = append(errs, readOciDeviceParameters(cgroupsDir, blkioThrottleReadIOPSFiles, &blockIO.ThrottleReadIOPSDevice))\n\terrs = append(errs, readOciDeviceParameters(cgroupsDir, blkioThrottleWriteIOPSFiles, &blockIO.ThrottleWriteIOPSDevice))\n\treturn blockIO, errors.Join(errs...)\n}\n\n// readOciDeviceParameters parses device lines used for weights and throttling rates\nfunc readOciDeviceParameters(baseDir string, filenames []string, params OciDeviceParameters) error {\n\terrs := []error{}\n\tcontents, err := readFromFileInDir(baseDir, filenames)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, line := range strings.Split(contents, \"\\n\") {\n\t\t// Device weight files may have \"default NNN\" line at the beginning. Skip it.\n\t\tif line == \"\" || strings.HasPrefix(line, \"default \") {\n\t\t\tcontinue\n\t\t}\n\t\t// Expect syntax MAJOR:MINOR VALUE\n\t\tdevVal := strings.Split(line, \" \")\n\t\tif len(devVal) != 2 {\n\t\t\terrs = append(errs, fmt.Errorf(\"invalid line %q, single space expected\", line))\n\t\t\tcontinue\n\t\t}\n\t\tmajMin := strings.Split(devVal[0], \":\")\n\t\tif len(majMin) != 2 {\n\t\t\terrs = append(errs, fmt.Errorf(\"invalid line %q, single colon expected before space\", line))\n\t\t\tcontinue\n\t\t}\n\t\tmajor, majErr := strconv.ParseInt(majMin[0], 10, 64)\n\t\tminor, minErr := strconv.ParseInt(majMin[1], 10, 64)\n\t\tvalue, valErr := strconv.ParseInt(devVal[1], 10, 64)\n\t\tif majErr != nil || minErr != nil || valErr != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"invalid number when parsing \\\"major:minor value\\\" from \\\"%s:%s %s\\\"\", majMin[0], majMin[1], devVal[1]))\n\t\t\tcontinue\n\t\t}\n\t\tparams.Append(major, minor, value)\n\t}\n\treturn errors.Join(errs...)\n}\n\n// readFromFileInDir returns content from the first successfully read file.\nfunc readFromFileInDir(baseDir string, filenames []string) (string, error) {\n\terrs := []error{}\n\t// If reading all the files fails, return list of read errors.\n\tfor _, filename := range filenames {\n\t\tfilepath := filepath.Join(baseDir, filename)\n\t\tcontent, err := currentPlatform.readFromFile(filepath)\n\t\tif err == nil {\n\t\t\treturn content, nil\n\t\t}\n\t\terrs = append(errs, err)\n\t}\n\terr := errors.Join(errs...)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"could not read any of files %q: %w\", filenames, err)\n\t}\n\treturn \"\", nil\n}\n\n// SetBlkioParameters writes OCI BlockIO parameters to files in cgroups blkio contoller directory.\nfunc SetBlkioParameters(cgroupsDir string, blockIO OciBlockIOParameters) error {\n\tlog.Debug(\"configuring cgroups blkio controller in directory %#v with parameters %+v\", cgroupsDir, blockIO)\n\terrs := []error{}\n\tif blockIO.Weight >= 0 {\n\t\terrs = append(errs, writeToFileInDir(cgroupsDir, blkioWeightFiles, strconv.FormatInt(blockIO.Weight, 10)))\n\t}\n\tfor _, weightDevice := range blockIO.WeightDevice {\n\t\terrs = append(errs, writeDevValueToFileInDir(cgroupsDir, blkioWeightDeviceFiles, weightDevice.Major, weightDevice.Minor, weightDevice.Weight))\n\t}\n\tfor _, rateDevice := range blockIO.ThrottleReadBpsDevice {\n\t\terrs = append(errs, writeDevValueToFileInDir(cgroupsDir, blkioThrottleReadBpsFiles, rateDevice.Major, rateDevice.Minor, rateDevice.Rate))\n\t}\n\tfor _, rateDevice := range blockIO.ThrottleWriteBpsDevice {\n\t\terrs = append(errs, writeDevValueToFileInDir(cgroupsDir, blkioThrottleWriteBpsFiles, rateDevice.Major, rateDevice.Minor, rateDevice.Rate))\n\t}\n\tfor _, rateDevice := range blockIO.ThrottleReadIOPSDevice {\n\t\terrs = append(errs, writeDevValueToFileInDir(cgroupsDir, blkioThrottleReadIOPSFiles, rateDevice.Major, rateDevice.Minor, rateDevice.Rate))\n\t}\n\tfor _, rateDevice := range blockIO.ThrottleWriteIOPSDevice {\n\t\terrs = append(errs, writeDevValueToFileInDir(cgroupsDir, blkioThrottleWriteIOPSFiles, rateDevice.Major, rateDevice.Minor, rateDevice.Rate))\n\t}\n\treturn errors.Join(errs...)\n}\n\n// writeDevValueToFileInDir writes MAJOR:MINOR VALUE to the first existing file under baseDir\nfunc writeDevValueToFileInDir(baseDir string, filenames []string, major, minor, value int64) error {\n\tcontent := fmt.Sprintf(\"%d:%d %d\", major, minor, value)\n\treturn writeToFileInDir(baseDir, filenames, content)\n}\n\n// writeToFileInDir writes content to the first existing file in the list under baseDir.\nfunc writeToFileInDir(baseDir string, filenames []string, content string) error {\n\terrs := []error{}\n\t// Returns list of errors from writes, list of single error due to all filenames missing or nil on success.\n\tfor _, filename := range filenames {\n\t\tfilepath := filepath.Join(baseDir, filename)\n\t\terr := currentPlatform.writeToFile(filepath, content)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\t\terrs = append(errs, err)\n\t}\n\terr := errors.Join(errs...)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"could not write content %#v to any of files %q: %w\", content, filenames, err)\n\t}\n\treturn nil\n}\n\n// platformInterface includes functions that access the system. Enables mocking the platform.\ntype platformInterface interface {\n\treadFromFile(filename string) (string, error)\n\twriteToFile(filename string, content string) error\n}\n\n// defaultPlatform versions of platformInterface functions access the underlying system.\ntype defaultPlatform struct{}\n\n// currentPlatform defines which platformInterface is used: defaultPlatform or a mock, for instance.\nvar currentPlatform platformInterface = defaultPlatform{}\n\n// readFromFile returns file contents as a string.\nfunc (dpm defaultPlatform) readFromFile(filename string) (string, error) {\n\tcontent, err := os.ReadFile(filename)\n\treturn string(content), err\n}\n\n// writeToFile writes content to an existing file.\nfunc (dpm defaultPlatform) writeToFile(filename string, content string) error {\n\tf, err := os.OpenFile(filename, os.O_WRONLY, 0666)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\t_, err = f.Write([]byte(content))\n\treturn err\n}\n"
  },
  {
    "path": "pkg/cgroups/cgroupblkio_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cgroups\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/testutils\"\n)\n\nfunc TestUpdateAppend(t *testing.T) {\n\ttcases := []struct {\n\t\tname                    string\n\t\tinputMajMinVals         [][]int64\n\t\tinputItem               []int64\n\t\texpectedMajMinVal       [][]int64\n\t\texpectedErrorCount      int\n\t\texpectedErrorSubstrings []string\n\t}{\n\t\t{\n\t\t\tname:              \"update empty list\",\n\t\t\tinputItem:         []int64{1, 2, 3},\n\t\t\texpectedMajMinVal: [][]int64{{1, 2, 3}},\n\t\t},\n\t\t{\n\t\t\tname:              \"update appends non-existing element\",\n\t\t\tinputMajMinVals:   [][]int64{{10, 20, 30}, {40, 50, 60}},\n\t\t\tinputItem:         []int64{1, 2, 3},\n\t\t\texpectedMajMinVal: [][]int64{{10, 20, 30}, {40, 50, 60}, {1, 2, 3}},\n\t\t},\n\t\t{\n\t\t\tname:              \"update the first existing element\",\n\t\t\tinputMajMinVals:   [][]int64{{10, 20, 30}, {40, 50, 60}, {40, 50, 60}},\n\t\t\tinputItem:         []int64{40, 50, 66},\n\t\t\texpectedMajMinVal: [][]int64{{10, 20, 30}, {40, 50, 66}, {40, 50, 60}},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tdevWeights := OciDeviceWeights{}\n\t\t\tdevRates := OciDeviceRates{}\n\t\t\texpDevWeights := OciDeviceWeights{}\n\t\t\texpDevRates := OciDeviceRates{}\n\t\t\tfor _, item := range tc.inputMajMinVals {\n\t\t\t\tdevWeights.Append(item[0], item[1], item[2])\n\t\t\t\tdevRates.Append(item[0], item[1], item[2])\n\t\t\t}\n\t\t\tdevWeights.Update(tc.inputItem[0], tc.inputItem[1], tc.inputItem[2])\n\t\t\tdevRates.Update(tc.inputItem[0], tc.inputItem[1], tc.inputItem[2])\n\t\t\tfor _, item := range tc.expectedMajMinVal {\n\t\t\t\texpDevWeights = append(expDevWeights, OciDeviceWeight{item[0], item[1], item[2]})\n\t\t\t\texpDevRates = append(expDevRates, OciDeviceRate{item[0], item[1], item[2]})\n\t\t\t}\n\t\t\ttestutils.VerifyDeepEqual(t, \"device weights\", expDevWeights, devWeights)\n\t\t\ttestutils.VerifyDeepEqual(t, \"device rates\", expDevRates, devRates)\n\t\t})\n\t}\n}\n\n// TestResetBlkioParameters: unit test for ResetBlkioParameters()\nfunc TestResetBlkioParameters(t *testing.T) {\n\ttcases := []struct {\n\t\tname                    string\n\t\tcgroupsDir              string\n\t\tblockIO                 OciBlockIOParameters\n\t\tfsContent               map[string]string\n\t\texpectedFsWrites        map[string]string\n\t\texpectedBlockIO         *OciBlockIOParameters\n\t\texpectedErrorCount      int\n\t\texpectedErrorSubstrings []string\n\t}{\n\t\t{\n\t\t\tname:       \"write to clean cgroups\",\n\t\t\tcgroupsDir: \"/write/to/clean\",\n\t\t\tblockIO: OciBlockIOParameters{\n\t\t\t\tWeight:                  222,\n\t\t\t\tWeightDevice:            OciDeviceWeights{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}},\n\t\t\t\tThrottleReadBpsDevice:   OciDeviceRates{{11, 12, 13}, {111, 112, 113}},\n\t\t\t\tThrottleWriteBpsDevice:  OciDeviceRates{{21, 22, 23}, {221, 222, 223}},\n\t\t\t\tThrottleReadIOPSDevice:  OciDeviceRates{{31, 32, 33}, {331, 332, 333}},\n\t\t\t\tThrottleWriteIOPSDevice: OciDeviceRates{{41, 42, 43}, {441, 442, 443}},\n\t\t\t},\n\t\t\tfsContent: map[string]string{\n\t\t\t\t\"/write/to/clean/blkio.bfq.weight\":                 \"100\\n\",\n\t\t\t\t\"/write/to/clean/blkio.bfq.weight_device\":          \"\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.read_bps_device\":   \"\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.write_bps_device\":  \"\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.read_iops_device\":  \"\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.write_iops_device\": \"\",\n\t\t\t},\n\t\t\texpectedFsWrites: map[string]string{\n\t\t\t\t\"/write/to/clean/blkio.bfq.weight\":                 \"222\",\n\t\t\t\t\"/write/to/clean/blkio.bfq.weight_device\":          \"1:2 3+4:5 6+7:8 9\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.read_bps_device\":   \"11:12 13+111:112 113\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.write_bps_device\":  \"21:22 23+221:222 223\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.read_iops_device\":  \"31:32 33+331:332 333\",\n\t\t\t\t\"/write/to/clean/blkio.throttle.write_iops_device\": \"41:42 43+441:442 443\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"reset all existing\",\n\t\t\tcgroupsDir: \"/reset/all\",\n\t\t\tblockIO:    NewOciBlockIOParameters(),\n\t\t\tfsContent: map[string]string{\n\t\t\t\t\"/reset/all/blkio.bfq.weight\":                 \"200\\n\",\n\t\t\t\t\"/reset/all/blkio.bfq.weight_device\":          \"default 200\\n1:2 3\\n4:5 6\\n\",\n\t\t\t\t\"/reset/all/blkio.throttle.read_bps_device\":   \"11:12 13\\n14:15 16\\n\",\n\t\t\t\t\"/reset/all/blkio.throttle.write_bps_device\":  \"21:22 23\\n\",\n\t\t\t\t\"/reset/all/blkio.throttle.read_iops_device\":  \"31:32 33\\n\",\n\t\t\t\t\"/reset/all/blkio.throttle.write_iops_device\": \"41:42 43\\n\",\n\t\t\t},\n\t\t\texpectedFsWrites: map[string]string{\n\t\t\t\t\"/reset/all/blkio.bfq.weight_device\":          \"1:2 0+4:5 0\",\n\t\t\t\t\"/reset/all/blkio.throttle.read_bps_device\":   \"11:12 0+14:15 0\",\n\t\t\t\t\"/reset/all/blkio.throttle.write_bps_device\":  \"21:22 0\",\n\t\t\t\t\"/reset/all/blkio.throttle.read_iops_device\":  \"31:32 0\",\n\t\t\t\t\"/reset/all/blkio.throttle.write_iops_device\": \"41:42 0\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"merge\",\n\t\t\tcgroupsDir: \"/merge\",\n\t\t\tblockIO: OciBlockIOParameters{\n\t\t\t\tWeight:                  80,\n\t\t\t\tWeightDevice:            OciDeviceWeights{{1, 2, 1113}, {7, 8, 9}},       // drop middle, update first, keep last\n\t\t\t\tThrottleReadBpsDevice:   OciDeviceRates{{11, 12, 13}},                    // keep the first entry\n\t\t\t\tThrottleWriteBpsDevice:  OciDeviceRates{{24, 25, 26}},                    // keep the last entry\n\t\t\t\tThrottleReadIOPSDevice:  OciDeviceRates{{31, 32, 33}, {331, 332, 333}},   // keep all\n\t\t\t\tThrottleWriteIOPSDevice: OciDeviceRates{{41, 42, 430}, {441, 442, 4430}}, // change all\n\t\t\t},\n\t\t\tfsContent: map[string]string{\n\t\t\t\t\"/merge/blkio.bfq.weight\":                 \"200\\n\",\n\t\t\t\t\"/merge/blkio.bfq.weight_device\":          \"default 200\\n1:2 3\\n4:5 6\\n7:8 9\",\n\t\t\t\t\"/merge/blkio.throttle.read_bps_device\":   \"11:12 13\\n14:15 16\\n\",\n\t\t\t\t\"/merge/blkio.throttle.write_bps_device\":  \"21:22 23\\n24:25 26\\n\",\n\t\t\t\t\"/merge/blkio.throttle.read_iops_device\":  \"31:32 33\\n331:332 333\\n\",\n\t\t\t\t\"/merge/blkio.throttle.write_iops_device\": \"41:42 43\\n441:442 443\\n\",\n\t\t\t},\n\t\t\texpectedFsWrites: map[string]string{\n\t\t\t\t\"/merge/blkio.bfq.weight\":                 \"80\",\n\t\t\t\t\"/merge/blkio.bfq.weight_device\":          \"1:2 1113+7:8 9+4:5 0\",\n\t\t\t\t\"/merge/blkio.throttle.read_bps_device\":   \"11:12 13+14:15 0\",\n\t\t\t\t\"/merge/blkio.throttle.write_bps_device\":  \"24:25 26+21:22 0\",\n\t\t\t\t\"/merge/blkio.throttle.read_iops_device\":  \"31:32 33+331:332 333\",\n\t\t\t\t\"/merge/blkio.throttle.write_iops_device\": \"41:42 430+441:442 4430\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tmpf := mockPlatform{\n\t\t\t\tfsOrigContent: tc.fsContent,\n\t\t\t\tfsWrites:      make(map[string]string),\n\t\t\t}\n\t\t\tcurrentPlatform = &mpf\n\t\t\terr := ResetBlkioParameters(tc.cgroupsDir, tc.blockIO)\n\t\t\ttestutils.VerifyError(t, err, tc.expectedErrorCount, tc.expectedErrorSubstrings)\n\t\t\tif tc.expectedFsWrites != nil {\n\t\t\t\ttestutils.VerifyDeepEqual(t, \"filesystem writes\", tc.expectedFsWrites, mpf.fsWrites)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestGetBlkioParameters: unit test for GetBlkioParameters()\nfunc TestGetBlkioParameters(t *testing.T) {\n\ttcases := []struct {\n\t\tname                    string\n\t\tcgroupsDir              string\n\t\treadsFail               int\n\t\tfsContent               map[string]string\n\t\texpectedBlockIO         *OciBlockIOParameters\n\t\texpectedErrorCount      int\n\t\texpectedErrorSubstrings []string\n\t}{\n\t\t{\n\t\t\tname:       \"empty files\",\n\t\t\tcgroupsDir: \"/empty/ok\",\n\t\t\tfsContent: map[string]string{\n\t\t\t\t\"/empty/ok/blkio.bfq.weight\":                 \"\",\n\t\t\t\t\"/empty/ok/blkio.bfq.weight_device\":          \"\",\n\t\t\t\t\"/empty/ok/blkio.throttle.read_bps_device\":   \"\",\n\t\t\t\t\"/empty/ok/blkio.throttle.write_bps_device\":  \"\",\n\t\t\t\t\"/empty/ok/blkio.throttle.read_iops_device\":  \"\",\n\t\t\t\t\"/empty/ok/blkio.throttle.write_iops_device\": \"\",\n\t\t\t},\n\t\t\texpectedBlockIO:         &OciBlockIOParameters{Weight: -1},\n\t\t\texpectedErrorCount:      1, // weight is not expected to be empty\n\t\t\texpectedErrorSubstrings: []string{\"parsing weight\"},\n\t\t},\n\t\t{\n\t\t\tname:       \"everything defined\",\n\t\t\tcgroupsDir: \"/read/ok\",\n\t\t\tfsContent: map[string]string{\n\t\t\t\t\"/read/ok/blkio.bfq.weight\": \"1\",\n\t\t\t\t// test weight_device file with real \"default\" line\n\t\t\t\t\"/read/ok/blkio.bfq.weight_device\": \"default 10\\n1:2 3\\n\",\n\t\t\t\t// test parsing two lines and skipping empty lines\n\t\t\t\t\"/read/ok/blkio.throttle.read_bps_device\": \"\\n11:22 33\\n\\n111:222 333\\n\",\n\t\t\t\t// test single line file\n\t\t\t\t\"/read/ok/blkio.throttle.write_bps_device\": \"1111:2222 3333\\n\",\n\t\t\t\t// test single line, missing LF at the end\n\t\t\t\t\"/read/ok/blkio.throttle.read_iops_device\": \"11111:22222 33333\",\n\t\t\t\t// test small and large values\n\t\t\t\t\"/read/ok/blkio.throttle.write_iops_device\": \"0:0 0\\n4294967296:4294967297 9223372036854775807\\n\",\n\t\t\t},\n\t\t\texpectedBlockIO: &OciBlockIOParameters{\n\t\t\t\tWeight:                  1,\n\t\t\t\tWeightDevice:            OciDeviceWeights{{1, 2, 3}},\n\t\t\t\tThrottleReadBpsDevice:   OciDeviceRates{{11, 22, 33}, {111, 222, 333}},\n\t\t\t\tThrottleWriteBpsDevice:  OciDeviceRates{{1111, 2222, 3333}},\n\t\t\t\tThrottleReadIOPSDevice:  OciDeviceRates{{11111, 22222, 33333}},\n\t\t\t\tThrottleWriteIOPSDevice: OciDeviceRates{{0, 0, 0}, {4294967296, 4294967297, 9223372036854775807}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"test bad lines\",\n\t\t\tcgroupsDir: \"read/bad\",\n\t\t\tfsContent: map[string]string{\n\t\t\t\t\"read/bad/blkio.bfq.weight\": \"xyz\",\n\t\t\t\t// test bad line in the middle\n\t\t\t\t\"read/bad/blkio.bfq.weight_device\": \"default 10\\n1:2 3\\nbad\\n4:5 6\\n\",\n\t\t\t\t// test no spaces\n\t\t\t\t\"read/bad/blkio.throttle.read_bps_device\": \"11:22:33\",\n\t\t\t\t// test too many spaces\n\t\t\t\t\"read/bad/blkio.throttle.write_bps_device\": \"1111 2222 3333 \\n\",\n\t\t\t\t// test no colons\n\t\t\t\t\"read/bad/blkio.throttle.read_iops_device\": \"1111122222 33333\",\n\t\t\t\t// test missing number\n\t\t\t\t\"read/bad/blkio.throttle.write_iops_device\": \"0: 0\\n\",\n\t\t\t},\n\t\t\texpectedErrorCount:      6,\n\t\t\texpectedErrorSubstrings: []string{\"bad\", \"xyz\", \"11:22:33\", \"1111 2222 3333 \", \"1111122222 33333\", \"0: 0\"},\n\t\t\texpectedBlockIO: &OciBlockIOParameters{\n\t\t\t\tWeight:       -1,\n\t\t\t\tWeightDevice: OciDeviceWeights{{1, 2, 3}, {4, 5, 6}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:               \"all files missing\",\n\t\t\tcgroupsDir:         \"/missing/err\",\n\t\t\tfsContent:          map[string]string{},\n\t\t\texpectedBlockIO:    &OciBlockIOParameters{Weight: -1},\n\t\t\texpectedErrorCount: 6,\n\t\t\texpectedErrorSubstrings: []string{\n\t\t\t\t\"file not found\",\n\t\t\t\t\"blkio.bfq.weight\",\n\t\t\t\t\"blkio.bfq.weight_device\",\n\t\t\t\t\"blkio.throttle.read_bps_device\",\n\t\t\t\t\"blkio.throttle.write_bps_device\",\n\t\t\t\t\"blkio.throttle.read_iops_device\",\n\t\t\t\t\"blkio.throttle.write_iops_device\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tmpf := mockPlatform{\n\t\t\t\tfsOrigContent: tc.fsContent,\n\t\t\t\treadsFail:     tc.readsFail,\n\t\t\t}\n\t\t\tcurrentPlatform = &mpf\n\t\t\tblockIO, err := GetBlkioParameters(tc.cgroupsDir)\n\t\t\ttestutils.VerifyError(t, err, tc.expectedErrorCount, tc.expectedErrorSubstrings)\n\t\t\tif tc.expectedBlockIO != nil {\n\t\t\t\ttestutils.VerifyDeepEqual(t, \"blockio parameters\", *tc.expectedBlockIO, blockIO)\n\t\t\t}\n\t\t})\n\t}\n\n}\n\n// TestSetBlkioParameters: unit test for SetBlkioParameters()\nfunc TestSetBlkioParameters(t *testing.T) {\n\ttcases := []struct {\n\t\tname                    string\n\t\tcgroupsDir              string\n\t\tblockIO                 OciBlockIOParameters\n\t\twritesFail              int\n\t\texpectedFsWrites        map[string]string\n\t\texpectedErrorCount      int\n\t\texpectedErrorSubstrings []string\n\t}{\n\t\t{\n\t\t\tname:       \"write full OCI struct\",\n\t\t\tcgroupsDir: \"/my/full\",\n\t\t\tblockIO: OciBlockIOParameters{\n\t\t\t\tWeight:                  10,\n\t\t\t\tWeightDevice:            OciDeviceWeights{{Major: 1, Minor: 2, Weight: 3}},\n\t\t\t\tThrottleReadBpsDevice:   OciDeviceRates{{Major: 11, Minor: 12, Rate: 13}},\n\t\t\t\tThrottleWriteBpsDevice:  OciDeviceRates{{Major: 21, Minor: 22, Rate: 23}},\n\t\t\t\tThrottleReadIOPSDevice:  OciDeviceRates{{Major: 31, Minor: 32, Rate: 33}},\n\t\t\t\tThrottleWriteIOPSDevice: OciDeviceRates{{Major: 41, Minor: 42, Rate: 43}},\n\t\t\t},\n\t\t\texpectedFsWrites: map[string]string{\n\t\t\t\t\"/my/full/blkio.bfq.weight\":                 \"10\",\n\t\t\t\t\"/my/full/blkio.bfq.weight_device\":          \"1:2 3\",\n\t\t\t\t\"/my/full/blkio.throttle.read_bps_device\":   \"11:12 13\",\n\t\t\t\t\"/my/full/blkio.throttle.write_bps_device\":  \"21:22 23\",\n\t\t\t\t\"/my/full/blkio.throttle.read_iops_device\":  \"31:32 33\",\n\t\t\t\t\"/my/full/blkio.throttle.write_iops_device\": \"41:42 43\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"write empty struct\",\n\t\t\tcgroupsDir: \"/my/empty\",\n\t\t\tblockIO:    OciBlockIOParameters{},\n\t\t\texpectedFsWrites: map[string]string{\n\t\t\t\t\"/my/empty/blkio.bfq.weight\": \"0\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"multidevice weight and throttling, no weight write on -1\",\n\t\t\tcgroupsDir: \"/my/multidev\",\n\t\t\tblockIO: OciBlockIOParameters{\n\t\t\t\tWeight:                  -1,\n\t\t\t\tWeightDevice:            OciDeviceWeights{{1, 2, 3}, {4, 5, 6}},\n\t\t\t\tThrottleReadBpsDevice:   OciDeviceRates{{11, 12, 13}, {111, 112, 113}},\n\t\t\t\tThrottleWriteBpsDevice:  OciDeviceRates{{21, 22, 23}, {221, 222, 223}},\n\t\t\t\tThrottleReadIOPSDevice:  OciDeviceRates{{31, 32, 33}, {331, 332, 333}},\n\t\t\t\tThrottleWriteIOPSDevice: OciDeviceRates{{41, 42, 43}, {441, 442, 443}},\n\t\t\t},\n\t\t\texpectedFsWrites: map[string]string{\n\t\t\t\t\"/my/multidev/blkio.bfq.weight_device\":          \"1:2 3+4:5 6\",\n\t\t\t\t\"/my/multidev/blkio.throttle.read_bps_device\":   \"11:12 13+111:112 113\",\n\t\t\t\t\"/my/multidev/blkio.throttle.write_bps_device\":  \"21:22 23+221:222 223\",\n\t\t\t\t\"/my/multidev/blkio.throttle.read_iops_device\":  \"31:32 33+331:332 333\",\n\t\t\t\t\"/my/multidev/blkio.throttle.write_iops_device\": \"41:42 43+441:442 443\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:             \"no bfq.weight\",\n\t\t\tcgroupsDir:       \"/my/nobfq\",\n\t\t\tblockIO:          OciBlockIOParameters{Weight: 100},\n\t\t\twritesFail:       1,\n\t\t\texpectedFsWrites: map[string]string{\"/my/nobfq/blkio.weight\": \"100\"},\n\t\t},\n\t\t{\n\t\t\tname:       \"all writes fail\",\n\t\t\tcgroupsDir: \"/my/writesfail\",\n\t\t\tblockIO: OciBlockIOParameters{\n\t\t\t\tWeight:       -1,\n\t\t\t\tWeightDevice: OciDeviceWeights{{1, 0, 100}},\n\t\t\t},\n\t\t\twritesFail:         9999,\n\t\t\texpectedErrorCount: 1,\n\t\t\texpectedErrorSubstrings: []string{\n\t\t\t\t\"could not write content \\\"1:0 100\\\" to any of files\",\n\t\t\t\t\"\\\"blkio.bfq.weight_device\\\"\",\n\t\t\t\t\"\\\"blkio.weight_device\\\"\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tmpf := mockPlatform{\n\t\t\t\tfsWrites:   make(map[string]string),\n\t\t\t\twritesFail: tc.writesFail,\n\t\t\t}\n\t\t\tcurrentPlatform = &mpf\n\t\t\terr := SetBlkioParameters(tc.cgroupsDir, tc.blockIO)\n\t\t\ttestutils.VerifyError(t, err, tc.expectedErrorCount, tc.expectedErrorSubstrings)\n\t\t\tif tc.expectedFsWrites != nil {\n\t\t\t\ttestutils.VerifyDeepEqual(t, \"filesystem writes\", tc.expectedFsWrites, mpf.fsWrites)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// mockPlatform implements mock versions of platformInterface functions.\ntype mockPlatform struct {\n\tfsOrigContent map[string]string\n\tfsWrites      map[string]string\n\treadsFail     int\n\twritesFail    int\n}\n\nfunc (mpf *mockPlatform) readFromFile(filename string) (string, error) {\n\tif mpf.readsFail > 0 {\n\t\tmpf.readsFail--\n\t\treturn \"\", fmt.Errorf(\"mockPlatofrm: reading from %#v failed\", filename)\n\t}\n\tif content, ok := mpf.fsOrigContent[filename]; ok {\n\t\treturn content, nil\n\t}\n\treturn \"\", fmt.Errorf(\"mockPlatform: file not found %#v\", filename)\n}\n\nfunc (mpf *mockPlatform) writeToFile(filename string, content string) error {\n\tvar newContent string\n\tif mpf.writesFail > 0 {\n\t\tmpf.writesFail--\n\t\treturn fmt.Errorf(\"mockPlatform: writing to %#v failed\", filename)\n\t}\n\tif oldContent, ok := mpf.fsWrites[filename]; ok {\n\t\tnewContent = fmt.Sprintf(\"%s+%s\", oldContent, content)\n\t} else {\n\t\tnewContent = content\n\t}\n\tmpf.fsWrites[filename] = newContent\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cgroups/cgroupcontrol.go",
    "content": "// Copyright 2020-2021 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cgroups\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\t\"syscall\"\n)\n\n// Controller is our enumerated type for cgroup controllers.\ntype Controller int\n\n// Group represents a control group.\ntype Group string\n\n// nolint\nconst (\n\t// UnkownController represents a controller of unknown type.\n\tUnknownController Controller = iota\n\t// blkio cgroup controller.\n\tBlkio\n\t// cpu cgroup controller.\n\tCpu\n\t// cpuacct cgroup controller.\n\tCpuacct\n\t// cpuset cgroup controller.\n\tCpuset\n\t// devices cgroup controller.\n\tDevices\n\t// freezer cgroup controller.\n\tFreezer\n\t// hugetlb cgroup controller.\n\tHugetlb\n\t// memory cgroup controller.\n\tMemory\n\t// net_cls cgroup controller.\n\tNetCls\n\t// net_prio cgroup controller.\n\tNetPrio\n\t// per_event cgroup controller.\n\tPerfEvent\n\t// pids cgroup controller.\n\tPids\n)\n\nvar (\n\t// controllerNames maps controllers to names/relative paths.\n\tcontrollerNames = map[Controller]string{\n\t\tBlkio:     \"blkio\",\n\t\tCpu:       \"cpu\",\n\t\tCpuacct:   \"cpuacct\",\n\t\tCpuset:    \"cpuset\",\n\t\tDevices:   \"devices\",\n\t\tFreezer:   \"freezer\",\n\t\tHugetlb:   \"hugetlb\",\n\t\tMemory:    \"memory\",\n\t\tNetCls:    \"net_cls\",\n\t\tNetPrio:   \"net_prio\",\n\t\tPerfEvent: \"perf_event\",\n\t\tPids:      \"pids\",\n\t}\n\n\t// controllerNames maps controllers to names/relative paths.\n\tcontrollerDirs = map[string]Controller{\n\t\t\"blkio\":      Blkio,\n\t\t\"cpu\":        Cpu,\n\t\t\"cpuacct\":    Cpuacct,\n\t\t\"cpuset\":     Cpuset,\n\t\t\"devices\":    Devices,\n\t\t\"freezer\":    Freezer,\n\t\t\"hugetlb\":    Hugetlb,\n\t\t\"memory\":     Memory,\n\t\t\"net_cls\":    NetCls,\n\t\t\"net_prio\":   NetPrio,\n\t\t\"perf_event\": PerfEvent,\n\t\t\"pids\":       Pids,\n\t}\n)\n\n// String returns the name of the given controller.\nfunc (c Controller) String() string {\n\tif name, ok := controllerNames[c]; ok {\n\t\treturn name\n\t}\n\treturn \"unknown\"\n}\n\n// Path returns the absolute path of the given controller.\nfunc (c Controller) Path() string {\n\tDetectSystemCgroupVersion()\n\tif systemCgroupVersion == 2 {\n\t\treturn GetMountDir()\n\t}\n\treturn path.Join(mountDir, c.String())\n}\n\n// RelPath returns the relative path of the given controller.\nfunc (c Controller) RelPath() string {\n\tDetectSystemCgroupVersion()\n\tif systemCgroupVersion == 2 {\n\t\treturn \"\"\n\t}\n\treturn c.String()\n}\n\n// Group returns the given group for the controller.\nfunc (c Controller) Group(group string) Group {\n\treturn Group(path.Join(c.Path(), group))\n}\n\n// AsGroup returns the group for the given absolute directory path.\nfunc AsGroup(absDir string) Group {\n\treturn Group(absDir)\n}\n\n// Controller returns the controller for the group.\nfunc (g Group) Controller() Controller {\n\tDetectSystemCgroupVersion()\n\tif systemCgroupVersion == 2 {\n\t\treturn UnknownController\n\t}\n\trelPath := strings.TrimPrefix(string(g), mountDir+\"/\")\n\tsplit := strings.SplitN(relPath, \"/\", 2)\n\tif len(split) > 0 {\n\t\treturn controllerDirs[split[0]]\n\t}\n\treturn UnknownController\n}\n\n// GetTasks reads the pids of threads currently assigned to the group.\nfunc (g Group) GetTasks() ([]string, error) {\n\treturn g.readPids(Tasks)\n}\n\n// GetProcesses reads the pids of processes currently assigned to the group.\nfunc (g Group) GetProcesses() ([]string, error) {\n\treturn g.readPids(Procs)\n}\n\n// AddTasks writes the given thread pids to the group.\nfunc (g Group) AddTasks(pids ...string) error {\n\treturn g.writePids(Tasks, pids...)\n}\n\n// AddProcesses writes the given process pids to the group.\nfunc (g Group) AddProcesses(pids ...string) error {\n\treturn g.writePids(Procs, pids...)\n}\n\n// Write writes the formatted data to the groups entry.\nfunc (g Group) Write(entry, format string, args ...interface{}) error {\n\tentryPath := path.Join(string(g), entry)\n\tf, err := os.OpenFile(entryPath, os.O_WRONLY, 0644)\n\tif err != nil {\n\t\treturn g.errorf(\"%q: failed to open: %v\", entry, err)\n\t}\n\tdefer f.Close()\n\n\tdata := fmt.Sprintf(format, args...)\n\tif _, err := f.Write([]byte(data)); err != nil {\n\t\treturn g.errorf(\"%q: failed to write %q: %v\", entry, data, err)\n\t}\n\n\treturn nil\n}\n\n// readPids reads pids from a cgroup's tasks or procs entry.\nfunc (g Group) readPids(entry string) ([]string, error) {\n\tvar pids []string\n\n\tpidFile := path.Join(string(g), entry)\n\n\tf, err := os.OpenFile(pidFile, os.O_RDONLY, 0644)\n\tif err != nil {\n\t\treturn nil, g.errorf(\"failed to open %q: %v\", entry, err)\n\t}\n\tdefer f.Close()\n\n\ts := bufio.NewScanner(f)\n\tfor s.Scan() {\n\t\tpids = append(pids, s.Text())\n\t}\n\tif s.Err() != nil {\n\t\treturn nil, g.errorf(\"failed to read %q: %v\", entry, err)\n\t}\n\n\treturn pids, nil\n}\n\n// writePids writes pids to a cgroup's tasks or procs entry.\nfunc (g Group) writePids(entry string, pids ...string) error {\n\tpidFile := path.Join(string(g), entry)\n\n\tf, err := os.OpenFile(pidFile, os.O_WRONLY, 0644)\n\tif err != nil {\n\t\treturn g.errorf(\"failed to write pids to %q: %v\", pidFile, err)\n\t}\n\tdefer f.Close()\n\n\tfor _, pid := range pids {\n\t\tif _, err := f.Write([]byte(pid)); err != nil {\n\t\t\tif !errors.Is(err, syscall.ESRCH) {\n\t\t\t\treturn g.errorf(\"failed to write pid %s to %q: %v\",\n\t\t\t\t\tpid, pidFile, err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// error returns a formatted group-specific error.\nfunc (g Group) errorf(format string, args ...interface{}) error {\n\tname := strings.TrimPrefix(string(g), mountDir+\"/\")\n\treturn fmt.Errorf(\"cgroup \"+name+\": \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cgroups/cgroupid.go",
    "content": "package cgroups\n\nimport (\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\n// CgroupID implements mapping kernel cgroup IDs to cgroupfs paths with transparent caching.\ntype CgroupID struct {\n\troot  string\n\tcache map[uint64]string\n\tsync.Mutex\n}\n\n// NewCgroupID creates a new CgroupID map/cache.\nfunc NewCgroupID(root string) *CgroupID {\n\treturn &CgroupID{\n\t\troot:  root,\n\t\tcache: make(map[uint64]string),\n\t}\n}\n\nfunc getID(path string) uint64 {\n\th, _, err := unix.NameToHandleAt(unix.AT_FDCWD, path, 0)\n\tif err != nil {\n\t\treturn 0\n\t}\n\n\treturn binary.LittleEndian.Uint64(h.Bytes())\n}\n\n// Find finds the path for the given cgroup id.\nfunc (cgid *CgroupID) Find(id uint64) (string, error) {\n\tfound := false\n\tvar p string\n\n\tcgid.Lock()\n\tdefer cgid.Unlock()\n\n\tif path, ok := cgid.cache[id]; ok {\n\t\treturn path, nil\n\t}\n\n\terr := filepath.Walk(cgid.root, func(path string, info os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tfmt.Printf(\"WalkFunc called with an error (path %q: %v\\n)\", path, err)\n\t\t\treturn err\n\t\t}\n\n\t\tif found {\n\t\t\treturn filepath.SkipDir\n\t\t}\n\n\t\tif info.IsDir() && id == getID(path) {\n\t\t\tfound = true\n\t\t\tp = path\n\t\t\treturn filepath.SkipDir\n\t\t}\n\t\treturn nil\n\t})\n\n\tif err != nil {\n\t\treturn \"\", err\n\t} else if !found {\n\t\treturn \"\", fmt.Errorf(\"cgroupid %v not found\", id)\n\t}\n\tcgid.cache[id] = p\n\treturn p, nil\n}\n"
  },
  {
    "path": "pkg/cgroups/cgrouppath.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cgroups\n\nimport (\n\t\"flag\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n)\n\n// nolint\nconst (\n\t// Tasks is a cgroup's \"tasks\" entry.\n\tTasks = \"tasks\"\n\t// Procs is cgroup's \"cgroup.procs\" entry.\n\tProcs = \"cgroup.procs\"\n\t// CpuShares is the cpu controller's \"cpu.shares\" entry.\n\tCpuShares = \"cpu.shares\"\n\t// CpuPeriod is the cpu controller's \"cpu.cfs_period_us\" entry.\n\tCpuPeriod = \"cpu.cfs_period_us\"\n\t// CpuQuota is the cpu controller's \"cpu.cfs_quota_us\" entry.\n\tCpuQuota = \"cpu.cfs_quota_us\"\n\t// CpusetCpus is the cpuset controller's cpuset.cpus entry.\n\tCpusetCpus = \"cpuset.cpus\"\n\t// CpusetMems is the cpuset controller's cpuset.mems entry.\n\tCpusetMems = \"cpuset.mems\"\n\t// Controllers is the cgroup v2 controllers file\n\tControllers = \"cgroup.controllers\"\n)\n\nvar (\n\t// mount is the parent directory for per-controller cgroupfs mounts.\n\tmountDir = \"/sys/fs/cgroup\"\n\t// v2Dir is the parent directory for per-controller cgroupfs mounts.\n\tv2Dir = path.Join(mountDir, \"unified\")\n\t// KubeletRoot is the --cgroup-root option the kubelet is running with.\n\tKubeletRoot = \"\"\n\t// detected system cgroup version, 0 is undetected\n\tsystemCgroupVersion = 0\n)\n\n// GetMountDir returns the common mount point for cgroup v1 controllers.\nfunc GetMountDir() string {\n\treturn mountDir\n}\n\n// SetMountDir sets the common mount point for the cgroup v1 controllers.\nfunc SetMountDir(dir string) {\n\tv2, _ := filepath.Rel(mountDir, v2Dir)\n\tmountDir = dir\n\tif v2 != \"\" {\n\t\tv2Dir = path.Join(mountDir, v2)\n\t}\n}\n\n// GetV2Dir() returns the cgroup v2 unified mount directory.\nfunc GetV2Dir() string {\n\treturn v2Dir\n}\n\n// SetV2Dir sets the unified cgroup v2 mount directory.\nfunc SetV2Dir(dir string) {\n\tif dir[0] == '/' {\n\t\tv2Dir = dir\n\t} else {\n\t\tv2Dir = path.Join(mountDir, v2Dir)\n\t}\n}\n\nfunc init() {\n\tflag.StringVar(&mountDir, \"cgroup-mount\", mountDir,\n\t\t\"directory under which cgroup v1 controllers are mounted\")\n\tflag.StringVar(&v2Dir, \"cgroup-v2-dir\",\n\t\tv2Dir, \"cgroup v2 unified mount directory\")\n\tflag.StringVar(&KubeletRoot, \"kubelet-cgroup-root\", KubeletRoot,\n\t\t\"--cgroup-root options the kubelet is running with\")\n}\n\nfunc DetectSystemCgroupVersion() int {\n\tif systemCgroupVersion == 0 {\n\t\tif _, err := os.Stat(path.Join(GetMountDir(), Controllers)); err == nil {\n\t\t\tsystemCgroupVersion = 2\n\t\t} else {\n\t\t\tsystemCgroupVersion = 1\n\t\t}\n\t}\n\treturn systemCgroupVersion\n}\n"
  },
  {
    "path": "pkg/cgroups/cgroupstats.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cgroups\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n)\n\n// BlkioDeviceBytes contains a single operations line of blkio.throttle.io_service_bytes_recursive file\ntype BlkioDeviceBytes struct {\n\tMajor      int\n\tMinor      int\n\tOperations map[string]int64\n}\n\n// BlkioThrottleBytes has parsed contents of blkio.throttle.io_service_bytes_recursive file\ntype BlkioThrottleBytes struct {\n\tDeviceBytes []*BlkioDeviceBytes\n\tTotalBytes  int64\n}\n\n// CPUAcctUsage has a parsed line of cpuacct.usage_all file\ntype CPUAcctUsage struct {\n\tCPU    int\n\tUser   int64\n\tSystem int64\n}\n\n// HugetlbUsage has parsed contents of huge pages usage in bytes.\ntype HugetlbUsage struct {\n\tSize     string\n\tBytes    int64\n\tMaxBytes int64\n}\n\n// MemoryUsage has parsed contents of memory usage in bytes.\ntype MemoryUsage struct {\n\tBytes    int64\n\tMaxBytes int64\n}\n\n// NumaLine represents one line in the NUMA statistics file.\ntype NumaLine struct {\n\tTotal int64\n\tNodes map[string]int64\n}\n\n// NumaStat has parsed contets of a NUMA statistics file.\ntype NumaStat struct {\n\tTotal       NumaLine\n\tFile        NumaLine\n\tAnon        NumaLine\n\tUnevictable NumaLine\n\n\tHierarchicalTotal       NumaLine\n\tHierarchicalFile        NumaLine\n\tHierarchicalAnon        NumaLine\n\tHierarchicalUnevictable NumaLine\n}\n\n// GlobalNumaStats has the statistics from one global NUMA nodestats file.\ntype GlobalNumaStats struct {\n\tNumaHit       int64\n\tNumaMiss      int64\n\tNumaForeign   int64\n\tInterleaveHit int64\n\tLocalNode     int64\n\tOtherNode     int64\n}\n\nfunc readCgroupFileLines(filePath string) ([]string, error) {\n\n\tf, err := os.ReadFile(filePath)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata := string(f)\n\n\trawLines := strings.Split(data, \"\\n\")\n\n\tlines := make([]string, 0)\n\n\t// Sanitize the lines and remove empty ones.\n\tfor _, rawLine := range rawLines {\n\t\tif len(strings.TrimSpace(rawLine)) > 0 {\n\t\t\tlines = append(lines, rawLine)\n\t\t}\n\t}\n\n\treturn lines, nil\n}\n\nfunc readCgroupSingleNumber(filePath string) (int64, error) {\n\n\t// File looks like this:\n\t//\n\t// 4\n\n\tlines, err := readCgroupFileLines(filePath)\n\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif len(lines) != 1 {\n\t\treturn 0, fmt.Errorf(\"error parsing file\")\n\t}\n\n\tnumber, err := strconv.ParseInt(lines[0], 10, 64)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn number, nil\n}\n\n// GetBlkioThrottleBytes returns amount of bytes transferred to/from the disk.\nfunc GetBlkioThrottleBytes(cgroupPath string) (BlkioThrottleBytes, error) {\n\tconst (\n\t\tcgroupEntry = \"blkio.throttle.io_service_bytes_recursive\"\n\t)\n\n\t// File looks like this:\n\t//\n\t// 8:16 Read 4223325184\n\t// 8:16 Write 3207528448\n\t// 8:16 Sync 5387592704\n\t// 8:16 Async 2043260928\n\t// 8:16 Discard 0\n\t// 8:16 Total 7430853632\n\t// 8:0 Read 5246572032\n\t// 8:0 Write 2361737216\n\t// 8:0 Sync 5575892480\n\t// 8:0 Async 2032416768\n\t// 8:0 Discard 0\n\t// 8:0 Total 7608309248\n\t// Total 15039162880\n\n\tentry := path.Join(cgroupPath, cgroupEntry)\n\tlines, err := readCgroupFileLines(entry)\n\tif err != nil {\n\t\treturn BlkioThrottleBytes{}, err\n\t}\n\n\tif len(lines) == 1 && lines[0] == \"Total 0\" {\n\t\treturn BlkioThrottleBytes{}, nil\n\t}\n\n\tresult := BlkioThrottleBytes{DeviceBytes: make([]*BlkioDeviceBytes, 0)}\n\tdevidx := map[string]int{}\n\n\tfor _, line := range lines {\n\t\tsplit := strings.Split(line, \" \")\n\t\tkey := split[0]\n\t\tif key == \"Total\" {\n\t\t\tif len(split) != 2 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttotalBytes, err := strconv.ParseInt(split[1], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn BlkioThrottleBytes{}, err\n\t\t\t}\n\t\t\tresult.TotalBytes = totalBytes\n\t\t} else {\n\t\t\tvar dev *BlkioDeviceBytes\n\n\t\t\tmajmin := strings.Split(key, \":\")\n\t\t\tif len(majmin) != 2 {\n\t\t\t\treturn BlkioThrottleBytes{}, fmt.Errorf(\"error parsing file %s\", entry)\n\t\t\t}\n\t\t\tmaj64, err := strconv.ParseInt(string(majmin[0]), 10, 32)\n\t\t\tif err != nil {\n\t\t\t\treturn BlkioThrottleBytes{}, err\n\t\t\t}\n\t\t\tmin64, err := strconv.ParseInt(string(majmin[1]), 10, 32)\n\t\t\tif err != nil {\n\t\t\t\treturn BlkioThrottleBytes{}, err\n\t\t\t}\n\t\t\tmajor := int(maj64)\n\t\t\tminor := int(min64)\n\n\t\t\tidx, ok := devidx[split[0]]\n\t\t\tif ok {\n\t\t\t\tdev = result.DeviceBytes[idx]\n\t\t\t} else {\n\t\t\t\tdev = &BlkioDeviceBytes{\n\t\t\t\t\tMajor:      major,\n\t\t\t\t\tMinor:      minor,\n\t\t\t\t\tOperations: make(map[string]int64),\n\t\t\t\t}\n\t\t\t\tidx = len(result.DeviceBytes)\n\t\t\t\tdevidx[key] = idx\n\t\t\t\tresult.DeviceBytes = append(result.DeviceBytes, dev)\n\t\t\t}\n\n\t\t\top, count := split[1], split[2]\n\t\t\tbytes, err := strconv.ParseInt(count, 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn BlkioThrottleBytes{}, err\n\t\t\t}\n\t\t\tdev.Operations[op] = bytes\n\t\t}\n\t}\n\n\treturn result, nil\n}\n\n// GetCPUAcctStats retrieves CPU account statistics for a given cgroup.\nfunc GetCPUAcctStats(cgroupPath string) ([]CPUAcctUsage, error) {\n\n\t// File looks like this:\n\t//\n\t// cpu user system\n\t// 0 3723082232186 2456599218\n\t// 1 3748398003001 1149546796\n\n\tlines, err := readCgroupFileLines(path.Join(cgroupPath, \"cpuacct.usage_all\"))\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresult := make([]CPUAcctUsage, 0, len(lines)-1)\n\n\tfor _, line := range lines[1:] {\n\t\ttokens := strings.Split(line, \" \")\n\t\tif len(tokens) != 3 {\n\t\t\tcontinue\n\t\t}\n\t\tcpu, err := strconv.ParseInt(tokens[0], 10, 32)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tuser, err := strconv.ParseInt(tokens[1], 10, 64)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tsystem, err := strconv.ParseInt(tokens[2], 10, 64)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresult = append(result, CPUAcctUsage{CPU: int(cpu), User: user, System: system})\n\t}\n\treturn result, nil\n}\n\n// GetCPUSetMemoryMigrate returns boolean indicating whether memory migration is enabled.\nfunc GetCPUSetMemoryMigrate(cgroupPath string) (bool, error) {\n\n\t// File looks like this:\n\t//\n\t// 0\n\n\tnumber, err := readCgroupSingleNumber(path.Join(cgroupPath, \"cpuset.memory_migrate\"))\n\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif number == 0 {\n\t\treturn false, nil\n\t} else if number == 1 {\n\t\treturn true, nil\n\t}\n\n\treturn false, fmt.Errorf(\"error parsing file\")\n}\n\n// GetHugetlbUsage retrieves huge pages statistics for a given cgroup.\nfunc GetHugetlbUsage(cgroupPath string) ([]HugetlbUsage, error) {\n\tconst (\n\t\tprefix         = \"/hugetlb.\"\n\t\tusageSuffix    = \".usage_in_bytes\"\n\t\tmaxUsageSuffix = \".max_usage_in_bytes\"\n\t)\n\n\t// Files look like this:\n\t//\n\t// 124\n\n\tusageFiles, err := filepath.Glob(path.Join(cgroupPath, prefix+\"*\"+usageSuffix))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresult := make([]HugetlbUsage, 0, len(usageFiles))\n\n\tfor _, file := range usageFiles {\n\t\tif strings.Contains(filepath.Base(file), \".rsvd\") {\n\t\t\t// Skip reservations files.\n\t\t\tcontinue\n\t\t}\n\t\tsize := strings.SplitN(filepath.Base(file), \".\", 3)[1]\n\t\tbytes, err := readCgroupSingleNumber(file)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tmax, err := readCgroupSingleNumber(strings.TrimSuffix(file, usageSuffix) + maxUsageSuffix)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresult = append(result, HugetlbUsage{\n\t\t\tSize:     size,\n\t\t\tBytes:    bytes,\n\t\t\tMaxBytes: max,\n\t\t})\n\t}\n\n\treturn result, nil\n}\n\n// GetMemoryUsage retrieves cgroup memory usage.\nfunc GetMemoryUsage(cgroupPath string) (MemoryUsage, error) {\n\n\t// Files look like this:\n\t//\n\t// 142\n\n\tusage, err := readCgroupSingleNumber(path.Join(cgroupPath, \"memory.usage_in_bytes\"))\n\tif err != nil {\n\t\treturn MemoryUsage{}, err\n\t}\n\n\tmaxUsage, err := readCgroupSingleNumber(path.Join(cgroupPath, \"memory.max_usage_in_bytes\"))\n\tif err != nil {\n\t\treturn MemoryUsage{}, err\n\t}\n\n\tresult := MemoryUsage{\n\t\tBytes:    usage,\n\t\tMaxBytes: maxUsage,\n\t}\n\n\treturn result, nil\n}\n\n// GetNumaStats returns parsed cgroup NUMA statistics.\nfunc GetNumaStats(cgroupPath string) (NumaStat, error) {\n\tconst (\n\t\tcgroupEntry = \"memory.numa_stat\"\n\t)\n\n\t// File looks like this:\n\t//\n\t// total=44611 N0=32631 N1=7501 N2=1982 N3=2497\n\t// file=44428 N0=32614 N1=7335 N2=1982 N3=2497\n\t// anon=183 N0=17 N1=166 N2=0 N3=0\n\t// unevictable=0 N0=0 N1=0 N2=0 N3=0\n\t// hierarchical_total=768133 N0=509113 N1=138887 N2=20464 N3=99669\n\t// hierarchical_file=722017 N0=496516 N1=119997 N2=20181 N3=85323\n\t// hierarchical_anon=46096 N0=12597 N1=18890 N2=283 N3=14326\n\t// hierarchical_unevictable=20 N0=0 N1=0 N2=0 N3=20\n\n\tentry := path.Join(cgroupPath, cgroupEntry)\n\tlines, err := readCgroupFileLines(entry)\n\tif err != nil {\n\t\treturn NumaStat{}, err\n\t}\n\n\tresult := NumaStat{}\n\tfor _, line := range lines {\n\t\tsplit := strings.Split(line, \" \")\n\t\tif len(line) < 2 {\n\t\t\treturn NumaStat{}, fmt.Errorf(\"error parsing file %s\", entry)\n\t\t}\n\n\t\tkeytotal := strings.Split(split[0], \"=\")\n\t\tif len(keytotal) != 2 {\n\t\t\treturn NumaStat{}, fmt.Errorf(\"error parsing file %s\", entry)\n\t\t}\n\t\tkey, tot := keytotal[0], keytotal[1]\n\n\t\ttotal, err := strconv.ParseInt(tot, 10, 64)\n\t\tif err != nil {\n\t\t\treturn NumaStat{}, fmt.Errorf(\"error parsing file %s: %v\", entry, err)\n\t\t}\n\n\t\tnodes := make(map[string]int64)\n\t\tfor _, nodeEntry := range split[1:] {\n\t\t\tnodeamount := strings.Split(nodeEntry, \"=\")\n\t\t\tif len(nodeamount) != 2 {\n\t\t\t\treturn NumaStat{}, fmt.Errorf(\"error parsing file %s\", entry)\n\t\t\t}\n\t\t\tnode, amount := nodeamount[0], nodeamount[1]\n\t\t\tnumber, err := strconv.ParseInt(amount, 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn NumaStat{}, fmt.Errorf(\"error parsing file %s: %v\", entry, err)\n\t\t\t}\n\t\t\tnodes[node] = number\n\t\t}\n\n\t\tswitch key {\n\t\tcase \"total\":\n\t\t\tresult.Total.Total = total\n\t\t\tresult.Total.Nodes = nodes\n\t\tcase \"file\":\n\t\t\tresult.File.Total = total\n\t\t\tresult.File.Nodes = nodes\n\t\tcase \"anon\":\n\t\t\tresult.Anon.Total = total\n\t\t\tresult.Anon.Nodes = nodes\n\t\tcase \"unevictable\":\n\t\t\tresult.Unevictable.Total = total\n\t\t\tresult.Unevictable.Nodes = nodes\n\t\tcase \"hierarchical_total\":\n\t\t\tresult.HierarchicalTotal.Total = total\n\t\t\tresult.HierarchicalTotal.Nodes = nodes\n\t\tcase \"hierarchical_file\":\n\t\t\tresult.HierarchicalFile.Total = total\n\t\t\tresult.HierarchicalFile.Nodes = nodes\n\t\tcase \"hierarchical_anon\":\n\t\t\tresult.HierarchicalAnon.Total = total\n\t\t\tresult.HierarchicalAnon.Nodes = nodes\n\t\tcase \"hierarchical_unevictable\":\n\t\t\tresult.HierarchicalUnevictable.Total = total\n\t\t\tresult.HierarchicalUnevictable.Nodes = nodes\n\t\tdefault:\n\t\t\treturn NumaStat{}, fmt.Errorf(\"error parsing file, unknown key %s\", key)\n\t\t}\n\t}\n\n\treturn result, nil\n}\n\n// GetGlobalNumaStats returns the global (non-cgroup) NUMA statistics per node.\nfunc GetGlobalNumaStats() (map[int]GlobalNumaStats, error) {\n\tconst (\n\t\tprefix = \"/sys/devices/system/node/node\"\n\t)\n\n\t// Files look like this:\n\t//\n\t// numa_hit 1851614569\n\t// numa_miss 0\n\t// numa_foreign 0\n\t// interleave_hit 49101\n\t// local_node 1851614569\n\t// other_node 0\n\n\tresult := make(map[int]GlobalNumaStats)\n\n\tnodeDirs, err := filepath.Glob(prefix + \"*\")\n\tif err != nil {\n\t\treturn map[int]GlobalNumaStats{}, err\n\t}\n\n\tfor _, dir := range nodeDirs {\n\t\tid := strings.TrimPrefix(dir, prefix)\n\t\tnode, err := strconv.ParseInt(id, 10, 0)\n\t\tif err != nil {\n\t\t\treturn map[int]GlobalNumaStats{}, fmt.Errorf(\"error parsing directory name\")\n\t\t}\n\n\t\tnodeStat := GlobalNumaStats{}\n\n\t\tnumastat := path.Join(dir, \"numastat\")\n\t\terr = sysfs.ParseFileEntries(numastat,\n\t\t\tmap[string]interface{}{\n\t\t\t\t\"numa_hit\":       &nodeStat.NumaHit,\n\t\t\t\t\"numa_miss\":      &nodeStat.NumaMiss,\n\t\t\t\t\"numa_foreign\":   &nodeStat.NumaForeign,\n\t\t\t\t\"interleave_hit\": &nodeStat.InterleaveHit,\n\t\t\t\t\"local_node\":     &nodeStat.LocalNode,\n\t\t\t\t\"other_node\":     &nodeStat.OtherNode,\n\t\t\t},\n\t\t\tfunc(line string) (string, string, error) {\n\t\t\t\tfields := strings.Fields(strings.TrimSpace(line))\n\t\t\t\tif len(fields) != 2 {\n\t\t\t\t\treturn \"\", \"\", fmt.Errorf(\"failed to parse line '%s'\", line)\n\t\t\t\t}\n\t\t\t\treturn fields[0], fields[1], nil\n\t\t\t},\n\t\t)\n\n\t\tif err != nil {\n\t\t\treturn map[int]GlobalNumaStats{}, fmt.Errorf(\"error parsing numastat file: %v\", err)\n\t\t}\n\n\t\tresult[int(node)] = nodeStat\n\t}\n\n\treturn result, nil\n}\n"
  },
  {
    "path": "pkg/cgroupstats/collector.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cgroupstats\n\nimport (\n\t\"flag\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/metrics\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\n// Prometheus Metric descriptor indices and descriptor table\nconst (\n\tnumaStatsDesc = iota\n\tmemoryUsageDesc\n\tmemoryMigrateDesc\n\tcpuAcctUsageDesc\n\thugeTlbUsageDesc\n\tblkioDeviceUsageDesc\n\tnumDescriptors\n)\n\nvar descriptors = [numDescriptors]*prometheus.Desc{\n\tnumaStatsDesc: prometheus.NewDesc(\n\t\t\"cgroup_numa_stats\",\n\t\t\"NUMA statistics for a given container and pod.\",\n\t\t[]string{\n\t\t\t// cgroup path\n\t\t\t\"container_id\",\n\t\t\t// NUMA node ID\n\t\t\t\"numa_node_id\",\n\t\t\t// NUMA memory type\n\t\t\t\"type\",\n\t\t}, nil,\n\t),\n\tmemoryUsageDesc: prometheus.NewDesc(\n\t\t\"cgroup_memory_usage\",\n\t\t\"Memory usage statistics for a given container and pod.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t\t\"type\",\n\t\t}, nil,\n\t),\n\tmemoryMigrateDesc: prometheus.NewDesc(\n\t\t\"cgroup_memory_migrate\",\n\t\t\"Memory migrate status for a given container and pod.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t}, nil,\n\t),\n\tcpuAcctUsageDesc: prometheus.NewDesc(\n\t\t\"cgroup_cpu_acct\",\n\t\t\"CPU accounting for a given container and pod.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t\t// CPU ID\n\t\t\t\"cpu\",\n\t\t\t\"type\",\n\t\t}, nil,\n\t),\n\thugeTlbUsageDesc: prometheus.NewDesc(\n\t\t\"cgroup_hugetlb_usage\",\n\t\t\"Hugepages usage for a given container and pod.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t\t\"size\",\n\t\t\t\"type\",\n\t\t}, nil,\n\t),\n\tblkioDeviceUsageDesc: prometheus.NewDesc(\n\t\t\"cgroup_blkio_device_usage\",\n\t\t\"Blkio Device bytes usage for a given container and pod.\",\n\t\t[]string{\n\t\t\t\"container_id\",\n\t\t\t\"major\",\n\t\t\t\"minor\",\n\t\t\t\"operation\",\n\t\t}, nil,\n\t),\n}\n\nvar (\n\t// cgroupRoot is the mount point for the cgroup (v1) filesystem\n\tcgroupRoot = \"/sys/fs/cgroup\"\n\t// our logger instance\n\tlog = logger.NewLogger(\"cgroupstats\")\n)\n\nconst (\n\tkubepodsDir = \"kubepods.slice\"\n)\n\ntype collector struct {\n}\n\n// NewCollector creates new Prometheus collector\nfunc NewCollector() (prometheus.Collector, error) {\n\treturn &collector{}, nil\n}\n\n// Describe implements prometheus.Collector interface\nfunc (c *collector) Describe(ch chan<- *prometheus.Desc) {\n\tfor _, d := range descriptors {\n\t\tch <- d\n\t}\n}\n\nfunc updateCPUAcctUsageMetric(ch chan<- prometheus.Metric, path string, metric []cgroups.CPUAcctUsage) {\n\tfor i, acct := range metric {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[cpuAcctUsageDesc],\n\t\t\tprometheus.CounterValue,\n\t\t\tfloat64(acct.CPU),\n\t\t\tpath, strconv.FormatInt(int64(i), 10), \"CPU\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[cpuAcctUsageDesc],\n\t\t\tprometheus.CounterValue,\n\t\t\tfloat64(acct.User),\n\t\t\tpath, strconv.FormatInt(int64(i), 10), \"User\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[cpuAcctUsageDesc],\n\t\t\tprometheus.CounterValue,\n\t\t\tfloat64(acct.System),\n\t\t\tpath, strconv.FormatInt(int64(i), 10), \"System\",\n\t\t)\n\t}\n}\n\nfunc updateMemoryMigrateMetric(ch chan<- prometheus.Metric, path string, migrate bool) {\n\tmigrateValue := 0\n\tif migrate {\n\t\tmigrateValue = 1\n\t}\n\tch <- prometheus.MustNewConstMetric(\n\t\tdescriptors[memoryMigrateDesc],\n\t\tprometheus.GaugeValue,\n\t\tfloat64(migrateValue),\n\t\tpath,\n\t)\n}\n\nfunc updateMemoryUsageMetric(ch chan<- prometheus.Metric, path string, metric cgroups.MemoryUsage) {\n\tch <- prometheus.MustNewConstMetric(\n\t\tdescriptors[memoryUsageDesc],\n\t\tprometheus.GaugeValue,\n\t\tfloat64(metric.Bytes),\n\t\tpath, \"Bytes\",\n\t)\n\tch <- prometheus.MustNewConstMetric(\n\t\tdescriptors[memoryUsageDesc],\n\t\tprometheus.GaugeValue,\n\t\tfloat64(metric.MaxBytes),\n\t\tpath, \"MaxBytes\",\n\t)\n}\n\nfunc updateNumaStatMetric(ch chan<- prometheus.Metric, path string, metric cgroups.NumaStat) {\n\t// TODO: use \"reflect\" to iterate through the struct fields of NumaStat?\n\n\tfor key, value := range metric.Total.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"Total\",\n\t\t)\n\t}\n\tfor key, value := range metric.File.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"File\",\n\t\t)\n\t}\n\tfor key, value := range metric.Anon.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"Anon\",\n\t\t)\n\t}\n\tfor key, value := range metric.Unevictable.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"Unevictable\",\n\t\t)\n\t}\n\tfor key, value := range metric.HierarchicalTotal.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"HierarchicalTotal\",\n\t\t)\n\t}\n\tfor key, value := range metric.HierarchicalFile.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"HierarchicalFile\",\n\t\t)\n\t}\n\tfor key, value := range metric.HierarchicalAnon.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"HierarchicalAnon\",\n\t\t)\n\t}\n\tfor key, value := range metric.HierarchicalUnevictable.Nodes {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[numaStatsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(value),\n\t\t\tpath, key, \"HierarchicalUnevictable\",\n\t\t)\n\t}\n}\n\nfunc updateHugeTlbUsageMetric(ch chan<- prometheus.Metric, path string, metric []cgroups.HugetlbUsage) {\n\t// One HugeTlbUsage for each size.\n\tfor _, hugeTlbUsage := range metric {\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[hugeTlbUsageDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(hugeTlbUsage.Bytes),\n\t\t\tpath, hugeTlbUsage.Size, \"Bytes\",\n\t\t)\n\t\tch <- prometheus.MustNewConstMetric(\n\t\t\tdescriptors[hugeTlbUsageDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(hugeTlbUsage.MaxBytes),\n\t\t\tpath, hugeTlbUsage.Size, \"MaxBytes\",\n\t\t)\n\t}\n}\n\nfunc updateBlkioDeviceUsageMetric(ch chan<- prometheus.Metric, path string, metric cgroups.BlkioThrottleBytes) {\n\tfor _, deviceBytes := range metric.DeviceBytes {\n\t\tfor operation, val := range deviceBytes.Operations {\n\t\t\tch <- prometheus.MustNewConstMetric(\n\t\t\t\tdescriptors[blkioDeviceUsageDesc],\n\t\t\t\tprometheus.CounterValue,\n\t\t\t\tfloat64(val),\n\t\t\t\tpath, strconv.FormatInt(int64(deviceBytes.Major), 10),\n\t\t\t\tstrconv.FormatInt(int64(deviceBytes.Minor), 10), operation,\n\t\t\t)\n\t\t}\n\t}\n}\n\nfunc walkCgroups() []string {\n\t// XXX TODO: add support for kubelet cgroupfs cgroup driver.\n\n\tcontainerDirs := []string{}\n\n\tcpuset := filepath.Join(cgroupRoot, \"cpuset\")\n\tfilepath.Walk(filepath.Join(cpuset, kubepodsDir),\n\t\tfunc(path string, info os.FileInfo, err error) error {\n\t\t\tif err != nil {\n\t\t\t\tif os.IsNotExist(err) {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif !info.IsDir() {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tdir := info.Name()\n\t\t\tif !strings.HasSuffix(dir, \".scope\") {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tswitch {\n\t\t\tcase strings.HasPrefix(dir, \"cri-containerd-\"):\n\t\t\t\tbreak\n\t\t\tcase strings.HasPrefix(dir, \"crio-\"):\n\t\t\t\tbreak\n\t\t\tcase strings.HasPrefix(dir, \"docker-\"):\n\t\t\t\tbreak\n\t\t\tdefault:\n\t\t\t\treturn filepath.SkipDir\n\t\t\t}\n\n\t\t\tpath = strings.TrimPrefix(path, cpuset+\"/\")\n\t\t\tcontainerDirs = append(containerDirs, path)\n\n\t\t\treturn nil\n\t\t})\n\n\treturn containerDirs\n}\n\nfunc cgroupPath(controller, path string) string {\n\treturn filepath.Join(cgroupRoot, controller, path)\n}\n\n// Collect implements prometheus.Collector interface\nfunc (c collector) Collect(ch chan<- prometheus.Metric) {\n\tvar wg sync.WaitGroup\n\n\t// We don't bail out on errors because those can happen if there is a race condition between\n\t// the destruction of a container and us getting to read the cgroup data. We just don't report\n\t// the values we don't get.\n\n\tcollectors := []func(string, *regexp.Regexp){\n\t\tfunc(path string, re *regexp.Regexp) {\n\t\t\tdefer wg.Done()\n\t\t\tnuma, err := cgroups.GetNumaStats(cgroupPath(\"memory\", path))\n\t\t\tif err == nil {\n\t\t\t\tupdateNumaStatMetric(ch, re.FindStringSubmatch(filepath.Base(path))[0], numa)\n\t\t\t} else {\n\t\t\t\tlog.Error(\"failed to collect NUMA stats for %s: %v\", path, err)\n\t\t\t}\n\t\t},\n\t\tfunc(path string, re *regexp.Regexp) {\n\t\t\tdefer wg.Done()\n\t\t\tmemory, err := cgroups.GetMemoryUsage(cgroupPath(\"memory\", path))\n\t\t\tif err == nil {\n\t\t\t\tupdateMemoryUsageMetric(ch, re.FindStringSubmatch(filepath.Base(path))[0], memory)\n\t\t\t} else {\n\t\t\t\tlog.Error(\"failed to collect memory usage stats for %s: %v\", path, err)\n\t\t\t}\n\t\t},\n\t\tfunc(path string, re *regexp.Regexp) {\n\t\t\tdefer wg.Done()\n\t\t\tmigrate, err := cgroups.GetCPUSetMemoryMigrate(cgroupPath(\"cpuset\", path))\n\t\t\tif err == nil {\n\t\t\t\tupdateMemoryMigrateMetric(ch, re.FindStringSubmatch(filepath.Base(path))[0], migrate)\n\t\t\t} else {\n\t\t\t\tlog.Error(\"failed to collect memory migration stats for %s: %v\", path, err)\n\t\t\t}\n\t\t},\n\t\tfunc(path string, re *regexp.Regexp) {\n\t\t\tdefer wg.Done()\n\t\t\tcpuAcctUsage, err := cgroups.GetCPUAcctStats(cgroupPath(\"cpuacct\", path))\n\t\t\tif err == nil {\n\t\t\t\tupdateCPUAcctUsageMetric(ch, re.FindStringSubmatch(filepath.Base(path))[0], cpuAcctUsage)\n\t\t\t} else {\n\t\t\t\tlog.Error(\"failed to collect CPU accounting stats for %s: %v\", path, err)\n\t\t\t}\n\t\t},\n\t\tfunc(path string, re *regexp.Regexp) {\n\t\t\tdefer wg.Done()\n\t\t\thugeTlbUsage, err := cgroups.GetHugetlbUsage(cgroupPath(\"hugetlb\", path))\n\t\t\tif err == nil {\n\t\t\t\tupdateHugeTlbUsageMetric(ch, re.FindStringSubmatch(filepath.Base(path))[0], hugeTlbUsage)\n\t\t\t} else {\n\t\t\t\tlog.Error(\"failed to collect hugetlb stats for %s: %v\", path, err)\n\t\t\t}\n\t\t},\n\t\tfunc(path string, re *regexp.Regexp) {\n\t\t\tdefer wg.Done()\n\t\t\tblkioDeviceUsage, err := cgroups.GetBlkioThrottleBytes(cgroupPath(\"blkio\", path))\n\t\t\tif err == nil {\n\t\t\t\tupdateBlkioDeviceUsageMetric(ch, re.FindStringSubmatch(filepath.Base(path))[0], blkioDeviceUsage)\n\t\t\t} else {\n\t\t\t\tlog.Error(\"failed to collect blkio stats for %s: %v\", path, err)\n\t\t\t}\n\t\t},\n\t}\n\n\tcontainerIDRegexp := regexp.MustCompile(`[a-z0-9]{64}`)\n\n\tfor _, path := range walkCgroups() {\n\t\twg.Add(len(collectors))\n\t\tfor _, fn := range collectors {\n\t\t\tgo fn(path, containerIDRegexp)\n\t\t}\n\t}\n\n\t// We need to wait so that the response channel doesn't get closed.\n\twg.Wait()\n}\n\nfunc init() {\n\tflag.StringVar(&cgroupRoot, \"cgroup-path\", cgroupRoot,\n\t\t\"Path to cgroup filesystem mountpoint\")\n\n\terr := metrics.RegisterCollector(\"cgroupstats\", NewCollector)\n\tif err != nil {\n\t\tlog.Error(\"failed register cgroupstats collector: %v\", err)\n\t}\n}\n"
  },
  {
    "path": "pkg/config/config.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"reflect\"\n\t\"sigs.k8s.io/yaml\"\n\t\"strings\"\n)\n\nconst (\n\t// MainModule is the default parent for all configuration.\n\tMainModule = \"main\"\n)\n\n// GetConfigFn is used to query a module for its default configuration.\ntype GetConfigFn func() interface{}\n\n// NotifyFn is used to notify a module about configuration changes.\ntype NotifyFn func(Event, Source) error\n\n// Event describes what triggered an invocation of a configuration notification callback.\ntype Event string\n\nconst (\n\t// UpdateEvent corresponds to a normal configuration udpate.\n\tUpdateEvent = \"update\"\n\t// RevertEvent corresponds to a configuration rollback in case of errors.\n\tRevertEvent = \"rollback\"\n)\n\n// Source describes where configuration is originated from.\ntype Source string\n\nconst (\n\t// ConfigFile is a YAML/JSON file configuration source.\n\tConfigFile Source = \"configuration file\"\n\t// ConfigExternal is an external configuration source, for instance a node agent.\n\tConfigExternal Source = \"external configuration\"\n\t// ConfigBackup is a backup of the previous configuration.\n\tConfigBackup Source = \"backed up configuration\"\n)\n\n// Module is a logical unit of configuration, declared using Declare().\ntype Module struct {\n\tpath        string             // fully qualified path in dotted notation, parent.name\n\tdescription string             // verbose module description\n\thelp        string             // verbose description/help about this module\n\tptr         interface{}        // pointer to module configuration data\n\tparent      *Module            // parent module\n\tname        string             // name relative to parent, last part of path\n\tchildren    map[string]*Module // modules nested under this module\n\tgetdefault  GetConfigFn        // getter for default configuration\n\tnotifiers   []NotifyFn         // update notification callbacks\n\tnoValidate  bool               // omit data validation\n}\n\n// main is the root of our configuration.\nvar main = &Module{\n\tpath:     MainModule,\n\tname:     MainModule,\n\tchildren: make(map[string]*Module),\n}\n\n// GetConfig returns the current configuration.\nfunc GetConfig() (Data, error) {\n\treturn main.getconfig()\n}\n\n// SetConfig updates the configuration using data from an external source.\nfunc SetConfig(cfg map[string]string) error {\n\tdata, err := DataFromStringMap(cfg)\n\tif err != nil {\n\t\treturn configError(\"failed to update configuration: %v\", err)\n\t}\n\treturn setconfig(data, ConfigExternal)\n}\n\n// SetConfigFromFile updates the configuration from the given file.\nfunc SetConfigFromFile(path string) error {\n\tdata, err := DataFromFile(path)\n\tif err != nil {\n\t\treturn configError(\"failed to apply configuration from file: %v\", err)\n\t}\n\treturn setconfig(data, ConfigFile)\n}\n\n// GetModule looks up the module for the given path, implicitly creating it if necessary.\nfunc GetModule(path string) *Module {\n\treturn lookup(path)\n}\n\n// AddNotify attaches the given update notification callback to the module.\nfunc (m *Module) AddNotify(fn NotifyFn) error {\n\treturn WithNotify(fn).apply(m)\n}\n\n// Register registers a unit of configuration data to be handled by this package.\nfunc Register(path, description string, ptr interface{}, getfn GetConfigFn, opts ...Option) *Module {\n\tm := lookup(path)\n\n\tif !m.isImplicit() {\n\t\tlog.Fatal(\"module %s: conflicting module with same path already declared (%s)\",\n\t\t\tpath, m.description)\n\t}\n\n\tm.setDescription(description)\n\tm.ptr = ptr\n\tm.getdefault = getfn\n\n\tm.check()\n\n\tforeign := m.notifiers\n\tm.notifiers = nil\n\tfor _, opt := range opts {\n\t\topt.apply(m)\n\t}\n\tm.notifiers = append(m.notifiers, foreign...)\n\n\treturn m\n}\n\n// setconfig updates the configuration, notifies all modules, and does a rollback if necessary.\nfunc setconfig(data Data, source Source) error {\n\tsnapshot, err := main.getconfig()\n\tif err != nil {\n\t\treturn configError(\"pre-update configuration snapshot failed: %v\", err)\n\t}\n\n\tlog.Info(\"validating configuration...\")\n\terr = main.validate(data)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tlog.Info(\"applying configuration...\")\n\terr = main.configure(data, false)\n\tif err != nil {\n\t\trevertconfig(snapshot, false)\n\t\treturn err\n\t}\n\n\tlog.Info(\"activating configuration...\")\n\terr = main.notify(UpdateEvent, source)\n\tif err != nil {\n\t\tlog.Error(\"configuration rejected: %v\", err)\n\t\trevertconfig(snapshot, true)\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// revertconfig reverts configuration using a previously taken snapshot\nfunc revertconfig(snapshot Data, notify bool) {\n\terr := main.configure(snapshot, true)\n\tif err != nil {\n\t\tlog.Error(\"failed to revert configuration: %v\", err)\n\t}\n\n\tif !notify {\n\t\treturn\n\t}\n\n\terr = main.notify(RevertEvent, ConfigBackup)\n\tif err != nil {\n\t\tlog.Error(\"reverted configuration rejected: %v\", err)\n\t}\n}\n\n// getconfig returns the configuration for the given module and its submodules.\nfunc (m *Module) getconfig() (Data, error) {\n\tvar mcfg, ccfg Data\n\tvar err error\n\n\tif m.isImplicit() {\n\t\tmcfg = make(Data)\n\t} else {\n\t\tmcfg, err = DataFromObject(m.ptr)\n\t\tif err != nil {\n\t\t\treturn nil, configError(\"module %s: failed to get confguration: %v\",\n\t\t\t\tm.path, err)\n\t\t}\n\t}\n\n\tfor name, child := range m.children {\n\t\tccfg, err = child.getconfig()\n\t\tif err != nil {\n\t\t\treturn nil, configError(\"module %s: failed to get child configuration for %s: %v\",\n\t\t\t\tm.path, child.path, err)\n\t\t}\n\t\tmcfg[name] = ccfg\n\t}\n\n\treturn mcfg, nil\n}\n\n// isImplict returns true if the module has not been explicitly declared.\nfunc (m *Module) isImplicit() bool {\n\treturn m.description == \"\"\n}\n\n// hasChild checks if the module has a child with the given name.\nfunc (m *Module) hasChild(name string) bool {\n\t_, ok := m.children[name]\n\treturn ok\n}\n\n// configure reconfigures the given module and its submodules with the provided data.\nfunc (m *Module) configure(data Data, force bool) error {\n\tlog.Debug(\"module %s: reconfiguring...\", m.path)\n\n\tmodcfg, subcfg := data.split(m.hasChild)\n\tif err := m.apply(modcfg); err != nil {\n\t\tif !force {\n\t\t\treturn err\n\t\t}\n\t\tlog.Error(\"%v\", err)\n\t}\n\n\tfor name, child := range m.children {\n\t\tchildcfg, err := subcfg.pick(name, true)\n\t\tif err != nil {\n\t\t\terr = configError(\"module %s: failed to pick configuration: %v\", child.path, err)\n\t\t\tif !force {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlog.Error(\"%v\", err)\n\t\t}\n\t\terr = child.configure(childcfg, force)\n\t\tif err != nil {\n\t\t\tif !force {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlog.Error(\"%v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// apply applies the given module-local configuration to the module.\nfunc (m *Module) apply(cfg Data) error {\n\tif m.isImplicit() {\n\t\treturn nil\n\t}\n\n\tlog.Debug(\"module %s: applying module configuration...\", m.path)\n\n\t// First, reset module config to defaults\n\tdefcfg, err := DataFromObject(m.getdefault())\n\tif err != nil {\n\t\treturn configError(\"module %s: failed to retrieve default configuration: %v\", m.path, err)\n\t}\n\traw, err := yaml.Marshal(defcfg)\n\tif err != nil {\n\t\treturn configError(\"module %s: failed to marshal default configuration: %v\", m.path, err)\n\t}\n\tif err = yaml.Unmarshal(raw, m.ptr); err != nil {\n\t\treturn configError(\"module %s: failed to pre-reset to default configuration: %v\", m.path, err)\n\t}\n\n\t// Second, apply given conf on top of the defaults\n\tif len(cfg) > 0 {\n\t\traw, err = yaml.Marshal(cfg)\n\t\tif err != nil {\n\t\t\treturn configError(\"module %s: failed to marshal configuration: %v\", m.path, err)\n\t\t}\n\t\tif err = yaml.Unmarshal(raw, m.ptr); err != nil {\n\t\t\treturn configError(\"module %s: failed to apply configuration: %v\", m.path, err)\n\t\t}\n\t}\n\treturn nil\n}\n\n// notify notifies this module and its children about a configuration change.\nfunc (m *Module) notify(event Event, source Source) error {\n\tfor _, child := range m.children {\n\t\tif err := child.notify(event, source); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tfor _, fn := range m.notifiers {\n\t\tif err := fn(event, source); err != nil {\n\t\t\treturn configError(\"module %s rejected %v configuration: %v\", m.path, event, err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// check performs basic sanity checks on the module.\nfunc (m *Module) check() {\n\tptrType := reflect.TypeOf(m.ptr)\n\tptr := reflect.ValueOf(m.ptr).Elem()\n\tif ptrType.Kind() != reflect.Ptr || ptr.Kind() != reflect.Struct {\n\t\tlog.Fatal(\"module %s: configuration data must be a pointer to a struct, not %T\",\n\t\t\tm.path, m.ptr)\n\t}\n\n\tif m.parent == nil || m.parent.isImplicit() {\n\t\treturn\n\t}\n\n\tptr = reflect.ValueOf(m.parent.ptr).Elem()\n\tfor i := 0; i < ptr.NumField(); i++ {\n\t\tfield := ptr.Type().Field(i)\n\t\tif m.name == fieldName(field) {\n\t\t\tlog.Fatal(\"module %s: parent has configuration data with conflicting field\", m.name)\n\t\t}\n\t}\n}\n\n// getFields() does a deep discovery of all fields of struct, handling also\n// embedded (i.e. struct composition) fields.\nfunc getFields(typ reflect.Type) map[string]struct{} {\n\tfields := make(map[string]struct{})\n\n\tvar get func(t reflect.Type)\n\tget = func(t reflect.Type) {\n\t\tfor i := 0; i < t.NumField(); i++ {\n\t\t\tf := t.Field(i)\n\n\t\t\tif f.Type.Kind() == reflect.Struct && f.Anonymous {\n\t\t\t\tget(f.Type)\n\t\t\t} else {\n\t\t\t\tfields[fieldName(f)] = struct{}{}\n\t\t\t}\n\t\t}\n\t}\n\n\tget(typ)\n\n\treturn fields\n}\n\n// validate checks that each field of data refers to either module data or a submodule.\nfunc (m *Module) validate(data Data) error {\n\tlog.Debug(\"validating data for module %s...\", m.path)\n\n\tmodcfg, subcfg := data.split(m.hasChild)\n\tfields := map[string]struct{}{}\n\n\tif m.isImplicit() {\n\t\tif len(modcfg) > 0 {\n\t\t\tnames := []string{}\n\t\t\tfor name := range modcfg {\n\t\t\t\tnames = append(names, name)\n\t\t\t}\n\t\t\tif !m.noValidate {\n\t\t\t\treturn configError(\"implicit module %s: given configuration data %s\",\n\t\t\t\t\tm.path, strings.Join(names, \",\"))\n\t\t\t}\n\t\t\tlog.Error(\"implicit module %s: given configuration date %s\",\n\t\t\t\tm.path, strings.Join(names, \",\"))\n\t\t}\n\t} else {\n\t\tfields = getFields(reflect.TypeOf(m.ptr).Elem())\n\t}\n\n\tfor field := range modcfg {\n\t\tif _, ok := fields[field]; !ok {\n\t\t\tif !m.noValidate {\n\t\t\t\treturn configError(\"module %s: given unknown configuration data %s\", m.path, field)\n\t\t\t}\n\t\t\tlog.Error(\"module %s: given unknown configuration data %s\", m.path, field)\n\t\t}\n\t}\n\n\tsubcfg = subcfg.copy()\n\tfor name, child := range m.children {\n\t\tchildcfg, err := subcfg.pick(name, true)\n\t\tif err != nil {\n\t\t\treturn configError(\"module %s: failed to pick configuration for child %s: %v\",\n\t\t\t\tm.path, child.path, err)\n\t\t}\n\t\terr = child.validate(childcfg)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif len(subcfg) > 0 {\n\t\tunconsumed := []string{}\n\t\tfor name := range subcfg {\n\t\t\tunconsumed = append(unconsumed, name)\n\t\t}\n\t\treturn configError(\"module %s: no child corresponding to data %s\",\n\t\t\tm.path, strings.Join(unconsumed, \",\"))\n\t}\n\n\treturn nil\n}\n\n// fieldName returns the name used to refer to the struct field in JSON/YAML encoding.\nfunc fieldName(f reflect.StructField) string {\n\tval, ok := f.Tag.Lookup(\"json\")\n\tif !ok {\n\t\treturn f.Name\n\t}\n\ttags := strings.Split(val, \",\")\n\tif len(tags) < 1 {\n\t\treturn f.Name\n\t}\n\tname := tags[0]\n\tif name == \"\" {\n\t\treturn f.Name\n\t}\n\treturn name\n}\n\n// lookup finds/creates a module corresponding to the given split module path.\nfunc lookup(path string) *Module {\n\tnames := strings.Split(path, \".\")\n\tpath = \"\"\n\tmodule := main\n\tfor _, name := range names {\n\t\tif path != \"\" {\n\t\t\tpath += \".\" + name\n\t\t} else {\n\t\t\tpath = name\n\t\t}\n\t\tm, ok := module.children[name]\n\t\tif !ok {\n\t\t\tm = &Module{\n\t\t\t\tpath:     path,\n\t\t\t\tparent:   module,\n\t\t\t\tname:     name,\n\t\t\t\tchildren: make(map[string]*Module),\n\t\t\t}\n\t\t\tmodule.children[name] = m\n\t\t}\n\t\tmodule = m\n\t}\n\treturn module\n}\n\n// Print prints the current configuration, using the given function or fmt.Printf.\nfunc Print(printfn func(string, ...interface{})) {\n\tdata, err := GetConfig()\n\tif err != nil {\n\t\tlog.Error(\"error: failed to get configuration: %v\", err)\n\t\treturn\n\t}\n\tdata.Print(printfn)\n}\n"
  },
  {
    "path": "pkg/config/data.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sigs.k8s.io/yaml\"\n\t\"strings\"\n)\n\n// Data is our internal representation of configuration data.\ntype Data map[string]interface{}\n\n// DataFromObject remarshals the given object into configuration data.\nfunc DataFromObject(obj interface{}) (Data, error) {\n\traw, err := yaml.Marshal(obj)\n\tif err != nil {\n\t\treturn nil, configError(\"failed to marshal object %T to data: %v\", obj, err)\n\t}\n\tdata := make(Data)\n\tif err = yaml.Unmarshal(raw, &data); err != nil {\n\t\treturn nil, configError(\"failed to unmarshal object %T to data: %v\", obj, err)\n\t}\n\treturn data, nil\n}\n\n// DataFromStringMap remarshals the given map into configuration data.\nfunc DataFromStringMap(smap map[string]string) (Data, error) {\n\tdata := make(Data)\n\tfor key, val := range smap {\n\t\tvar obj interface{}\n\t\tif err := yaml.Unmarshal([]byte(val), &obj); err != nil {\n\t\t\treturn nil, configError(\"failed to unmarshal data from map: %v\", err)\n\t\t}\n\t\tdata[key] = obj\n\t}\n\treturn data, nil\n}\n\n// DataFromFile unmarshals the content of the given file into configuration data.\nfunc DataFromFile(path string) (Data, error) {\n\traw, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn nil, configError(\"failed to read file %q: %v\", path, err)\n\t}\n\tdata := make(Data)\n\tif err := yaml.Unmarshal(raw, &data); err != nil {\n\t\treturn nil, configError(\"failed to load configuration from file %q: %v\", path, err)\n\t}\n\treturn data, nil\n}\n\n// copy does a shallow copy of the given data.\nfunc (d Data) copy() Data {\n\tdata := make(Data)\n\tfor key, value := range d {\n\t\tdata[key] = value\n\t}\n\treturn data\n}\n\n// split splits up the given data to module- and child-specific parts.\nfunc (d Data) split(hasChild func(string) bool) (Data, Data) {\n\tmod, sub := make(Data), make(Data)\n\tfor key, val := range d {\n\t\tif hasChild(key) || strings.IndexByte(key, '.') != -1 {\n\t\t\tsub[key] = val\n\t\t} else {\n\t\t\tmod[key] = val\n\t\t}\n\t}\n\treturn mod, sub\n}\n\n// pick picks data for the given key.\nfunc (d Data) pick(key string, removePicked bool) (Data, error) {\n\tvar data Data\n\tvar err error\n\n\tif obj, ok := d[key]; ok {\n\t\tdata, err = DataFromObject(obj)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif removePicked {\n\t\t\tdelete(d, key)\n\t\t}\n\t}\n\n\t// pick/remove data for all dotted keys matching the key being picked\n\tfor k, v := range d {\n\t\tsplit := strings.Split(k, \".\")\n\t\tif len(split) > 1 && split[0] == key {\n\t\t\tif data == nil {\n\t\t\t\tdata = make(Data)\n\t\t\t}\n\t\t\tsubkey := strings.Join(split[1:], \".\")\n\t\t\tif _, ok := data[subkey]; ok {\n\t\t\t\treturn nil, configError(\"dotted key %q conflicts with nested key %q\", k, subkey)\n\t\t\t}\n\t\t\tdata[subkey] = v\n\t\t\tif removePicked {\n\t\t\t\tdelete(d, k)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn data, nil\n}\n\n// String returns configuration data as a string.\nfunc (d Data) String() string {\n\traw, err := yaml.Marshal(d)\n\tif err != nil {\n\t\treturn fmt.Sprintf(\"<config.data: failed to marshal: %v>\", err)\n\t}\n\treturn string(raw)\n}\n\n// Print prints the configuration data using the given function or fmt.Printf.\nfunc (d Data) Print(fn func(string, ...interface{})) {\n\tif fn == nil {\n\t\tfn = func(format string, args ...interface{}) {\n\t\t\tfmt.Printf(format+\"\\n\", args...)\n\t\t}\n\t}\n\n\tfor _, line := range strings.Split(d.String(), \"\\n\") {\n\t\tfn(\"%s\", line)\n\t}\n}\n"
  },
  {
    "path": "pkg/config/duration.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"time\"\n)\n\n// Duration is a time.Duration which implements JSON marshalling/unmarshalling.\ntype Duration time.Duration\n\n// MarshalJSON is the JSON marshaller for (time.)Duration.\nfunc (d Duration) MarshalJSON() ([]byte, error) {\n\treturn []byte(\"\\\"\" + time.Duration(d).String() + \"\\\"\"), nil\n}\n\n// UnmarshalJSON is the JSON unmarshaller for (time.)Duration.\nfunc (d *Duration) UnmarshalJSON(data []byte) error {\n\tif len(data) < 2 {\n\t\treturn fmt.Errorf(\"invalid Duration data\")\n\t}\n\tparsed, err := time.ParseDuration(string(data[1 : len(data)-1]))\n\tif err != nil {\n\t\treturn err\n\t}\n\t*d = Duration(parsed)\n\treturn nil\n}\n\n// String returns the value of Duration as a string.\nfunc (d *Duration) String() string {\n\treturn time.Duration(*d).String()\n}\n"
  },
  {
    "path": "pkg/config/error.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n)\n\n// configError creates a formatted configuration-specific error.\nfunc configError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"config error: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/config/help.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n)\n\n// Describe provides help about configuration of the given modules.\nfunc Describe(names ...string) {\n\tmodules := findModules(names, nil)\n\n\tif len(modules) == 0 {\n\t\tfmt.Printf(\"No matching modules found.\\n\")\n\t\treturn\n\t}\n\n\tfor _, m := range modules {\n\t\tm.showHelp()\n\t\tfmt.Printf(\"\\n\\n\")\n\t}\n}\n\nfunc (m *Module) setDescription(description string) {\n\tdescription = strings.Trim(description, \"\\n\")\n\n\tif description == \"\" {\n\t\tm.description = \"Module \" + m.path + \" has no description.\"\n\t\treturn\n\t}\n\n\tif strings.IndexByte(description, '\\n') == -1 {\n\t\tm.description = description\n\t} else {\n\t\tlines := strings.Split(description, \"\\n\")\n\t\tm.description = lines[0]\n\t\tm.help = strings.Trim(strings.Join(lines[1:], \"\\n\"), \"\\n\")\n\t}\n}\n\nfunc (m *Module) showHelp() {\n\tkind := \"module\"\n\tif m.isImplicit() {\n\t\tkind = \"implicit module\"\n\t}\n\tfmt.Printf(\"- %s %s: %s\\n\", kind, m.name, m.description)\n\tfmt.Printf(\"  full path: %s\\n\", m.path)\n\tif len(m.children) > 0 {\n\t\tsubmodules, sep := \"\", \"\"\n\t\tfor _, child := range m.children {\n\t\t\tsubmodules += sep + child.path\n\t\t\tsep = \", \"\n\t\t}\n\t\tfmt.Printf(\"  sub-modules: %s\\n\", submodules)\n\t}\n\tfmt.Printf(\"  description:\\n\")\n\tif m.help != \"\" {\n\t\tfmt.Printf(\"\\n\")\n\t\tfor _, line := range strings.Split(m.help, \"\\n\") {\n\t\t\tfmt.Printf(\"    %s\\n\", line)\n\t\t}\n\t} else {\n\t\tm.describeData()\n\t}\n}\n\nfunc (m *Module) describeData() {\n\tif m.isImplicit() {\n\t\treturn\n\t}\n\n\tcfg := reflect.ValueOf(m.ptr).Elem()\n\tfmt.Printf(\"    No runtime configuration documentation for this package...\\n\")\n\tfmt.Printf(\"    Package runtime configuration data type: %s %s.\\n\",\n\t\tcfg.Type().Kind().String(), cfg.Type().String())\n}\n\nfunc findModules(names []string, m *Module) []*Module {\n\tif m == nil {\n\t\tm = main\n\t}\n\n\tmatches := []*Module{}\n\n\tif len(names) == 0 {\n\t\tmatches = append(matches, m)\n\t} else {\n\t\tfor _, name := range names {\n\t\t\tswitch {\n\t\t\tcase name == m.name || name == m.path:\n\t\t\t\tmatches = append(matches, m)\n\t\t\tcase name[0] == '.' && name[len(name)-1] == '.' && strings.Index(m.path, name) > 0:\n\t\t\t\tmatches = append(matches, m)\n\t\t\tcase name[0] == '.' && strings.HasSuffix(m.path, name):\n\t\t\t\tmatches = append(matches, m)\n\t\t\tcase name[len(name)-1] == '.' && strings.HasPrefix(m.path, name):\n\t\t\t\tmatches = append(matches, m)\n\t\t\t}\n\t\t}\n\t}\n\n\tchildren := []*Module{}\n\tfor _, child := range m.children {\n\t\tchildren = append(children, child)\n\t}\n\tsort.Slice(children,\n\t\tfunc(i, j int) bool {\n\t\t\treturn strings.Compare(children[i].path, children[j].path) < 0\n\t\t},\n\t)\n\tfor _, child := range children {\n\t\tmatches = append(matches, findModules(names, child)...)\n\t}\n\n\treturn matches\n}\n"
  },
  {
    "path": "pkg/config/log.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\n//\n// Notes:\n//   Unless we split the Logger interface (pkg/log.Logger) from its actual implementation\n//   we cannot import it import here. pkg/log itself implements its runtime configurability\n//   using this module so we would end up with an import cycle. As a workaround for now we\n//   let out logger be set externally and we set it from pkg/log.\n//\n\n// Logger is our set of logging functions.\ntype Logger struct {\n\tDebugEnabled func() bool\n\tDebug        func(string, ...interface{})\n\tInfo         func(string, ...interface{})\n\tWarning      func(string, ...interface{})\n\tError        func(string, ...interface{})\n\tFatal        func(string, ...interface{})\n\tPanic        func(string, ...interface{})\n}\n\n// log is our Logger.\nvar log = defaultLogger()\n\n// SetLogger sets our logger.\nfunc SetLogger(logger Logger) {\n\tif logger.DebugEnabled != nil {\n\t\tlog.DebugEnabled = logger.DebugEnabled\n\t}\n\tif logger.Debug != nil {\n\t\tlog.Debug = logger.Debug\n\t}\n\tif logger.Info != nil {\n\t\tlog.Info = logger.Info\n\t}\n\tif logger.Warning != nil {\n\t\tlog.Warning = logger.Warning\n\t}\n\tif logger.Error != nil {\n\t\tlog.Error = logger.Error\n\t}\n\tif logger.Panic != nil {\n\t\tlog.Panic = logger.Panic\n\t}\n\tif logger.Fatal != nil {\n\t\tlog.Fatal = logger.Fatal\n\t}\n}\n\nfunc defaultLogger() Logger {\n\treturn Logger{\n\t\tDebugEnabled: debugEnabled,\n\t\tDebug:        debugmsg,\n\t\tInfo:         infomsg,\n\t\tWarning:      warningmsg,\n\t\tError:        errormsg,\n\t\tFatal:        fatalmsg,\n\t\tPanic:        panicmsg,\n\t}\n}\n\nfunc debugEnabled() bool {\n\treturn true\n}\n\nfunc debugmsg(format string, args ...interface{}) {\n\tfmt.Printf(\"D: [config] \"+format+\"\\n\", args...)\n}\n\nfunc infomsg(format string, args ...interface{}) {\n\tfmt.Printf(\"I: [config] \"+format+\"\\n\", args...)\n}\n\nfunc warningmsg(format string, args ...interface{}) {\n\tfmt.Printf(\"W: [config] \"+format+\"\\n\", args...)\n}\n\nfunc errormsg(format string, args ...interface{}) {\n\tfmt.Printf(\"E: [config] \"+format+\"\\n\", args...)\n}\n\nfunc fatalmsg(format string, args ...interface{}) {\n\tfmt.Printf(\"E: [config] fatal error: \"+format+\"\\n\", args...)\n\tos.Exit(1)\n}\n\nfunc panicmsg(format string, args ...interface{}) {\n\terrormsg(format, args...)\n\tpanic(fmt.Sprintf(\"fatal error: \"+format+\"\\n\", args...))\n}\n"
  },
  {
    "path": "pkg/config/options.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\n// WithNotify specifies a notification function to be called after configuration updates.\nfunc WithNotify(fn NotifyFn) Option {\n\treturn newFuncOption(func(o interface{}) error {\n\t\tswitch o.(type) {\n\t\tcase *Module:\n\t\t\tm := o.(*Module)\n\t\t\tm.notifiers = append(m.notifiers, fn)\n\t\tdefault:\n\t\t\treturn configError(\"WithNotify is not valid for object of type %T\", o)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// WithoutDataValidation specifies that data passed to this module should not be validated.\nfunc WithoutDataValidation() Option {\n\treturn newFuncOption(func(o interface{}) error {\n\t\tswitch o.(type) {\n\t\tcase *Module:\n\t\t\tm := o.(*Module)\n\t\t\tm.noValidate = true\n\t\tdefault:\n\t\t\treturn configError(\"WithoutDataValidation is not valid for object of type %T\", o)\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Option is the generic interface for any option applicable to a Module or Config.\ntype Option interface {\n\tapply(interface{}) error\n}\n\n// funcOption is a generic functional option.\ntype funcOption struct {\n\tf func(interface{}) error\n}\n\n// apply applies a functional option to an object.\nfunc (fo *funcOption) apply(o interface{}) error {\n\treturn fo.f(o)\n}\n\n// newFuncOption creates a new option instance.\nfunc newFuncOption(f func(interface{}) error) *funcOption {\n\treturn &funcOption{\n\t\tf: f,\n\t}\n}\n"
  },
  {
    "path": "pkg/cpuallocator/allocator.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpuallocator\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/intel/goresctrl/pkg/sst\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// AllocFlag represents CPU allocation preferences.\ntype AllocFlag uint\n\nconst (\n\t// AllocIdlePackages requests allocation of full idle packages.\n\tAllocIdlePackages AllocFlag = 1 << iota\n\t// AllocIdleNodes requests allocation of full idle NUMA nodes.\n\tAllocIdleNodes\n\t// AllocIdleCores requests allocation of full idle cores (all threads in core).\n\tAllocIdleCores\n\t// AllocDefault is the default allocation preferences.\n\tAllocDefault = AllocIdlePackages | AllocIdleCores\n\n\tlogSource = \"cpuallocator\"\n)\n\n// allocatorHelper encapsulates state for allocating CPUs.\ntype allocatorHelper struct {\n\tlogger.Logger               // allocatorHelper logger instance\n\tsys           sysfs.System  // sysfs CPU and topology information\n\ttopology      topologyCache // cached topology information\n\tflags         AllocFlag     // allocation preferences\n\tfrom          cpuset.CPUSet // set of CPUs to allocate from\n\tprefer        CPUPriority   // CPU priority to prefer\n\tcnt           int           // number of CPUs to allocate\n\tresult        cpuset.CPUSet // set of CPUs allocated\n\n\tpkgs []sysfs.CPUPackage // physical CPU packages, sorted by preference\n\tcpus []sysfs.CPU        // CPU cores, sorted by preference\n}\n\n// CPUAllocator is an interface for a generic CPU allocator\ntype CPUAllocator interface {\n\tAllocateCpus(from *cpuset.CPUSet, cnt int, prefer CPUPriority) (cpuset.CPUSet, error)\n\tReleaseCpus(from *cpuset.CPUSet, cnt int, prefer CPUPriority) (cpuset.CPUSet, error)\n}\n\ntype CPUPriority int\n\nconst (\n\tPriorityHigh CPUPriority = iota\n\tPriorityNormal\n\tPriorityLow\n\tNumCPUPriorities\n\tPriorityNone = NumCPUPriorities\n)\n\ntype cpuAllocator struct {\n\tlogger.Logger\n\tsys           sysfs.System  // wrapped sysfs.System instance\n\ttopologyCache topologyCache // topology lookups\n}\n\n// topologyCache caches topology lookups\ntype topologyCache struct {\n\tpkg  map[idset.ID]cpuset.CPUSet\n\tnode map[idset.ID]cpuset.CPUSet\n\tcore map[idset.ID]cpuset.CPUSet\n\n\tcpuPriorities cpuPriorities // CPU priority mapping\n}\n\ntype cpuPriorities [NumCPUPriorities]cpuset.CPUSet\n\n// IDFilter helps filtering Ids.\ntype IDFilter func(idset.ID) bool\n\n// IDSorter helps sorting Ids.\ntype IDSorter func(int, int) bool\n\n// our logger instance\nvar log = logger.NewLogger(logSource)\n\n// NewCPUAllocator return a new cpuAllocator instance\nfunc NewCPUAllocator(sys sysfs.System) CPUAllocator {\n\tca := cpuAllocator{\n\t\tLogger:        log,\n\t\tsys:           sys,\n\t\ttopologyCache: newTopologyCache(sys),\n\t}\n\n\treturn &ca\n}\n\n// Pick packages, nodes or CPUs by filtering according to a function.\nfunc pickIds(idSlice []idset.ID, f IDFilter) []idset.ID {\n\tids := make([]idset.ID, len(idSlice))\n\n\tidx := 0\n\tfor _, id := range idSlice {\n\t\tif f == nil || f(id) {\n\t\t\tids[idx] = id\n\t\t\tidx++\n\t\t}\n\t}\n\n\treturn ids[0:idx]\n}\n\n// newAllocatorHelper creates a new CPU allocatorHelper.\nfunc newAllocatorHelper(sys sysfs.System, topo topologyCache) *allocatorHelper {\n\ta := &allocatorHelper{\n\t\tLogger:   log,\n\t\tsys:      sys,\n\t\ttopology: topo,\n\t\tflags:    AllocDefault,\n\t}\n\n\treturn a\n}\n\n// Allocate full idle CPU packages.\nfunc (a *allocatorHelper) takeIdlePackages() {\n\ta.Debug(\"* takeIdlePackages()...\")\n\n\toffline := a.sys.Offlined()\n\n\t// pick idle packages\n\tpkgs := pickIds(a.sys.PackageIDs(),\n\t\tfunc(id idset.ID) bool {\n\t\t\tcset := a.topology.pkg[id].Difference(offline)\n\t\t\treturn cset.Intersection(a.from).Equals(cset)\n\t\t})\n\n\t// sorted by number of preferred cpus and then by cpu id\n\tsort.Slice(pkgs,\n\t\tfunc(i, j int) bool {\n\t\t\tif res := a.topology.cpuPriorities.cmpCPUSet(a.topology.pkg[pkgs[i]], a.topology.pkg[pkgs[j]], a.prefer, -1); res != 0 {\n\t\t\t\treturn res > 0\n\t\t\t}\n\t\t\treturn pkgs[i] < pkgs[j]\n\t\t})\n\n\ta.Debug(\" => idle packages sorted by preference: %v\", pkgs)\n\n\t// take as many idle packages as we need/can\n\tfor _, id := range pkgs {\n\t\tcset := a.topology.pkg[id].Difference(offline)\n\t\ta.Debug(\" => considering package %v (#%s)...\", id, cset)\n\t\tif a.cnt >= cset.Size() {\n\t\t\ta.Debug(\" => taking package %v...\", id)\n\t\t\ta.result = a.result.Union(cset)\n\t\t\ta.from = a.from.Difference(cset)\n\t\t\ta.cnt -= cset.Size()\n\n\t\t\tif a.cnt == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Allocate full idle CPU cores.\nfunc (a *allocatorHelper) takeIdleCores() {\n\ta.Debug(\"* takeIdleCores()...\")\n\n\toffline := a.sys.Offlined()\n\n\t// pick (first id for all) idle cores\n\tcores := pickIds(a.sys.CPUIDs(),\n\t\tfunc(id idset.ID) bool {\n\t\t\tcset := a.topology.core[id].Difference(offline)\n\t\t\tif cset.IsEmpty() {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn cset.Intersection(a.from).Equals(cset) && cset.List()[0] == int(id)\n\t\t})\n\n\t// sorted by id\n\tsort.Slice(cores,\n\t\tfunc(i, j int) bool {\n\t\t\tif res := a.topology.cpuPriorities.cmpCPUSet(a.topology.core[cores[i]], a.topology.core[cores[j]], a.prefer, -1); res != 0 {\n\t\t\t\treturn res > 0\n\t\t\t}\n\t\t\treturn cores[i] < cores[j]\n\t\t})\n\n\ta.Debug(\" => idle cores sorted by preference: %v\", cores)\n\n\t// take as many idle cores as we can\n\tfor _, id := range cores {\n\t\tcset := a.topology.core[id].Difference(offline)\n\t\ta.Debug(\" => considering core %v (#%s)...\", id, cset)\n\t\tif a.cnt >= cset.Size() {\n\t\t\ta.Debug(\" => taking core %v...\", id)\n\t\t\ta.result = a.result.Union(cset)\n\t\t\ta.from = a.from.Difference(cset)\n\t\t\ta.cnt -= cset.Size()\n\n\t\t\tif a.cnt == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Allocate idle CPU hyperthreads.\nfunc (a *allocatorHelper) takeIdleThreads() {\n\toffline := a.sys.Offlined()\n\n\t// pick all threads with free capacity\n\tcores := pickIds(a.sys.CPUIDs(),\n\t\tfunc(id idset.ID) bool {\n\t\t\treturn a.from.Difference(offline).Contains(int(id))\n\t\t})\n\n\ta.Debug(\" => idle threads unsorted: %v\", cores)\n\n\t// sorted for preference by id, mimicking cpus_assignment.go for now:\n\t//   IOW, prefer CPUs\n\t//     - from packages with higher number of CPUs/cores already in a.result\n\t//     - from packages having larger number of available cpus with preferred priority\n\t//     - from a single package\n\t//     - from the list of cpus with preferred priority\n\t//     - from packages with fewer remaining free CPUs/cores in a.from\n\t//     - from cores with fewer remaining free CPUs/cores in a.from\n\t//     - from packages with lower id\n\t//     - with lower id\n\tsort.Slice(cores,\n\t\tfunc(i, j int) bool {\n\t\t\tiCore := cores[i]\n\t\t\tjCore := cores[j]\n\t\t\tiPkg := a.sys.CPU(iCore).PackageID()\n\t\t\tjPkg := a.sys.CPU(jCore).PackageID()\n\n\t\t\tiCoreSet := a.topology.core[iCore]\n\t\t\tjCoreSet := a.topology.core[jCore]\n\t\t\tiPkgSet := a.topology.pkg[iPkg]\n\t\t\tjPkgSet := a.topology.pkg[jPkg]\n\n\t\t\tiPkgColo := iPkgSet.Intersection(a.result).Size()\n\t\t\tjPkgColo := jPkgSet.Intersection(a.result).Size()\n\t\t\tif iPkgColo != jPkgColo {\n\t\t\t\treturn iPkgColo > jPkgColo\n\t\t\t}\n\n\t\t\t// Always sort cores in package order\n\t\t\tif res := a.topology.cpuPriorities.cmpCPUSet(iPkgSet.Intersection(a.from), jPkgSet.Intersection(a.from), a.prefer, a.cnt); res != 0 {\n\t\t\t\treturn res > 0\n\t\t\t}\n\t\t\tif iPkg != jPkg {\n\t\t\t\treturn iPkg < jPkg\n\t\t\t}\n\n\t\t\tiCset := cpuset.New(int(cores[i]))\n\t\t\tjCset := cpuset.New(int(cores[j]))\n\t\t\tif res := a.topology.cpuPriorities.cmpCPUSet(iCset, jCset, a.prefer, 0); res != 0 {\n\t\t\t\treturn res > 0\n\t\t\t}\n\n\t\t\tiPkgFree := iPkgSet.Intersection(a.from).Size()\n\t\t\tjPkgFree := jPkgSet.Intersection(a.from).Size()\n\t\t\tif iPkgFree != jPkgFree {\n\t\t\t\treturn iPkgFree < jPkgFree\n\t\t\t}\n\n\t\t\tiCoreFree := iCoreSet.Intersection(a.from).Size()\n\t\t\tjCoreFree := jCoreSet.Intersection(a.from).Size()\n\t\t\tif iCoreFree != jCoreFree {\n\t\t\t\treturn iCoreFree < jCoreFree\n\t\t\t}\n\n\t\t\treturn iCore < jCore\n\t\t})\n\n\ta.Debug(\" => idle threads sorted: %v\", cores)\n\n\t// take as many idle cores as we can\n\tfor _, id := range cores {\n\t\tcset := a.topology.core[id].Difference(offline)\n\t\ta.Debug(\" => considering thread %v (#%s)...\", id, cset)\n\t\tcset = cpuset.New(int(id))\n\t\ta.result = a.result.Union(cset)\n\t\ta.from = a.from.Difference(cset)\n\t\ta.cnt -= cset.Size()\n\n\t\tif a.cnt == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n// takeAny is a dummy allocator not dependent on sysfs topology information\nfunc (a *allocatorHelper) takeAny() {\n\ta.Debug(\"* takeAnyCores()...\")\n\n\tcpus := a.from.List()\n\n\tif len(cpus) >= a.cnt {\n\t\tcset := cpuset.New(cpus[0:a.cnt]...)\n\t\ta.result = a.result.Union(cset)\n\t\ta.from = a.from.Difference(cset)\n\t\ta.cnt = 0\n\t}\n}\n\n// Perform CPU allocation.\nfunc (a *allocatorHelper) allocate() cpuset.CPUSet {\n\tif a.sys != nil {\n\t\tif (a.flags & AllocIdlePackages) != 0 {\n\t\t\ta.takeIdlePackages()\n\t\t}\n\t\tif a.cnt > 0 && (a.flags&AllocIdleCores) != 0 {\n\t\t\ta.takeIdleCores()\n\t\t}\n\t\tif a.cnt > 0 {\n\t\t\ta.takeIdleThreads()\n\t\t}\n\t} else {\n\t\ta.takeAny()\n\t}\n\tif a.cnt == 0 {\n\t\treturn a.result\n\t}\n\n\treturn cpuset.New()\n}\n\nfunc (ca *cpuAllocator) allocateCpus(from *cpuset.CPUSet, cnt int, prefer CPUPriority) (cpuset.CPUSet, error) {\n\tvar result cpuset.CPUSet\n\tvar err error\n\n\tswitch {\n\tcase from.Size() < cnt:\n\t\tresult, err = cpuset.New(), fmt.Errorf(\"cpuset %s does not have %d CPUs\", from, cnt)\n\tcase from.Size() == cnt:\n\t\tresult, err, *from = from.Clone(), nil, cpuset.New()\n\tdefault:\n\t\ta := newAllocatorHelper(ca.sys, ca.topologyCache)\n\t\ta.from = from.Clone()\n\t\ta.cnt = cnt\n\t\ta.prefer = prefer\n\n\t\tresult, err, *from = a.allocate(), nil, a.from.Clone()\n\n\t\ta.Debug(\"%d cpus from #%v (preferring #%v) => #%v\", cnt, from.Union(result), a.prefer, result)\n\t}\n\n\treturn result, err\n}\n\n// AllocateCpus allocates a number of CPUs from the given set.\nfunc (ca *cpuAllocator) AllocateCpus(from *cpuset.CPUSet, cnt int, prefer CPUPriority) (cpuset.CPUSet, error) {\n\tresult, err := ca.allocateCpus(from, cnt, prefer)\n\treturn result, err\n}\n\n// ReleaseCpus releases a number of CPUs from the given set.\nfunc (ca *cpuAllocator) ReleaseCpus(from *cpuset.CPUSet, cnt int, prefer CPUPriority) (cpuset.CPUSet, error) {\n\toset := from.Clone()\n\n\tresult, err := ca.allocateCpus(from, from.Size()-cnt, prefer)\n\n\tca.Debug(\"ReleaseCpus(#%s, %d) => kept: #%s, released: #%s\", oset, cnt, from, result)\n\n\treturn result, err\n}\n\nfunc newTopologyCache(sys sysfs.System) topologyCache {\n\tc := topologyCache{\n\t\tpkg:  make(map[idset.ID]cpuset.CPUSet),\n\t\tnode: make(map[idset.ID]cpuset.CPUSet),\n\t\tcore: make(map[idset.ID]cpuset.CPUSet)}\n\tif sys != nil {\n\t\tfor _, id := range sys.PackageIDs() {\n\t\t\tc.pkg[id] = sys.Package(id).CPUSet()\n\t\t}\n\t\tfor _, id := range sys.NodeIDs() {\n\t\t\tc.node[id] = sys.Node(id).CPUSet()\n\t\t}\n\t\tfor _, id := range sys.CPUIDs() {\n\t\t\tc.core[id] = sys.CPU(id).ThreadCPUSet()\n\t\t}\n\t}\n\n\tc.discoverCPUPriorities(sys)\n\n\treturn c\n}\n\nfunc (c *topologyCache) discoverCPUPriorities(sys sysfs.System) {\n\tif sys == nil {\n\t\treturn\n\t}\n\tvar prio cpuPriorities\n\n\t// Discover on per-package basis\n\tfor id := range c.pkg {\n\t\tcpuPriorities, sstActive := c.discoverSstCPUPriority(sys, id)\n\n\t\tif !sstActive {\n\t\t\tcpuPriorities = c.discoverCpufreqPriority(sys, id)\n\t\t}\n\n\t\tfor p, cpus := range cpuPriorities {\n\t\t\tsource := map[bool]string{true: \"sst\", false: \"cpufreq\"}[sstActive]\n\t\t\tcset := sysfs.CPUSetFromIDSet(idset.NewIDSet(cpus...))\n\t\t\tlog.Debug(\"package #%d (%s): %d %s priority cpus (%v)\", id, source, len(cpus), CPUPriority(p), cset)\n\t\t\tprio[p] = prio[p].Union(cset)\n\t\t}\n\t}\n\tc.cpuPriorities = prio\n}\n\nfunc (c *topologyCache) discoverSstCPUPriority(sys sysfs.System, pkgID idset.ID) ([NumCPUPriorities][]idset.ID, bool) {\n\tactive := false\n\n\tpkg := sys.Package(pkgID)\n\tsst := pkg.SstInfo()\n\tcpuIDs := c.pkg[pkgID].List()\n\tprios := make(map[idset.ID]CPUPriority, len(cpuIDs))\n\n\t// Determine SST-based priority. Based on experimentation there is some\n\t// hierarchy between the SST features. Without trying to be too smart\n\t// we follow the principles below:\n\t// 1. SST-TF has highest preference, mastering over SST-BF and making most\n\t//    of SST-CP settings ineffective\n\t// 2. SST-CP dictates over SST-BF\n\t// 3. SST-BF is meaningful if neither SST-TF nor SST-CP is enabled\n\tswitch {\n\tcase sst == nil:\n\tcase sst.TFEnabled:\n\t\tlog.Debug(\"package #%d: using SST-TF based CPU prioritization\", pkgID)\n\t\t// We only look at the CLOS id as SST-TF (seems to) follows ordered CLOS priority\n\t\tfor _, i := range cpuIDs {\n\t\t\tid := idset.ID(i)\n\t\t\tp := PriorityLow\n\t\t\t// First two CLOSes are prioritized by SST\n\t\t\tif sys.CPU(id).SstClos() < 2 {\n\t\t\t\tp = PriorityHigh\n\t\t\t}\n\t\t\tprios[id] = p\n\t\t}\n\t\tactive = true\n\tcase sst.CPEnabled:\n\t\tclosPrio := c.sstClosPriority(sys, pkgID)\n\t\tlog.Debug(\"package #%d: using SST-CP based CPU prioritization with CLOS mapping %v\", pkgID, closPrio)\n\n\t\tactive = false\n\t\tfor _, i := range cpuIDs {\n\t\t\tid := idset.ID(i)\n\t\t\tclos := sys.CPU(id).SstClos()\n\t\t\tp := closPrio[clos]\n\t\t\tif p != PriorityNormal {\n\t\t\t\tactive = true\n\t\t\t}\n\t\t\tprios[id] = p\n\t\t}\n\t}\n\n\tif !active && sst != nil && sst.BFEnabled {\n\t\tlog.Debug(\"package #%d: using SST-BF based CPU prioritization\", pkgID)\n\t\tfor _, i := range cpuIDs {\n\t\t\tid := idset.ID(i)\n\t\t\tp := PriorityLow\n\t\t\tif sst.BFCores.Has(id) {\n\t\t\t\tp = PriorityHigh\n\t\t\t}\n\t\t\tprios[id] = p\n\t\t}\n\t\tactive = true\n\t}\n\n\tvar ret [NumCPUPriorities][]idset.ID\n\n\tfor cpu, prio := range prios {\n\t\tret[prio] = append(ret[prio], cpu)\n\t}\n\treturn ret, active\n}\n\nfunc (c *topologyCache) sstClosPriority(sys sysfs.System, pkgID idset.ID) map[int]CPUPriority {\n\tsortedKeys := func(m map[int]int) []int {\n\t\tkeys := make([]int, 0, len(m))\n\t\tfor k := range m {\n\t\t\tkeys = append(keys, k)\n\t\t}\n\t\tsort.Ints(keys)\n\t\treturn keys\n\t}\n\n\tpkg := sys.Package(pkgID)\n\tsstinfo := pkg.SstInfo()\n\n\t// Get a list of unique CLOS proportional priority values\n\tclosPps := make(map[int]int)\n\tclosIds := make(map[int]int)\n\tfor _, cpuID := range c.pkg[pkgID].List() {\n\t\tclos := sys.CPU(idset.ID(cpuID)).SstClos()\n\t\tpp := sstinfo.ClosInfo[clos].ProportionalPriority\n\t\tclosPps[pp] = clos\n\t\tclosIds[clos] = 0 // 0 is a dummy value here\n\t}\n\n\t// Form a list of (active) CLOS ids in sorted order\n\tvar closSorted []int\n\tif sstinfo.CPPriority == sst.Ordered {\n\t\t// In ordered mode the priority is simply the CLOS id\n\t\tclosSorted = sortedKeys(closIds)\n\t\tlog.Debug(\"package #%d, ordered SST-CP priority with CLOS ids %v\", pkgID, closSorted)\n\t} else {\n\t\t// In proportional mode we sort by the proportional priority parameter\n\t\tclosPpSorted := sortedKeys(closPps)\n\n\t\tfor _, pp := range closPpSorted {\n\t\t\tclosSorted = append(closSorted, closPps[pp])\n\t\t}\n\t\tlog.Debug(\"package #%d, proportional SST-CP priority with PP-to-CLOS parity %v\", pkgID, closPps)\n\t}\n\n\t// Map from CLOS id to cpuallocator CPU priority\n\tclosPriority := make(map[int]CPUPriority, len(closSorted))\n\tfor _, id := range closSorted {\n\t\t// Default to normal priority\n\t\tclosPriority[id] = PriorityNormal\n\t}\n\tif len(closSorted) > 1 {\n\t\t// Highest CLOS id maps to high CPU priority\n\t\tclosPriority[closSorted[0]] = PriorityHigh\n\t\tclosPriority[closSorted[len(closSorted)-1]] = PriorityLow\n\t}\n\n\treturn closPriority\n}\n\nfunc (c *topologyCache) discoverCpufreqPriority(sys sysfs.System, pkgID idset.ID) [NumCPUPriorities][]idset.ID {\n\tvar prios [NumCPUPriorities][]idset.ID\n\n\t// Group cpus by base frequency and energy performance profile\n\tfreqs := map[uint64][]idset.ID{}\n\tepps := map[sysfs.EPP][]idset.ID{}\n\tcpuIDs := c.pkg[pkgID].List()\n\tfor _, num := range cpuIDs {\n\t\tid := idset.ID(num)\n\t\tcpu := sys.CPU(id)\n\t\tbf := cpu.BaseFrequency()\n\t\tfreqs[bf] = append(freqs[bf], id)\n\n\t\tepp := cpu.EPP()\n\t\tepps[epp] = append(epps[epp], id)\n\t}\n\n\t// Construct a sorted lists of detected frequencies and epp values\n\tfreqList := []uint64{}\n\tfor freq := range freqs {\n\t\tif freq > 0 {\n\t\t\tfreqList = append(freqList, freq)\n\t\t}\n\t}\n\tutils.SortUint64s(freqList)\n\n\teppList := []int{}\n\tfor e := range epps {\n\t\tif e != sysfs.EPPUnknown {\n\t\t\teppList = append(eppList, int(e))\n\t\t}\n\t}\n\tsort.Ints(eppList)\n\n\t// Finally, determine priority of each CPU\n\tfor _, num := range cpuIDs {\n\t\tid := idset.ID(num)\n\t\tcpu := sys.CPU(id)\n\t\tp := PriorityNormal\n\n\t\tif len(freqList) > 1 {\n\t\t\tbf := cpu.BaseFrequency()\n\n\t\t\t// All cpus NOT in the lowest base frequency bin are considered high prio\n\t\t\tif bf > freqList[0] {\n\t\t\t\tp = PriorityHigh\n\t\t\t} else {\n\t\t\t\tp = PriorityLow\n\t\t\t}\n\t\t}\n\n\t\t// All cpus NOT in the lowest performance epp are considered high prio\n\t\t// NOTE: higher EPP value denotes lower performance preference\n\t\tif len(eppList) > 1 {\n\t\t\tepp := cpu.EPP()\n\t\t\tif int(epp) < eppList[len(eppList)-1] {\n\t\t\t\tp = PriorityHigh\n\t\t\t} else {\n\t\t\t\tp = PriorityLow\n\t\t\t}\n\t\t}\n\n\t\tprios[p] = append(prios[p], id)\n\t}\n\n\treturn prios\n}\n\nfunc (p CPUPriority) String() string {\n\tswitch p {\n\tcase PriorityHigh:\n\t\treturn \"high\"\n\tcase PriorityNormal:\n\t\treturn \"normal\"\n\tcase PriorityLow:\n\t\treturn \"low\"\n\t}\n\treturn \"none\"\n}\n\n// cmpCPUSet compares two cpusets in terms of preferred cpu priority. Returns:\n//\n//\t> 0 if cpuset A is preferred\n//\t< 0 if cpuset B is preferred\n//\t0 if cpusets A and B are equal in terms of cpu priority\nfunc (c *cpuPriorities) cmpCPUSet(csetA, csetB cpuset.CPUSet, prefer CPUPriority, cpuCnt int) int {\n\tif prefer == PriorityNone {\n\t\treturn 0\n\t}\n\n\t// Favor cpuset having CPUs with priorities equal to or lower than what was requested\n\tfor prio := prefer; prio < NumCPUPriorities; prio++ {\n\t\tprefA := csetA.Intersection(c[prio]).Size()\n\t\tprefB := csetB.Intersection(c[prio]).Size()\n\t\tif cpuCnt > 0 && prio == prefer && prefA >= cpuCnt && prefB >= cpuCnt {\n\t\t\t// Prefer the tightest fitting if both cpusets satisfy the\n\t\t\t// requested amount of CPUs with the preferred priority\n\t\t\treturn prefB - prefA\n\t\t}\n\t\tif prefA != prefB {\n\t\t\treturn prefA - prefB\n\t\t}\n\t}\n\t// Repel cpuset having CPUs with higher priority than what was requested\n\tfor prio := PriorityHigh; prio < prefer; prio++ {\n\t\tnonprefA := csetA.Intersection(c[prio]).Size()\n\t\tnonprefB := csetB.Intersection(c[prio]).Size()\n\t\tif nonprefA != nonprefB {\n\t\t\treturn nonprefB - nonprefA\n\t\t}\n\t}\n\treturn 0\n}\n"
  },
  {
    "path": "pkg/cpuallocator/cpuallocator_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpuallocator\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nfunc TestAllocatorHelper(t *testing.T) {\n\t// Create tmpdir and decompress testdata there\n\ttmpdir, err := os.MkdirTemp(\"\", \"cri-resource-manager-test-\")\n\tif err != nil {\n\t\tt.Fatalf(\"failed to create tmpdir: %v\", err)\n\t}\n\tdefer os.RemoveAll(tmpdir)\n\n\tif err := utils.UncompressTbz2(path.Join(\"testdata\", \"sysfs.tar.bz2\"), tmpdir); err != nil {\n\t\tt.Fatalf(\"failed to decompress testdata: %v\", err)\n\t}\n\n\t// Discover mock system from the testdata\n\tsys, err := sysfs.DiscoverSystemAt(path.Join(tmpdir, \"sysfs\", \"2-socket-4-node-40-core\", \"sys\"))\n\tif err != nil {\n\t\tt.Fatalf(\"failed to discover mock system: %v\", err)\n\t}\n\ttopoCache := newTopologyCache(sys)\n\n\t// Fake cpu priorities: 5 cores from pkg #0 as high prio\n\t// Package CPUs: #0: [0-19,40-59], #1: [20-39,60-79]\n\ttopoCache.cpuPriorities = [NumCPUPriorities]cpuset.CPUSet{\n\t\tcpuset.MustParse(\"2,5,8,15,17,42,45,48,55,57\"),\n\t\tcpuset.MustParse(\"20-39,60-79\"),\n\t\tcpuset.MustParse(\"0,1,3,4,6,7,9-14,16,18,19,40,41,43,44,46,47,49-54,56,58,59\"),\n\t}\n\n\ttcs := []struct {\n\t\tdescription string\n\t\tfrom        cpuset.CPUSet\n\t\tprefer      CPUPriority\n\t\tcnt         int\n\t\texpected    cpuset.CPUSet\n\t}{\n\t\t{\n\t\t\tdescription: \"too few available CPUs\",\n\t\t\tfrom:        cpuset.MustParse(\"2,3,10-14,20\"),\n\t\t\tprefer:      PriorityNormal,\n\t\t\tcnt:         9,\n\t\t\texpected:    cpuset.New(),\n\t\t},\n\t\t{\n\t\t\tdescription: \"request all available CPUs\",\n\t\t\tfrom:        cpuset.MustParse(\"2,3,10-14,20\"),\n\t\t\tprefer:      PriorityNormal,\n\t\t\tcnt:         8,\n\t\t\texpected:    cpuset.MustParse(\"2,3,10-14,20\"),\n\t\t},\n\t\t{\n\t\t\tdescription: \"prefer high priority cpus\",\n\t\t\tfrom:        cpuset.MustParse(\"2,3,10-25\"),\n\t\t\tprefer:      PriorityHigh,\n\t\t\tcnt:         4,\n\t\t\texpected:    cpuset.New(2, 3, 15, 17),\n\t\t},\n\t}\n\n\t// Run tests\n\tfor _, tc := range tcs {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\ta := newAllocatorHelper(sys, topoCache)\n\t\t\ta.from = tc.from\n\t\t\ta.prefer = tc.prefer\n\t\t\ta.cnt = tc.cnt\n\t\t\tresult := a.allocate()\n\t\t\tif !result.Equals(tc.expected) {\n\t\t\t\tt.Errorf(\"expected %q, result was %q\", tc.expected, result)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/client/client.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage client\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"google.golang.org/grpc\"\n\t\"google.golang.org/grpc/connectivity\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\n\tv1 \"github.com/intel/cri-resource-manager/pkg/cri/client/v1\"\n)\n\n// DialNotifyFn is a function to call after a successful net.Dial[Timeout]().\ntype DialNotifyFn func(string, int, int, os.FileMode, error)\n\n// Options contains the configurable options of our CRI client.\ntype Options struct {\n\t// ImageSocket is the socket path for the CRI image service.\n\tImageSocket string\n\t// RuntimeSocket is the socket path for the CRI runtime service.\n\tRuntimeSocket string\n\t// DialNotify is an optional function to notify after net.Dial returns for a socket.\n\tDialNotify DialNotifyFn\n}\n\n// ConnectOptions contains options for connecting to the server.\ntype ConnectOptions struct {\n\t// Wait indicates whether Connect() should wait (indefinitely) for the server.\n\tWait bool\n\t// Reconnect indicates whether CheckConnection() should attempt to Connect().\n\tReconnect bool\n}\n\n// Client is the interface we expose to our CRI client.\ntype Client interface {\n\t// Connect tries to connect the client to the specified image and runtime services.\n\tConnect(ConnectOptions) error\n\t// Close closes any existing client connections.\n\tClose()\n\t// CheckConnection checks if we have (un-Close()'d as opposed to working) connections.\n\tCheckConnection(ConnectOptions) error\n\t// HasRuntimeService checks if the client is configured with runtime services.\n\tHasRuntimeService() bool\n\n\t// We expose full image and runtime client services.\n\tcriv1.ImageServiceClient\n\tcriv1.RuntimeServiceClient\n}\n\ntype criClient interface {\n\tcriv1.ImageServiceClient\n\tcriv1.RuntimeServiceClient\n}\n\n// client is the implementation of Client.\ntype client struct {\n\tlogger.Logger\n\tcriv1.ImageServiceClient\n\tcriv1.RuntimeServiceClient\n\toptions Options          // client options\n\ticc     *grpc.ClientConn // our gRPC connection to the image service\n\trcc     *grpc.ClientConn // our gRPC connection to the runtime service\n\n\tclient criClient\n}\n\nconst (\n\t// DontConnect is used to mark a socket to not be connected.\n\tDontConnect = \"-\"\n)\n\n// NewClient creates a new client instance.\nfunc NewClient(options Options) (Client, error) {\n\tif options.ImageSocket == DontConnect && options.RuntimeSocket == DontConnect {\n\t\treturn nil, clientError(\"neither image nor runtime socket specified\")\n\t}\n\n\tc := &client{\n\t\tLogger:  logger.NewLogger(\"cri/client\"),\n\t\toptions: options,\n\t}\n\n\treturn c, nil\n}\n\n// Connect attempts to establish gRPC client connections to the configured services.\nfunc (c *client) Connect(options ConnectOptions) error {\n\tvar err error\n\n\tkind, socket := \"image services\", c.options.ImageSocket\n\tif c.icc, err = c.connect(kind, socket, options); err != nil {\n\t\treturn err\n\t}\n\n\tkind, socket = \"runtime services\", c.options.RuntimeSocket\n\tif socket == c.options.ImageSocket {\n\t\tc.rcc = c.icc\n\t} else {\n\t\tif c.rcc, err = c.connect(kind, socket, options); err != nil {\n\t\t\tc.icc = nil\n\t\t\treturn err\n\t\t}\n\t}\n\n\tclient, err := v1.Connect(c.rcc, c.icc)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tc.client = client\n\treturn nil\n}\n\n// Close any open service connection.\nfunc (c *client) Close() {\n\tif c.icc != nil {\n\t\tc.Debug(\"closing image service connection...\")\n\t\tc.icc.Close()\n\t}\n\n\tif c.rcc != nil {\n\t\tc.Debug(\"closing runtime service connection...\")\n\t\tif c.rcc != c.icc {\n\t\t\tc.rcc.Close()\n\t\t}\n\t}\n\n\tc.icc = nil\n\tc.rcc = nil\n}\n\n// Check if the connecton to CRI services is up, try to reconnect if requested.\nfunc (c *client) CheckConnection(options ConnectOptions) error {\n\tif (c.icc == nil || c.icc.GetState() == connectivity.Ready) &&\n\t\t(c.rcc == nil || c.rcc.GetState() == connectivity.Ready) {\n\t\treturn nil\n\t}\n\n\tc.Close()\n\n\tif options.Reconnect {\n\t\tc.Warn(\"client connections are down\")\n\t\tif err := c.Connect(ConnectOptions{Wait: false}); err == nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\treturn clientError(\"client connections are down\")\n}\n\n// HasRuntimeService checks if the client is configured with runtime services.\nfunc (c *client) HasRuntimeService() bool {\n\treturn c.options.RuntimeSocket != \"\" && c.options.RuntimeSocket != DontConnect\n}\n\nfunc (c *client) checkRuntimeService() error {\n\tif c.client == nil || c.rcc == nil {\n\t\treturn clientError(\"no CRI RuntimeService client\")\n\t}\n\treturn nil\n}\n\nfunc (c *client) checkImageService() error {\n\tif c.client == nil || c.icc == nil {\n\t\treturn clientError(\"no CRI ImageService client\")\n\t}\n\treturn nil\n}\n\n// connect attempts to create a gRPC client connection to the given socket.\nfunc (c *client) connect(kind, socket string, options ConnectOptions) (*grpc.ClientConn, error) {\n\tvar cc *grpc.ClientConn\n\tvar err error\n\n\tif socket == DontConnect {\n\t\treturn nil, nil\n\t}\n\n\tdialOpts := instrumentation.InjectGrpcClientTrace(\n\t\tgrpc.WithInsecure(),\n\t\tgrpc.WithBlock(),\n\t\tgrpc.FailOnNonTempDialError(true),\n\t\tgrpc.WithDialer(func(socket string, timeout time.Duration) (net.Conn, error) {\n\t\t\tconn, err := net.DialTimeout(\"unix\", socket, timeout)\n\t\t\tif err != nil {\n\t\t\t\treturn conn, err\n\t\t\t}\n\t\t\tc.dialNotify(socket)\n\t\t\treturn conn, err\n\t\t}))\n\n\tif options.Wait {\n\t\tc.Info(\"waiting for %s on socket %s...\", kind, socket)\n\t\tif err = utils.WaitForServer(socket, -1, dialOpts, &cc); err != nil {\n\t\t\treturn nil, clientError(\"failed to connect to %s: %v\", kind, err)\n\t\t}\n\t} else {\n\t\tif cc, err = grpc.Dial(socket, dialOpts...); err != nil {\n\t\t\treturn nil, clientError(\"failed to connect to %s: %v\", kind, err)\n\t\t}\n\t}\n\n\treturn cc, nil\n}\n\nfunc (c *client) dialNotify(socket string) {\n\tif c.options.DialNotify == nil {\n\t\treturn\n\t}\n\n\tinfo, err := os.Stat(socket)\n\tif err != nil {\n\t\tc.options.DialNotify(socket, -1, -1, 0, err)\n\t\treturn\n\t}\n\n\tst, ok := info.Sys().(*syscall.Stat_t)\n\tif !ok {\n\t\terr := clientError(\"failed to stat socket %q: %v\", socket, err)\n\t\tc.options.DialNotify(socket, -1, -1, 0, err)\n\t\treturn\n\t}\n\n\tuid, gid := int(st.Uid), int(st.Gid)\n\tmode := info.Mode() & os.ModePerm\n\tc.options.DialNotify(socket, uid, gid, mode, nil)\n}\n\nfunc (c *client) Version(ctx context.Context, in *criv1.VersionRequest, _ ...grpc.CallOption) (*criv1.VersionResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.Version(ctx, in)\n}\n\nfunc (c *client) RunPodSandbox(ctx context.Context, in *criv1.RunPodSandboxRequest, _ ...grpc.CallOption) (*criv1.RunPodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.RunPodSandbox(ctx, in)\n}\n\nfunc (c *client) StopPodSandbox(ctx context.Context, in *criv1.StopPodSandboxRequest, _ ...grpc.CallOption) (*criv1.StopPodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.StopPodSandbox(ctx, in)\n}\n\nfunc (c *client) RemovePodSandbox(ctx context.Context, in *criv1.RemovePodSandboxRequest, _ ...grpc.CallOption) (*criv1.RemovePodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.RemovePodSandbox(ctx, in)\n}\n\nfunc (c *client) PodSandboxStatus(ctx context.Context, in *criv1.PodSandboxStatusRequest, _ ...grpc.CallOption) (*criv1.PodSandboxStatusResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.PodSandboxStatus(ctx, in)\n}\n\nfunc (c *client) ListPodSandbox(ctx context.Context, in *criv1.ListPodSandboxRequest, _ ...grpc.CallOption) (*criv1.ListPodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ListPodSandbox(ctx, in)\n}\n\nfunc (c *client) CreateContainer(ctx context.Context, in *criv1.CreateContainerRequest, _ ...grpc.CallOption) (*criv1.CreateContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.CreateContainer(ctx, in)\n}\n\nfunc (c *client) StartContainer(ctx context.Context, in *criv1.StartContainerRequest, _ ...grpc.CallOption) (*criv1.StartContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.StartContainer(ctx, in)\n}\n\nfunc (c *client) StopContainer(ctx context.Context, in *criv1.StopContainerRequest, _ ...grpc.CallOption) (*criv1.StopContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.StopContainer(ctx, in)\n}\n\nfunc (c *client) RemoveContainer(ctx context.Context, in *criv1.RemoveContainerRequest, _ ...grpc.CallOption) (*criv1.RemoveContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.RemoveContainer(ctx, in)\n}\n\nfunc (c *client) ListContainers(ctx context.Context, in *criv1.ListContainersRequest, _ ...grpc.CallOption) (*criv1.ListContainersResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ListContainers(ctx, in)\n}\n\nfunc (c *client) ContainerStatus(ctx context.Context, in *criv1.ContainerStatusRequest, _ ...grpc.CallOption) (*criv1.ContainerStatusResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ContainerStatus(ctx, in)\n}\n\nfunc (c *client) UpdateContainerResources(ctx context.Context, in *criv1.UpdateContainerResourcesRequest, _ ...grpc.CallOption) (*criv1.UpdateContainerResourcesResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.UpdateContainerResources(ctx, in)\n}\n\nfunc (c *client) ReopenContainerLog(ctx context.Context, in *criv1.ReopenContainerLogRequest, _ ...grpc.CallOption) (*criv1.ReopenContainerLogResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ReopenContainerLog(ctx, in)\n}\n\nfunc (c *client) ExecSync(ctx context.Context, in *criv1.ExecSyncRequest, _ ...grpc.CallOption) (*criv1.ExecSyncResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ExecSync(ctx, in)\n}\n\nfunc (c *client) Exec(ctx context.Context, in *criv1.ExecRequest, _ ...grpc.CallOption) (*criv1.ExecResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.Exec(ctx, in)\n}\n\nfunc (c *client) Attach(ctx context.Context, in *criv1.AttachRequest, _ ...grpc.CallOption) (*criv1.AttachResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.Attach(ctx, in)\n}\n\nfunc (c *client) PortForward(ctx context.Context, in *criv1.PortForwardRequest, _ ...grpc.CallOption) (*criv1.PortForwardResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.PortForward(ctx, in)\n}\n\nfunc (c *client) ContainerStats(ctx context.Context, in *criv1.ContainerStatsRequest, _ ...grpc.CallOption) (*criv1.ContainerStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ContainerStats(ctx, in)\n}\n\nfunc (c *client) ListContainerStats(ctx context.Context, in *criv1.ListContainerStatsRequest, _ ...grpc.CallOption) (*criv1.ListContainerStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ListContainerStats(ctx, in)\n}\n\nfunc (c *client) PodSandboxStats(ctx context.Context, in *criv1.PodSandboxStatsRequest, _ ...grpc.CallOption) (*criv1.PodSandboxStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.PodSandboxStats(ctx, in)\n}\n\nfunc (c *client) ListPodSandboxStats(ctx context.Context, in *criv1.ListPodSandboxStatsRequest, _ ...grpc.CallOption) (*criv1.ListPodSandboxStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ListPodSandboxStats(ctx, in)\n}\n\nfunc (c *client) UpdateRuntimeConfig(ctx context.Context, in *criv1.UpdateRuntimeConfigRequest, _ ...grpc.CallOption) (*criv1.UpdateRuntimeConfigResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.UpdateRuntimeConfig(ctx, in)\n}\n\nfunc (c *client) Status(ctx context.Context, in *criv1.StatusRequest, _ ...grpc.CallOption) (*criv1.StatusResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.Status(ctx, in)\n}\n\nfunc (c *client) CheckpointContainer(ctx context.Context, in *criv1.CheckpointContainerRequest, _ ...grpc.CallOption) (*criv1.CheckpointContainerResponse, error) {\n\treturn c.client.CheckpointContainer(ctx, in)\n}\n\nfunc (c *client) GetContainerEvents(ctx context.Context, in *criv1.GetEventsRequest, _ ...grpc.CallOption) (criv1.RuntimeService_GetContainerEventsClient, error) {\n\treturn c.client.GetContainerEvents(ctx, in)\n}\n\nfunc (c *client) ListMetricDescriptors(ctx context.Context, in *criv1.ListMetricDescriptorsRequest, _ ...grpc.CallOption) (*criv1.ListMetricDescriptorsResponse, error) {\n\treturn c.client.ListMetricDescriptors(ctx, in)\n}\n\nfunc (c *client) ListPodSandboxMetrics(ctx context.Context, in *criv1.ListPodSandboxMetricsRequest, _ ...grpc.CallOption) (*criv1.ListPodSandboxMetricsResponse, error) {\n\treturn c.client.ListPodSandboxMetrics(ctx, in)\n}\n\nfunc (c *client) RuntimeConfig(ctx context.Context, in *criv1.RuntimeConfigRequest, _ ...grpc.CallOption) (*criv1.RuntimeConfigResponse, error) {\n\treturn c.client.RuntimeConfig(ctx, in)\n}\n\nfunc (c *client) ListImages(ctx context.Context, in *criv1.ListImagesRequest, _ ...grpc.CallOption) (*criv1.ListImagesResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ListImages(ctx, in)\n}\n\nfunc (c *client) ImageStatus(ctx context.Context, in *criv1.ImageStatusRequest, _ ...grpc.CallOption) (*criv1.ImageStatusResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ImageStatus(ctx, in)\n}\n\nfunc (c *client) PullImage(ctx context.Context, in *criv1.PullImageRequest, _ ...grpc.CallOption) (*criv1.PullImageResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.PullImage(ctx, in)\n}\n\nfunc (c *client) RemoveImage(ctx context.Context, in *criv1.RemoveImageRequest, _ ...grpc.CallOption) (*criv1.RemoveImageResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.RemoveImage(ctx, in)\n}\n\nfunc (c *client) ImageFsInfo(ctx context.Context, in *criv1.ImageFsInfoRequest, _ ...grpc.CallOption) (*criv1.ImageFsInfoResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.client.ImageFsInfo(ctx, in)\n}\n\n// Return a formatted client-specific error.\nfunc clientError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"cri/client: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/client/v1/client.go",
    "content": "// Copyright Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"google.golang.org/grpc\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\ntype Client interface {\n\tcriv1.ImageServiceClient\n\tcriv1.RuntimeServiceClient\n}\n\ntype client struct {\n\tlogger.Logger\n\tisc criv1.ImageServiceClient\n\trsc criv1.RuntimeServiceClient\n\trcc *grpc.ClientConn\n\ticc *grpc.ClientConn\n}\n\n// Connect v2alpha1 RuntimeService and ImageService clients.\nfunc Connect(runtime, image *grpc.ClientConn) (Client, error) {\n\tc := &client{\n\t\tLogger: logger.Get(\"cri/client\"),\n\t\trcc:    runtime,\n\t\ticc:    image,\n\t}\n\n\tif c.rcc != nil {\n\t\tc.Info(\"probing CRI v1 RuntimeService client...\")\n\t\tc.rsc = criv1.NewRuntimeServiceClient(c.rcc)\n\t\t_, err := c.rsc.Version(context.Background(), &criv1.VersionRequest{})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif c.icc != nil {\n\t\tc.Info(\"probing CRI v1 ImageService client...\")\n\t\tc.isc = criv1.NewImageServiceClient(c.icc)\n\t\t_, err := c.isc.ListImages(context.Background(), &criv1.ListImagesRequest{})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn c, nil\n}\n\nfunc (c *client) checkRuntimeService() error {\n\tif c.rcc == nil {\n\t\treturn fmt.Errorf(\"no CRI v1 RuntimeService client\")\n\t}\n\treturn nil\n}\n\nfunc (c *client) checkImageService() error {\n\tif c.icc == nil {\n\t\treturn fmt.Errorf(\"no CRI v1 ImageService client\")\n\t}\n\treturn nil\n}\n\nfunc (c *client) Version(ctx context.Context, in *criv1.VersionRequest, _ ...grpc.CallOption) (*criv1.VersionResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.Version(ctx, in)\n}\n\nfunc (c *client) RunPodSandbox(ctx context.Context, in *criv1.RunPodSandboxRequest, _ ...grpc.CallOption) (*criv1.RunPodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.RunPodSandbox(ctx, in)\n}\n\nfunc (c *client) StopPodSandbox(ctx context.Context, in *criv1.StopPodSandboxRequest, _ ...grpc.CallOption) (*criv1.StopPodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.StopPodSandbox(ctx, in)\n}\n\nfunc (c *client) RemovePodSandbox(ctx context.Context, in *criv1.RemovePodSandboxRequest, _ ...grpc.CallOption) (*criv1.RemovePodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.RemovePodSandbox(ctx, in)\n}\n\nfunc (c *client) PodSandboxStatus(ctx context.Context, in *criv1.PodSandboxStatusRequest, _ ...grpc.CallOption) (*criv1.PodSandboxStatusResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.PodSandboxStatus(ctx, in)\n}\n\nfunc (c *client) ListPodSandbox(ctx context.Context, in *criv1.ListPodSandboxRequest, _ ...grpc.CallOption) (*criv1.ListPodSandboxResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ListPodSandbox(ctx, in)\n}\n\nfunc (c *client) CreateContainer(ctx context.Context, in *criv1.CreateContainerRequest, _ ...grpc.CallOption) (*criv1.CreateContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.CreateContainer(ctx, in)\n}\n\nfunc (c *client) StartContainer(ctx context.Context, in *criv1.StartContainerRequest, _ ...grpc.CallOption) (*criv1.StartContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.StartContainer(ctx, in)\n}\n\nfunc (c *client) StopContainer(ctx context.Context, in *criv1.StopContainerRequest, _ ...grpc.CallOption) (*criv1.StopContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.StopContainer(ctx, in)\n}\n\nfunc (c *client) RemoveContainer(ctx context.Context, in *criv1.RemoveContainerRequest, _ ...grpc.CallOption) (*criv1.RemoveContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.RemoveContainer(ctx, in)\n}\n\nfunc (c *client) ListContainers(ctx context.Context, in *criv1.ListContainersRequest, _ ...grpc.CallOption) (*criv1.ListContainersResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ListContainers(ctx, in)\n}\n\nfunc (c *client) ContainerStatus(ctx context.Context, in *criv1.ContainerStatusRequest, _ ...grpc.CallOption) (*criv1.ContainerStatusResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ContainerStatus(ctx, in)\n}\n\nfunc (c *client) UpdateContainerResources(ctx context.Context, in *criv1.UpdateContainerResourcesRequest, _ ...grpc.CallOption) (*criv1.UpdateContainerResourcesResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.UpdateContainerResources(ctx, in)\n}\n\nfunc (c *client) ReopenContainerLog(ctx context.Context, in *criv1.ReopenContainerLogRequest, _ ...grpc.CallOption) (*criv1.ReopenContainerLogResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ReopenContainerLog(ctx, in)\n}\n\nfunc (c *client) ExecSync(ctx context.Context, in *criv1.ExecSyncRequest, _ ...grpc.CallOption) (*criv1.ExecSyncResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ExecSync(ctx, in)\n}\n\nfunc (c *client) Exec(ctx context.Context, in *criv1.ExecRequest, _ ...grpc.CallOption) (*criv1.ExecResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.Exec(ctx, in)\n}\n\nfunc (c *client) Attach(ctx context.Context, in *criv1.AttachRequest, _ ...grpc.CallOption) (*criv1.AttachResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.Attach(ctx, in)\n}\n\nfunc (c *client) PortForward(ctx context.Context, in *criv1.PortForwardRequest, _ ...grpc.CallOption) (*criv1.PortForwardResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.PortForward(ctx, in)\n}\n\nfunc (c *client) ContainerStats(ctx context.Context, in *criv1.ContainerStatsRequest, _ ...grpc.CallOption) (*criv1.ContainerStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ContainerStats(ctx, in)\n}\n\nfunc (c *client) ListContainerStats(ctx context.Context, in *criv1.ListContainerStatsRequest, _ ...grpc.CallOption) (*criv1.ListContainerStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ListContainerStats(ctx, in)\n}\n\nfunc (c *client) PodSandboxStats(ctx context.Context, in *criv1.PodSandboxStatsRequest, _ ...grpc.CallOption) (*criv1.PodSandboxStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.PodSandboxStats(ctx, in)\n}\n\nfunc (c *client) ListPodSandboxStats(ctx context.Context, in *criv1.ListPodSandboxStatsRequest, _ ...grpc.CallOption) (*criv1.ListPodSandboxStatsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ListPodSandboxStats(ctx, in)\n}\n\nfunc (c *client) UpdateRuntimeConfig(ctx context.Context, in *criv1.UpdateRuntimeConfigRequest, _ ...grpc.CallOption) (*criv1.UpdateRuntimeConfigResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.UpdateRuntimeConfig(ctx, in)\n}\n\nfunc (c *client) Status(ctx context.Context, in *criv1.StatusRequest, _ ...grpc.CallOption) (*criv1.StatusResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.Status(ctx, in)\n}\n\nfunc (c *client) CheckpointContainer(ctx context.Context, in *criv1.CheckpointContainerRequest, _ ...grpc.CallOption) (*criv1.CheckpointContainerResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.CheckpointContainer(ctx, in)\n}\n\nfunc (c *client) GetContainerEvents(ctx context.Context, in *criv1.GetEventsRequest, _ ...grpc.CallOption) (criv1.RuntimeService_GetContainerEventsClient, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\teventsClient, err := c.rsc.GetContainerEvents(ctx, in)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn eventsClient, err\n}\n\nfunc (c *client) ListMetricDescriptors(ctx context.Context, in *criv1.ListMetricDescriptorsRequest, _ ...grpc.CallOption) (*criv1.ListMetricDescriptorsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ListMetricDescriptors(ctx, in)\n}\n\nfunc (c *client) ListPodSandboxMetrics(ctx context.Context, in *criv1.ListPodSandboxMetricsRequest, _ ...grpc.CallOption) (*criv1.ListPodSandboxMetricsResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.ListPodSandboxMetrics(ctx, in)\n}\n\nfunc (c *client) RuntimeConfig(ctx context.Context, in *criv1.RuntimeConfigRequest, _ ...grpc.CallOption) (*criv1.RuntimeConfigResponse, error) {\n\tif err := c.checkRuntimeService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.rsc.RuntimeConfig(ctx, in)\n}\n\nfunc (c *client) ListImages(ctx context.Context, in *criv1.ListImagesRequest, _ ...grpc.CallOption) (*criv1.ListImagesResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.isc.ListImages(ctx, in)\n}\n\nfunc (c *client) ImageStatus(ctx context.Context, in *criv1.ImageStatusRequest, _ ...grpc.CallOption) (*criv1.ImageStatusResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.isc.ImageStatus(ctx, in)\n}\n\nfunc (c *client) PullImage(ctx context.Context, in *criv1.PullImageRequest, _ ...grpc.CallOption) (*criv1.PullImageResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.isc.PullImage(ctx, in)\n}\n\nfunc (c *client) RemoveImage(ctx context.Context, in *criv1.RemoveImageRequest, _ ...grpc.CallOption) (*criv1.RemoveImageResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.isc.RemoveImage(ctx, in)\n}\n\nfunc (c *client) ImageFsInfo(ctx context.Context, in *criv1.ImageFsInfoRequest, _ ...grpc.CallOption) (*criv1.ImageFsInfoResponse, error) {\n\tif err := c.checkImageService(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.isc.ImageFsInfo(ctx, in)\n}\n\n// Return a formatted client-specific error.\nfunc clientError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"cri/client: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/relay/image-service.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage relay\n\nimport (\n\t\"context\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n)\n\nfunc (r *relay) ListImages(ctx context.Context,\n\treq *criv1.ListImagesRequest) (*criv1.ListImagesResponse, error) {\n\treturn r.client.ListImages(ctx, req)\n}\n\nfunc (r *relay) ImageStatus(ctx context.Context,\n\treq *criv1.ImageStatusRequest) (*criv1.ImageStatusResponse, error) {\n\treturn r.client.ImageStatus(ctx, req)\n}\n\nfunc (r *relay) PullImage(ctx context.Context,\n\treq *criv1.PullImageRequest) (*criv1.PullImageResponse, error) {\n\treturn r.client.PullImage(ctx, req)\n}\n\nfunc (r *relay) RemoveImage(ctx context.Context,\n\treq *criv1.RemoveImageRequest) (*criv1.RemoveImageResponse, error) {\n\treturn r.client.RemoveImage(ctx, req)\n}\n\nfunc (r *relay) ImageFsInfo(ctx context.Context,\n\treq *criv1.ImageFsInfoRequest) (*criv1.ImageFsInfoResponse, error) {\n\treturn r.client.ImageFsInfo(ctx, req)\n}\n"
  },
  {
    "path": "pkg/cri/relay/relay.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage relay\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/server\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// DisableService is used to mark a socket/service to not be connected.\n\tDisableService = client.DontConnect\n\t// DefaultImageSocket uses the runtime socket for the image servie, too.\n\tDefaultImageSocket = \"default\"\n)\n\n// Options contains the configurable options of our CRI relay.\ntype Options struct {\n\t// RelaySocket is the socket path for the CRI relay services.\n\tRelaySocket string\n\t// ImageSocket is the socket path for the (real) CRI image services.\n\tImageSocket string\n\t// RuntimeSocket is the socket path for the (real) CRI runtime services.\n\tRuntimeSocket string\n\t// QualifyReqFn produces context for disambiguating a CRI request/reply.\n\tQualifyReqFn func(interface{}) string\n}\n\n// Relay is the interface we expose for controlling our CRI relay.\ntype Relay interface {\n\t// Setup prepares the relay to start processing CRI requests.\n\tSetup() error\n\t// Start starts the relay.\n\tStart() error\n\t// Stop stops the relay.\n\tStop()\n\t// Client returns the relays client interface.\n\tClient() client.Client\n\t// Server returns the relays server interface.\n\tServer() server.Server\n}\n\n// relay is the implementation of Relay.\ntype relay struct {\n\tlogger.Logger\n\tsync.Mutex\n\toptions Options       // relay options\n\tclient  client.Client // relay CRI client\n\tserver  server.Server // relay CRI server\n\n\tevtClient criv1.RuntimeService_GetContainerEventsClient\n\tevtChans  map[*criv1.GetEventsRequest]chan *criv1.ContainerEventResponse\n}\n\n// NewRelay creates a new relay instance.\nfunc NewRelay(options Options) (Relay, error) {\n\tvar err error\n\n\tr := &relay{\n\t\tLogger:   logger.NewLogger(\"cri/relay\"),\n\t\toptions:  options,\n\t\tevtChans: map[*criv1.GetEventsRequest]chan *criv1.ContainerEventResponse{},\n\t}\n\n\timageSocket := r.options.ImageSocket\n\tif imageSocket == DefaultImageSocket {\n\t\timageSocket = r.options.RuntimeSocket\n\t}\n\n\tcltopts := client.Options{\n\t\tImageSocket:   imageSocket,\n\t\tRuntimeSocket: r.options.RuntimeSocket,\n\t\tDialNotify:    r.dialNotify,\n\t}\n\tif r.client, err = client.NewClient(cltopts); err != nil {\n\t\treturn nil, relayError(\"failed to create relay client: %v\", err)\n\t}\n\n\tsrvopts := server.Options{\n\t\tSocket:       r.options.RelaySocket,\n\t\tUser:         -1,\n\t\tGroup:        -1,\n\t\tMode:         0660,\n\t\tQualifyReqFn: r.options.QualifyReqFn,\n\t}\n\tif r.server, err = server.NewServer(srvopts); err != nil {\n\t\treturn nil, relayError(\"failed to create relay server: %v\", err)\n\t}\n\n\treturn r, nil\n}\n\n// Setup prepares the relay to start processing requests.\nfunc (r *relay) Setup() error {\n\tif err := r.client.Connect(client.ConnectOptions{Wait: true}); err != nil {\n\t\treturn relayError(\"client connection failed: %v\", err)\n\t}\n\n\tif r.options.ImageSocket != DisableService {\n\t\tif err := r.server.RegisterImageService(r); err != nil {\n\t\t\treturn relayError(\"failed to register image service: %v\", err)\n\t\t}\n\t}\n\n\tif r.options.RuntimeSocket != DisableService {\n\t\tif err := r.server.RegisterRuntimeService(r); err != nil {\n\t\t\treturn relayError(\"failed to register runtime service: %v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Start starts the relays request processing goroutine.\nfunc (r *relay) Start() error {\n\tif err := r.server.Start(); err != nil {\n\t\treturn relayError(\"failed to start relay: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// Stop stops the relay.\nfunc (r *relay) Stop() {\n\tr.client.Close()\n\tr.server.Stop()\n}\n\n// Client returns the relays Client interface.\nfunc (r *relay) Client() client.Client {\n\treturn r.client\n}\n\n// Server returns the relays Server interface.\nfunc (r *relay) Server() server.Server {\n\treturn r.server\n}\n\nfunc (r *relay) dialNotify(socket string, uid int, gid int, mode os.FileMode, err error) {\n\tif err != nil {\n\t\tr.Error(\"failed to determine permissions/ownership of client socket %q: %v\",\n\t\t\tsocket, err)\n\t\treturn\n\t}\n\n\t// Notes:\n\t//   Kubelet has separate configuration/command line options for the container\n\t//   runtime's Image and Runtime Services. Hence, in principle it is possible\n\t//   that we run with two separate sockets for those. However, we always expose\n\t//   both services over our single relay socket. Since we cannot set two set of\n\t//   ownerships and permissions on a single socket, if this situation arises in\n\t//   practice we choose to go with the runtime socket's properties.\n\tif r.options.ImageSocket != r.options.RuntimeSocket {\n\t\tif socket != r.options.RuntimeSocket && r.options.RuntimeSocket != client.DontConnect {\n\t\t\tr.Warn(\"ignoring ownership/permissions of dedicated CR Image Service socket...\")\n\t\t\treturn\n\t\t}\n\t}\n\n\tif err := r.server.Chown(uid, gid); err != nil {\n\t\tr.Error(\"server socket ownership change request failed: %v\", err)\n\t} else {\n\t\tif err := r.server.Chmod(mode); err != nil {\n\t\t\tr.Error(\"server socket permissions change request failed: %v\", err)\n\t\t}\n\t}\n}\n\n// relayError creates a formatted relay-specific error.\nfunc relayError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"cri/relay: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/relay/runtime-service.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage relay\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/dump\"\n)\n\nfunc (r *relay) dump(method string, req interface{}) {\n\tif r.DebugEnabled() {\n\t\tqualif := r.qualifier(req)\n\t\tdump.RequestMessage(\"relayed\", method, qualif, req, true)\n\t}\n}\n\n// qualifier pulls a qualifier for disambiguation from a CRI request message.\nfunc (r *relay) qualifier(msg interface{}) string {\n\tif fn := r.options.QualifyReqFn; fn != nil {\n\t\treturn fn(msg)\n\t}\n\treturn \"\"\n}\n\nfunc (r *relay) Version(ctx context.Context,\n\treq *criv1.VersionRequest) (*criv1.VersionResponse, error) {\n\tr.dump(\"Version\", req)\n\treturn r.client.Version(ctx, req)\n}\n\nfunc (r *relay) RunPodSandbox(ctx context.Context,\n\treq *criv1.RunPodSandboxRequest) (*criv1.RunPodSandboxResponse, error) {\n\tr.dump(\"RunPodSandbox\", req)\n\treturn r.client.RunPodSandbox(ctx, req)\n}\n\nfunc (r *relay) StopPodSandbox(ctx context.Context,\n\treq *criv1.StopPodSandboxRequest) (*criv1.StopPodSandboxResponse, error) {\n\tr.dump(\"StopPodSandbox\", req)\n\treturn r.client.StopPodSandbox(ctx, req)\n}\n\nfunc (r *relay) RemovePodSandbox(ctx context.Context,\n\treq *criv1.RemovePodSandboxRequest) (*criv1.RemovePodSandboxResponse, error) {\n\tr.dump(\"RemovePodSandbox\", req)\n\treturn r.client.RemovePodSandbox(ctx, req)\n}\n\nfunc (r *relay) PodSandboxStatus(ctx context.Context,\n\treq *criv1.PodSandboxStatusRequest) (*criv1.PodSandboxStatusResponse, error) {\n\tr.dump(\"PodSandboxStatus\", req)\n\treturn r.client.PodSandboxStatus(ctx, req)\n}\n\nfunc (r *relay) ListPodSandbox(ctx context.Context,\n\treq *criv1.ListPodSandboxRequest) (*criv1.ListPodSandboxResponse, error) {\n\tr.dump(\"ListPodSandbox\", req)\n\treturn r.client.ListPodSandbox(ctx, req)\n}\n\nfunc (r *relay) CreateContainer(ctx context.Context,\n\treq *criv1.CreateContainerRequest) (*criv1.CreateContainerResponse, error) {\n\tr.dump(\"CreateContainer\", req)\n\treturn r.client.CreateContainer(ctx, req)\n}\n\nfunc (r *relay) StartContainer(ctx context.Context,\n\treq *criv1.StartContainerRequest) (*criv1.StartContainerResponse, error) {\n\tr.dump(\"StartContainer\", req)\n\treturn r.client.StartContainer(ctx, req)\n}\n\nfunc (r *relay) StopContainer(ctx context.Context,\n\treq *criv1.StopContainerRequest) (*criv1.StopContainerResponse, error) {\n\tr.dump(\"StopContainer\", req)\n\treturn r.client.StopContainer(ctx, req)\n}\n\nfunc (r *relay) RemoveContainer(ctx context.Context,\n\treq *criv1.RemoveContainerRequest) (*criv1.RemoveContainerResponse, error) {\n\tr.dump(\"RemoveContainer\", req)\n\treturn r.client.RemoveContainer(ctx, req)\n}\n\nfunc (r *relay) ListContainers(ctx context.Context,\n\treq *criv1.ListContainersRequest) (*criv1.ListContainersResponse, error) {\n\tr.dump(\"ListContainers\", req)\n\treturn r.client.ListContainers(ctx, req)\n}\n\nfunc (r *relay) ContainerStatus(ctx context.Context,\n\treq *criv1.ContainerStatusRequest) (*criv1.ContainerStatusResponse, error) {\n\tr.dump(\"ContainerStatus\", req)\n\treturn r.client.ContainerStatus(ctx, req)\n}\n\nfunc (r *relay) UpdateContainerResources(ctx context.Context,\n\treq *criv1.UpdateContainerResourcesRequest) (*criv1.UpdateContainerResourcesResponse, error) {\n\tr.dump(\"UpdateContainerResources\", req)\n\treturn r.client.UpdateContainerResources(ctx, req)\n}\n\nfunc (r *relay) ReopenContainerLog(ctx context.Context,\n\treq *criv1.ReopenContainerLogRequest) (*criv1.ReopenContainerLogResponse, error) {\n\tr.dump(\"ReopenContainerLog\", req)\n\treturn r.client.ReopenContainerLog(ctx, req)\n}\n\nfunc (r *relay) ExecSync(ctx context.Context,\n\treq *criv1.ExecSyncRequest) (*criv1.ExecSyncResponse, error) {\n\tr.dump(\"ExecSync\", req)\n\treturn r.client.ExecSync(ctx, req)\n}\n\nfunc (r *relay) Exec(ctx context.Context,\n\treq *criv1.ExecRequest) (*criv1.ExecResponse, error) {\n\tr.dump(\"Exec\", req)\n\treturn r.client.Exec(ctx, req)\n}\n\nfunc (r *relay) Attach(ctx context.Context,\n\treq *criv1.AttachRequest) (*criv1.AttachResponse, error) {\n\tr.dump(\"Attach\", req)\n\treturn r.client.Attach(ctx, req)\n}\n\nfunc (r *relay) PortForward(ctx context.Context,\n\treq *criv1.PortForwardRequest) (*criv1.PortForwardResponse, error) {\n\tr.dump(\"PortForward\", req)\n\treturn r.client.PortForward(ctx, req)\n}\n\nfunc (r *relay) ContainerStats(ctx context.Context,\n\treq *criv1.ContainerStatsRequest) (*criv1.ContainerStatsResponse, error) {\n\tr.dump(\"ContainerStats\", req)\n\treturn r.client.ContainerStats(ctx, req)\n}\n\nfunc (r *relay) ListContainerStats(ctx context.Context,\n\treq *criv1.ListContainerStatsRequest) (*criv1.ListContainerStatsResponse, error) {\n\tr.dump(\"ListContainerStats\", req)\n\treturn r.client.ListContainerStats(ctx, req)\n}\n\nfunc (r *relay) PodSandboxStats(ctx context.Context,\n\treq *criv1.PodSandboxStatsRequest) (*criv1.PodSandboxStatsResponse, error) {\n\tr.dump(\"PodSandboxStats\", req)\n\treturn r.client.PodSandboxStats(ctx, req)\n}\n\nfunc (r *relay) ListPodSandboxStats(ctx context.Context,\n\treq *criv1.ListPodSandboxStatsRequest) (*criv1.ListPodSandboxStatsResponse, error) {\n\tr.dump(\"ListPodSandboxStats\", req)\n\treturn r.client.ListPodSandboxStats(ctx, req)\n}\n\nfunc (r *relay) UpdateRuntimeConfig(ctx context.Context,\n\treq *criv1.UpdateRuntimeConfigRequest) (*criv1.UpdateRuntimeConfigResponse, error) {\n\tr.dump(\"UpdateRuntimeConfig\", req)\n\treturn r.client.UpdateRuntimeConfig(ctx, req)\n}\n\nfunc (r *relay) Status(ctx context.Context,\n\treq *criv1.StatusRequest) (*criv1.StatusResponse, error) {\n\tr.dump(\"Status\", req)\n\treturn r.client.Status(ctx, req)\n}\n\nfunc (r *relay) CheckpointContainer(ctx context.Context, req *criv1.CheckpointContainerRequest) (*criv1.CheckpointContainerResponse, error) {\n\tr.dump(\"CheckpointContainer\", req)\n\treturn r.client.CheckpointContainer(ctx, req)\n}\n\nfunc (r *relay) GetContainerEvents(req *criv1.GetEventsRequest, srv criv1.RuntimeService_GetContainerEventsServer) error {\n\tevtC := r.addEventServer(req)\n\n\tif err := r.startEventRelay(req); err != nil {\n\t\tr.delEventServer(req)\n\t\treturn err\n\t}\n\n\tfor evt := range evtC {\n\t\tif err := srv.Send(evt); err != nil {\n\t\t\tr.Errorf(\"failed to relay/send container event: %v\", err)\n\t\t\tr.delEventServer(req)\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (r *relay) ListMetricDescriptors(ctx context.Context, req *criv1.ListMetricDescriptorsRequest) (*criv1.ListMetricDescriptorsResponse, error) {\n\tr.dump(\"ListMetricDescriptors\", req)\n\treturn r.client.ListMetricDescriptors(ctx, req)\n}\n\nfunc (r *relay) ListPodSandboxMetrics(ctx context.Context, req *criv1.ListPodSandboxMetricsRequest) (*criv1.ListPodSandboxMetricsResponse, error) {\n\tr.dump(\"ListPodSandboxMetrics\", req)\n\treturn r.client.ListPodSandboxMetrics(ctx, req)\n}\n\nfunc (r *relay) RuntimeConfig(ctx context.Context, req *criv1.RuntimeConfigRequest) (*criv1.RuntimeConfigResponse, error) {\n\tr.dump(\"RuntimeConfig\", req)\n\treturn r.client.RuntimeConfig(ctx, req)\n}\n\nconst (\n\teventRelayTimeout = 1 * time.Second\n)\n\nfunc (r *relay) addEventServer(req *criv1.GetEventsRequest) chan *criv1.ContainerEventResponse {\n\tr.Lock()\n\tdefer r.Unlock()\n\n\tevtC := make(chan *criv1.ContainerEventResponse, 128)\n\tr.evtChans[req] = evtC\n\n\treturn evtC\n}\n\nfunc (r *relay) delEventServer(req *criv1.GetEventsRequest) chan *criv1.ContainerEventResponse {\n\tr.Lock()\n\tdefer r.Unlock()\n\n\tevtC := r.evtChans[req]\n\tdelete(r.evtChans, req)\n\n\treturn evtC\n}\n\nfunc (r *relay) startEventRelay(req *criv1.GetEventsRequest) error {\n\tr.Lock()\n\tdefer r.Unlock()\n\n\tif r.evtClient != nil {\n\t\treturn nil\n\t}\n\n\tc, err := r.client.GetContainerEvents(context.Background(), req)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create container event client: %w\", err)\n\t}\n\n\tr.evtClient = c\n\tgo r.relayEvents()\n\n\treturn nil\n}\n\nfunc (r *relay) relayEvents() {\n\tfor {\n\t\tevt, err := r.evtClient.Recv()\n\t\tif err != nil {\n\t\t\tr.Errorf(\"failed to relay/receive container event: %v\", err)\n\t\t}\n\n\t\tr.Lock()\n\n\t\tif err != nil {\n\t\t\tfor req, evtC := range r.evtChans {\n\t\t\t\tdelete(r.evtChans, req)\n\t\t\t\tclose(evtC)\n\t\t\t}\n\t\t\tr.evtClient = nil\n\t\t} else {\n\t\t\tfor req, evtC := range r.evtChans {\n\t\t\t\tselect {\n\t\t\t\tcase evtC <- evt:\n\t\t\t\tcase _ = <-time.After(eventRelayTimeout):\n\t\t\t\t\tdelete(r.evtChans, req)\n\t\t\t\t\tclose(evtC)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tr.Unlock()\n\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/agent/agent.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage agent\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"google.golang.org/grpc\"\n\tcore_v1 \"k8s.io/api/core/v1\"\n\n\tagent_v1 \"github.com/intel/cri-resource-manager/pkg/agent/api/v1\"\n)\n\nconst (\n\tSocketDisabled = \"disabled\"\n)\n\n// Interface describe interfaces of cri-resource-manager agent\ntype Interface interface {\n\tIsDisabled() bool\n\n\tGetNode(time.Duration) (core_v1.Node, error)\n\tPatchNode([]*agent_v1.JsonPatch, time.Duration) error\n\tUpdateNodeCapacity(map[string]string, time.Duration) error\n\n\tGetLabels(time.Duration) (map[string]string, error)\n\tSetLabels(map[string]string, time.Duration) error\n\tRemoveLabels([]string, time.Duration) error\n\n\tGetAnnotations(time.Duration) (map[string]string, error)\n\tSetAnnotations(map[string]string, time.Duration) error\n\tRemoveAnnotations([]string, time.Duration) error\n\n\tGetTaints(time.Duration) ([]core_v1.Taint, error)\n\tSetTaints([]core_v1.Taint, time.Duration) error\n\tRemoveTaints([]core_v1.Taint, time.Duration) error\n\n\tFindTaintIndex([]core_v1.Taint, *core_v1.Taint) (int, bool)\n}\n\n// agentInterface implements Interface\ntype agentInterface struct {\n\tsocket string\n\tcli    agent_v1.AgentClient\n}\n\n// NewAgentInterface connects to cri-resource-manager-agent gRPC server\n// and return a new Interface\nfunc NewAgentInterface(socket string) (Interface, error) {\n\ta := &agentInterface{\n\t\tsocket: socket,\n\t}\n\n\tif a.IsDisabled() {\n\t\treturn a, nil\n\t}\n\n\tdialOpts := []grpc.DialOption{\n\t\t//\t\tgrpc.WithBlock(),\n\t\t//\t\tgrpc.WithTimeout(10 * time.Second),\n\t\tgrpc.WithInsecure(),\n\t\t//\t\tgrpc.FailOnNonTempDialError(true),\n\t\tgrpc.WithDialer(func(sock string, timeout time.Duration) (net.Conn, error) {\n\t\t\treturn net.Dial(\"unix\", sock)\n\t\t}),\n\t}\n\tconn, err := grpc.Dial(socket, dialOpts...)\n\tif err != nil {\n\t\treturn nil, agentError(\"failed to connect to cri-resmgr agent: %v\", err)\n\t}\n\ta.cli = agent_v1.NewAgentClient(conn)\n\n\treturn a, nil\n}\n\n// IsDisabled returns true if the agent interface is disabled.\nfunc (a *agentInterface) IsDisabled() bool {\n\treturn a.socket == SocketDisabled || a.socket == \"\"\n}\n\nfunc (a *agentInterface) GetNode(timeout time.Duration) (core_v1.Node, error) {\n\tif a.IsDisabled() {\n\t\treturn core_v1.Node{}, agentError(\"agent interface is disabled\")\n\t}\n\n\tctx, cancel, callOpts := prepareCall(timeout)\n\tdefer cancel()\n\n\treq := &agent_v1.GetNodeRequest{}\n\n\tnode := core_v1.Node{}\n\trsp, err := a.cli.GetNode(ctx, req, callOpts...)\n\tif err != nil {\n\t\treturn node, agentError(\"failed to get node object: %v\", err)\n\t}\n\n\tif err = json.Unmarshal([]byte(rsp.Node), &node); err != nil {\n\t\treturn node, agentError(\"invalid response, failed to unmarshal v1.Node: %v\", err)\n\t}\n\n\treturn node, nil\n}\n\nfunc (a *agentInterface) PatchNode(patches []*agent_v1.JsonPatch, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tctx, cancel, callOpts := prepareCall(timeout)\n\tdefer cancel()\n\n\treq := &agent_v1.PatchNodeRequest{\n\t\tPatches: patches,\n\t}\n\n\t_, err := a.cli.PatchNode(ctx, req, callOpts...)\n\tif err != nil {\n\t\treturn agentError(\"failed to patch node object: %v\", err)\n\t}\n\treturn nil\n}\n\nfunc (a *agentInterface) UpdateNodeCapacity(caps map[string]string, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tctx, cancel, callOpts := prepareCall(timeout)\n\tdefer cancel()\n\n\treq := &agent_v1.UpdateNodeCapacityRequest{\n\t\tCapacities: caps,\n\t}\n\t_, err := a.cli.UpdateNodeCapacity(ctx, req, callOpts...)\n\tif err != nil {\n\t\treturn agentError(\"failed to update node capacities: %v\", err)\n\t}\n\treturn nil\n}\n\nconst (\n\t// PatchAdd specifies an add operation.\n\tPatchAdd string = \"add\"\n\t// PatchRemove specifies an remove operation.\n\tPatchRemove string = \"remove\"\n\t// PatchReplace specifies an replace operation.\n\tPatchReplace string = \"replace\"\n)\n\nfunc patchPath(class, key string) string {\n\treturn \"/metadata/\" + class + \"/\" + strings.Replace(key, \"/\", \"~1\", -1)\n}\n\nfunc labelPatchPath(key string) string {\n\treturn patchPath(\"labels\", key)\n}\n\nfunc annotationPatchPath(key string) string {\n\treturn patchPath(\"annotations\", key)\n}\n\nfunc taintPatchPath(idx int) string {\n\treturn fmt.Sprintf(\"/spec/taints/%d\", idx)\n}\n\nfunc (a *agentInterface) GetLabels(timeout time.Duration) (map[string]string, error) {\n\tif a.IsDisabled() {\n\t\treturn nil, agentError(\"agent interface is disabled\")\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.Labels, nil\n}\n\nfunc (a *agentInterface) SetLabels(labels map[string]string, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tif len(labels) == 0 {\n\t\treturn nil\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpatches := []*agent_v1.JsonPatch{}\n\tfor key, val := range labels {\n\t\tpatch := &agent_v1.JsonPatch{\n\t\t\tPath: labelPatchPath(key),\n\t\t\t// Value is supposed to be in marshalled JSON format. Thus, we need\n\t\t\t// to add quotes so that it will be interpreted as a string.\n\t\t\tValue: \"\\\"\" + val + \"\\\"\",\n\t\t}\n\t\tif _, ok := node.Labels[key]; ok {\n\t\t\tpatch.Op = PatchReplace\n\t\t} else {\n\t\t\tpatch.Op = PatchAdd\n\t\t}\n\t\tpatches = append(patches, patch)\n\t}\n\n\treturn a.PatchNode(patches, timeout)\n}\n\nfunc (a *agentInterface) RemoveLabels(keys []string, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tif len(keys) == 0 {\n\t\treturn nil\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpatches := []*agent_v1.JsonPatch{}\n\tfor _, key := range keys {\n\t\tif _, ok := node.Labels[key]; !ok {\n\t\t\tcontinue\n\t\t}\n\t\tpatch := &agent_v1.JsonPatch{\n\t\t\tOp:   PatchRemove,\n\t\t\tPath: labelPatchPath(key),\n\t\t}\n\t\tpatches = append(patches, patch)\n\t}\n\tif len(patches) == 0 {\n\t\treturn nil\n\t}\n\n\treturn a.PatchNode(patches, timeout)\n}\n\nfunc (a *agentInterface) GetAnnotations(timeout time.Duration) (map[string]string, error) {\n\tif a.IsDisabled() {\n\t\treturn nil, agentError(\"agent interface is disabled\")\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn node.Annotations, nil\n}\n\nfunc (a *agentInterface) SetAnnotations(annotations map[string]string, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tif len(annotations) == 0 {\n\t\treturn nil\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpatches := []*agent_v1.JsonPatch{}\n\tfor key, val := range annotations {\n\t\tpatch := &agent_v1.JsonPatch{\n\t\t\tPath:  annotationPatchPath(key),\n\t\t\tValue: val,\n\t\t}\n\t\tif _, ok := node.Annotations[key]; ok {\n\t\t\tpatch.Op = PatchReplace\n\t\t} else {\n\t\t\tpatch.Op = PatchAdd\n\t\t}\n\t\tpatches = append(patches, patch)\n\t}\n\n\treturn a.PatchNode(patches, timeout)\n}\n\nfunc (a *agentInterface) RemoveAnnotations(keys []string, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tif len(keys) == 0 {\n\t\treturn nil\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpatches := []*agent_v1.JsonPatch{}\n\tfor _, key := range keys {\n\t\tif _, ok := node.Annotations[key]; !ok {\n\t\t\tcontinue\n\t\t}\n\n\t\tpatch := &agent_v1.JsonPatch{\n\t\t\tOp:   PatchRemove,\n\t\t\tPath: annotationPatchPath(key),\n\t\t}\n\t\tpatches = append(patches, patch)\n\t}\n\tif len(patches) == 0 {\n\t\treturn nil\n\t}\n\n\treturn a.PatchNode(patches, timeout)\n}\n\nfunc (a *agentInterface) GetTaints(timeout time.Duration) ([]core_v1.Taint, error) {\n\tif a.IsDisabled() {\n\t\treturn nil, agentError(\"agent interface is disabled\")\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn node.Spec.Taints, nil\n}\n\nfunc (a *agentInterface) SetTaints(taints []core_v1.Taint, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tif len(taints) == 0 {\n\t\treturn nil\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpatches := []*agent_v1.JsonPatch{}\n\tif node.Spec.Taints == nil {\n\t\tpatch := &agent_v1.JsonPatch{\n\t\t\tOp:    PatchAdd,\n\t\t\tPath:  \"/spec/taints\",\n\t\t\tValue: \"[]\"}\n\t\tpatches = append(patches, patch)\n\t}\n\n\tfor _, t := range taints {\n\t\tvalue, err := json.Marshal(t)\n\t\tif err != nil {\n\t\t\treturn agentError(\"BUG: failed to marshal taint %v: %v\", t, err)\n\t\t}\n\t\tidx, found := findTaintIndex(node.Spec.Taints, &t)\n\t\tpatch := &agent_v1.JsonPatch{Value: string(value)}\n\t\tpatch.Path = taintPatchPath(idx)\n\t\tif !found {\n\t\t\tpatch.Op = PatchAdd\n\t\t} else {\n\t\t\tpatch.Op = PatchReplace\n\t\t}\n\t\tpatches = append(patches, patch)\n\t}\n\n\treturn a.PatchNode(patches, timeout)\n}\n\nfunc (a *agentInterface) RemoveTaints(taints []core_v1.Taint, timeout time.Duration) error {\n\tif a.IsDisabled() {\n\t\treturn agentError(\"agent interface is disabled\")\n\t}\n\n\tif len(taints) == 0 {\n\t\treturn nil\n\t}\n\n\tnode, err := a.GetNode(timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif node.Spec.Taints == nil {\n\t\treturn nil\n\t}\n\n\tpatches := []*agent_v1.JsonPatch{}\n\tfor _, t := range taints {\n\t\tidx, found := findTaintIndex(node.Spec.Taints, &t)\n\t\tif found {\n\t\t\tpatch := &agent_v1.JsonPatch{\n\t\t\t\tOp:   \"remove\",\n\t\t\t\tPath: taintPatchPath(idx),\n\t\t\t}\n\t\t\tpatches = append(patches, patch)\n\t\t}\n\t}\n\tif len(patches) == 0 {\n\t\treturn nil\n\t}\n\n\treturn a.PatchNode(patches, timeout)\n}\n\nfunc findTaintIndex(taints []core_v1.Taint, taint *core_v1.Taint) (int, bool) {\n\tfor idx, t := range taints {\n\t\tif t.Key == taint.Key && t.Value == taint.Value && t.Effect == taint.Effect {\n\t\t\treturn idx, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc (a *agentInterface) FindTaintIndex(taints []core_v1.Taint, taint *core_v1.Taint) (int, bool) {\n\treturn findTaintIndex(taints, taint)\n}\n\nfunc agentError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"agent-client: \"+format, args...)\n}\n\nfunc prepareCall(timeout time.Duration) (context.Context, context.CancelFunc, []grpc.CallOption) {\n\tcallOpts := []grpc.CallOption{grpc.FailFast(false)}\n\tctx := context.Background()\n\tcancel := func() {}\n\tif timeout >= 0 {\n\t\tctx, cancel = context.WithTimeout(context.Background(), timeout)\n\t}\n\n\treturn ctx, cancel, callOpts\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/builtin-policies.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t// List of builtin policies\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/balloons\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/dynamic-pools\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/none\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/podpools\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/static\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/static-plus\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/static-pools\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy/builtin/topology-aware\"\n)\n\n// TODO: add unit tests to verify that all builtin policies are found\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/affinity.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"fmt\"\n\n\t\"sigs.k8s.io/yaml\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n)\n\nconst (\n\t// annotation key for specifying container affinity rules\n\tkeyAffinity = \"affinity\"\n\t// annotation key for specifying container anti-affinity rules\n\tkeyAntiAffinity = \"anti-affinity\"\n)\n\n// Expression is used to describe affinity container scope and matching criteria.\ntype Expression struct {\n\tresmgr.Expression\n}\n\n// simpleAffinity is an alternative, simplified syntax for intra-pod container affinity.\ntype simpleAffinity map[string][]string\n\n// PodContainerAffinity defines a set of per-container affinities and anti-affinities.\ntype podContainerAffinity map[string][]*Affinity\n\n// Affinity specifies a single container affinity.\ntype Affinity struct {\n\tScope  *resmgr.Expression `json:\"scope,omitempty\"`  // scope for evaluating this affinity\n\tMatch  *resmgr.Expression `json:\"match\"`            // affinity expression\n\tWeight int32              `json:\"weight,omitempty\"` // (optional) weight for this affinity\n}\n\nconst (\n\t// UserWeightCutoff is the cutoff we clamp user-provided weights to.\n\tUserWeightCutoff = 1000\n\t// DefaultWeight is the default assigned weight if omitted in annotations.\n\tDefaultWeight int32 = 1\n)\n\n// ImplicitAffinity can implicitly inject affinities to containers.\ntype ImplicitAffinity func(Container, bool) *Affinity\n\n// Validate checks the affinity for (obvious) invalidity.\nfunc (a *Affinity) Validate() error {\n\tif err := a.Scope.Validate(); err != nil {\n\t\treturn cacheError(\"invalid affinity scope: %v\", err)\n\t}\n\n\tif err := a.Match.Validate(); err != nil {\n\t\treturn cacheError(\"invalid affinity match: %v\", err)\n\t}\n\n\tswitch {\n\tcase a.Weight > UserWeightCutoff:\n\t\ta.Weight = UserWeightCutoff\n\tcase a.Weight < -UserWeightCutoff:\n\t\ta.Weight = -UserWeightCutoff\n\t}\n\n\treturn nil\n}\n\n// EvaluateAffinity evaluates the given affinity against all known in-scope containers.\nfunc (cch *cache) EvaluateAffinity(a *Affinity) map[string]int32 {\n\tresults := make(map[string]int32)\n\tfor _, c := range cch.FilterScope(a.Scope) {\n\t\tif a.Match.Evaluate(c) {\n\t\t\tid := c.GetCacheID()\n\t\t\tresults[id] += a.Weight\n\t\t}\n\t}\n\treturn results\n}\n\n// FilterScope returns the containers selected by the scope expression.\nfunc (cch *cache) FilterScope(scope *resmgr.Expression) []Container {\n\tcch.Debug(\"calculating scope %s\", scope.String())\n\tresult := []Container{}\n\tfor _, c := range cch.GetContainers() {\n\t\tif scope.Evaluate(c) {\n\t\t\tcch.Debug(\"  + container %s: IN scope\", c.PrettyName())\n\t\t\tresult = append(result, c)\n\t\t} else {\n\t\t\tcch.Debug(\"  - container %s: NOT IN scope\", c.PrettyName())\n\t\t}\n\t}\n\treturn result\n}\n\n// String returns the affinity as a string.\nfunc (a *Affinity) String() string {\n\tkind := \"\"\n\tif a.Weight < 0 {\n\t\tkind = \"anti-\"\n\t}\n\treturn fmt.Sprintf(\"<%saffinity: scope %s %s => %d>\",\n\t\tkind, a.Scope.String(), a.Match.String(), a.Weight)\n}\n\n// Try to parse affinities in simplified notation from the given annotation value.\nfunc (pca *podContainerAffinity) parseSimple(pod *pod, value string, weight int32) bool {\n\tparsed := simpleAffinity{}\n\tif err := yaml.UnmarshalStrict([]byte(value), &parsed); err != nil {\n\t\treturn false\n\t}\n\n\tpodScope := pod.ScopeExpression()\n\n\t//\n\t// Notes:\n\t//   We turn affinities given in the simple notation into a symmetric set of\n\t//   affinities. IOW, if X has affinity on Y with wight W, then Y will have\n\t//   affinity on X with W as well. In practice this is done by\n\t//     1) ensuring there is an affinity Y: X for every affinity X: Y\n\t//     2) generating an affinity expression for every container with affinities\n\t//  The generated expression uses the operator Equal or In depending on whether\n\t//  if the container has affinities on exactly one container in the symmetric\n\t//  set.\n\t//\n\n\tsymmetric := map[string]map[string]struct{}{}\n\n\tfor name, values := range parsed {\n\t\tfor _, v := range values {\n\t\t\tforw, ok := symmetric[name]\n\t\t\tif !ok {\n\t\t\t\tforw = map[string]struct{}{}\n\t\t\t\tsymmetric[name] = forw\n\t\t\t}\n\t\t\tback, ok := symmetric[v]\n\t\t\tif !ok {\n\t\t\t\tback = map[string]struct{}{}\n\t\t\t\tsymmetric[v] = back\n\t\t\t}\n\t\t\tforw[v], back[name] = struct{}{}, struct{}{}\n\t\t}\n\t}\n\n\tvar op resmgr.Operator\n\tfor name, affinities := range symmetric {\n\t\tothers := []string{}\n\t\tfor o := range affinities {\n\t\t\tothers = append(others, o)\n\t\t}\n\t\tif len(others) == 1 {\n\t\t\top = resmgr.Equals\n\t\t} else {\n\t\t\top = resmgr.In\n\t\t}\n\t\t(*pca)[name] = append((*pca)[name],\n\t\t\t&Affinity{\n\t\t\t\tScope: podScope,\n\t\t\t\tMatch: &resmgr.Expression{\n\t\t\t\t\tKey:    kubernetes.ContainerNameLabel,\n\t\t\t\t\tOp:     op,\n\t\t\t\t\tValues: others,\n\t\t\t\t},\n\t\t\t\tWeight: weight,\n\t\t\t})\n\t}\n\n\treturn true\n}\n\n// Try to parse affinities in full notation from the given annotation value.\nfunc (pca *podContainerAffinity) parseFull(pod *pod, value string, weight int32) error {\n\tparsed := podContainerAffinity{}\n\tif err := yaml.UnmarshalStrict([]byte(value), &parsed); err != nil {\n\t\treturn cacheError(\"failed to parse affinity annotation '%s': %v\", value, err)\n\t}\n\n\tpodScope := pod.ScopeExpression()\n\tfor name, pa := range parsed {\n\t\tca, ok := (*pca)[name]\n\t\tif !ok {\n\t\t\tca = make([]*Affinity, 0, len(pa))\n\t\t}\n\n\t\tfor _, a := range pa {\n\t\t\tif a.Scope == nil {\n\t\t\t\ta.Scope = podScope\n\t\t\t}\n\t\t\tif a.Weight == 0 {\n\t\t\t\ta.Weight = weight\n\t\t\t} else {\n\t\t\t\tif weight < 0 {\n\t\t\t\t\ta.Weight *= -1\n\t\t\t\t}\n\t\t\t}\n\t\t\tif err := a.Validate(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tca = append(ca, a)\n\t\t}\n\n\t\t(*pca)[name] = ca\n\t}\n\n\treturn nil\n}\n\n// GlobalAffinity creates an affinity with all containers in scope.\nfunc GlobalAffinity(key string, weight int32) *Affinity {\n\treturn &Affinity{\n\t\tScope: &resmgr.Expression{\n\t\t\tOp: resmgr.AlwaysTrue, // evaluate against all containers\n\t\t},\n\t\tMatch: &resmgr.Expression{\n\t\t\tKey: key,\n\t\t\tOp:  resmgr.Exists,\n\t\t},\n\t\tWeight: weight,\n\t}\n}\n\n// GlobalAntiAffinity creates an anti-affinity with all containers in scope.\nfunc GlobalAntiAffinity(key string, weight int32) *Affinity {\n\treturn GlobalAffinity(key, -weight)\n}\n\n// AddImplicitAffinities registers a set of implicit affinities.\nfunc (cch *cache) AddImplicitAffinities(implicit map[string]ImplicitAffinity) error {\n\tfor name := range implicit {\n\t\tif _, ok := cch.implicit[name]; ok {\n\t\t\treturn cacheError(\"implicit affinity %s already defined\", name)\n\t\t}\n\t}\n\tfor name, a := range implicit {\n\t\tcch.implicit[name] = a\n\t}\n\treturn nil\n}\n\n// DeleteImplicitAffinities removes a previously registered set of implicit affinities.\nfunc (cch *cache) DeleteImplicitAffinities(names []string) {\n\tfor _, name := range names {\n\t\tdelete(cch.implicit, name)\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/affinity_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"testing\"\n)\n\nfunc TestSimpleParsingSymmetry(t *testing.T) {\n\tc1, c2, c3, c4, c5 := \"c1\", \"c2\", \"c3\", \"c4\", \"c5\"\n\n\ttcases := []struct {\n\t\tname   string\n\t\tsource string\n\t\tresult map[string][]string\n\t}{\n\t\t{\n\t\t\tname:   \"trivial 2 by 2\",\n\t\t\tsource: `c1: [ c2 ]`,\n\t\t\tresult: map[string][]string{\n\t\t\t\tc1: {c2},\n\t\t\t\tc2: {c1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:   \"simple\",\n\t\t\tsource: `c1: [ c2, c3, c4, c5 ]`,\n\t\t\tresult: map[string][]string{\n\t\t\t\tc1: {c2, c3, c4, c5},\n\t\t\t\tc2: {c1},\n\t\t\t\tc3: {c1},\n\t\t\t\tc4: {c1},\n\t\t\t\tc5: {c1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"a bit more complex\",\n\t\t\tsource: `\nc1: [ c2 ]\nc2: [ c3, c4, c5 ]\nc4: [ c5 ]\n`,\n\t\t\tresult: map[string][]string{\n\t\t\t\tc1: {c2},\n\t\t\t\tc2: {c1, c3, c4, c5},\n\t\t\t\tc3: {c2},\n\t\t\t\tc4: {c2, c5},\n\t\t\t\tc5: {c2, c4},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tpca := podContainerAffinity{}\n\t\t\tif !pca.parseSimple(&pod{Name: \"testpod\"}, tc.source, 1) {\n\t\t\t\tt.Errorf(\"failed to parse simple container affinity %q\", tc.source)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfound := map[string]map[string]struct{}{}\n\t\t\tfor name, affinities := range pca {\n\t\t\t\tfor _, a := range affinities {\n\t\t\t\t\tfor _, o := range a.Match.Values {\n\t\t\t\t\t\tforw, ok := found[name]\n\t\t\t\t\t\tif !ok {\n\t\t\t\t\t\t\tforw = map[string]struct{}{}\n\t\t\t\t\t\t\tfound[name] = forw\n\t\t\t\t\t\t}\n\t\t\t\t\t\tback, ok := found[o]\n\t\t\t\t\t\tif !ok {\n\t\t\t\t\t\t\tback = map[string]struct{}{}\n\t\t\t\t\t\t\tfound[o] = back\n\t\t\t\t\t\t}\n\t\t\t\t\t\tforw[o] = struct{}{}\n\t\t\t\t\t\tback[name] = struct{}{}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor name, others := range tc.result {\n\t\t\t\tfor _, o := range others {\n\t\t\t\t\tif _, ok := found[name][o]; !ok {\n\t\t\t\t\t\tt.Errorf(\"simple affinity %q did not produce %s: %s\",\n\t\t\t\t\t\t\ttc.source, name, o)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tdelete(found[name], o)\n\t\t\t\t\t\tif len(found[name]) == 0 {\n\t\t\t\t\t\t\tdelete(found, name)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor name, others := range found {\n\t\t\t\tval := \"\"\n\t\t\t\tsep := \"\"\n\t\t\t\tfor o := range others {\n\t\t\t\t\tval += sep + o\n\t\t\t\t\tsep = \", \"\n\t\t\t\t}\n\t\t\t\tt.Errorf(\"simple affinity %q produced unexpected %s: [ %s ]\", tc.source, name, val)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestStrictParsing(t *testing.T) {\n\ttcases := []struct {\n\t\tname    string\n\t\tsource  string\n\t\tinvalid bool\n\t}{\n\t\t{\n\t\t\tname: \"invalid annotation\",\n\t\t\tsource: `\n  memtier-benchmark:\n    - scope:\n      key: pod/name\n      operator: Matches\n      values:\n        - redis-*\n      match:\n        key: name\n        operator: Equals\n        values:\n          - redis\n      weight: 10\n`,\n\t\t\tinvalid: true,\n\t\t},\n\t\t{\n\t\t\tname: \"valid annotation\",\n\t\t\tsource: `\n  memtier-benchmark:\n    - scope:\n        key: pod/name\n        operator: Matches\n        values:\n          - redis-*\n      match:\n        key: name\n        operator: Equals\n        values:\n          - redis\n      weight: 10\n`,\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tpca := podContainerAffinity{}\n\t\t\terr := pca.parseFull(&pod{Name: \"testpod\"}, tc.source, 1)\n\t\t\tif tc.invalid && err == nil {\n\t\t\t\tt.Errorf(\"parsing invalid affinity expression should have failed\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif !tc.invalid && err != nil {\n\t\t\t\tt.Errorf(\"parsing valid affinity expression should not fail\")\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/cache.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// CPU marks changes that can be applied by the CPU controller.\n\tCPU = \"cpu\"\n\t// CRI marks changes that can be applied by the CRI controller.\n\tCRI = \"cri\"\n\t// RDT marks changes that can be applied by the RDT controller.\n\tRDT = \"rdt\"\n\t// BlockIO marks changes that can be applied by the BlockIO controller.\n\tBlockIO = \"blockio\"\n\t// Memory marks changes that can be applied by the Memory controller.\n\tMemory = \"memory\"\n\t// PageMigration marks changes that can be applied by the PageMigration controller.\n\tPageMigration = \"page-migration\"\n\n\t// TagAVX512 tags containers that use AVX512 instructions.\n\tTagAVX512 = \"AVX512\"\n\n\t// RDTClassKey is the pod annotation key for specifying a container RDT class.\n\tRDTClassKey = \"rdtclass\" + \".\" + kubernetes.ResmgrKeyNamespace\n\t// BlockIOClassKey is the pod annotation key for specifying a container Block I/O class.\n\tBlockIOClassKey = \"blockioclass\" + \".\" + kubernetes.ResmgrKeyNamespace\n\t// ToptierLimitKey is the pod annotation key for specifying container top tier memory limits.\n\tToptierLimitKey = \"toptierlimit\" + \".\" + kubernetes.ResmgrKeyNamespace\n\n\t// RDTClassPodQoS denotes that the RDTClass should be taken from PodQosClass\n\tRDTClassPodQoS = \"/PodQos\"\n\n\t// ToptierLimitUnset is the reserved value for indicating unset top tier limits.\n\tToptierLimitUnset int64 = -1\n\n\t// TopologyHintsKey can be used to opt out from automatic topology hint generation.\n\tTopologyHintsKey = \"topologyhints\" + \".\" + kubernetes.ResmgrKeyNamespace\n)\n\n// allControllers is a slice of all controller domains.\nvar allControllers = []string{CPU, CRI, RDT, BlockIO, Memory}\n\n// PodState is the pod state in the runtime.\ntype PodState int32\n\nconst (\n\t// PodStateReady marks a pod ready.\n\tPodStateReady = PodState(int32(criv1.PodSandboxState_SANDBOX_READY))\n\t// PodStateNotReady marks a pod as not ready.\n\tPodStateNotReady = PodState(int32(criv1.PodSandboxState_SANDBOX_NOTREADY))\n\t// PodStateStale marks a pod as removed.\n\tPodStateStale = PodState(int32(PodStateNotReady) + 1)\n)\n\n// PodResourceRequirements are per container resource requirements, annotated by our webhook.\ntype PodResourceRequirements struct {\n\t// InitContainers is the resource requirements by init containers.\n\tInitContainers map[string]v1.ResourceRequirements `json:\"initContainers\"`\n\t// Containers is the resource requirements by normal container.\n\tContainers map[string]v1.ResourceRequirements `json:\"containers\"`\n}\n\n// PodStatus wraps a PodSandboxStatus response for data extraction.\ntype PodStatus struct {\n\tCgroupParent string // extracted CgroupParent\n}\n\n// Pod is the exposed interface from a cached pod.\ntype Pod interface {\n\tresmgr.Evaluable\n\tfmt.Stringer\n\t// GetInitContainers returns the init containers of the pod.\n\tGetInitContainers() []Container\n\t// GetContainers returns the (non-init) containers of the pod.\n\tGetContainers() []Container\n\t// GetContainer returns the named container of the pod.\n\tGetContainer(string) (Container, bool)\n\t// GetId returns the pod id of the pod.\n\tGetID() string\n\t// GetUID returns the (kubernetes) unique id of the pod.\n\tGetUID() string\n\t// GetName returns the name of the pod.\n\tGetName() string\n\t// GetNamespace returns the namespace of the pod.\n\tGetNamespace() string\n\t// GetState returns the PodState of the pod.\n\tGetState() PodState\n\t// GetQOSClass returns the PodQOSClass of the pod.\n\tGetQOSClass() v1.PodQOSClass\n\t// GetLabelKeys returns the keys of all pod labels as a string slice.\n\tGetLabelKeys() []string\n\t// GetLabel returns the value of the given label and whether it was found.\n\tGetLabel(string) (string, bool)\n\t// GetResmgrLabelKeys returns pod label keys (without the namespace\n\t// part) in cri-resource-manager namespace.\n\tGetResmgrLabelKeys() []string\n\t// GetResmgrLabel returns the value of a pod label from the\n\t// cri-resource-manager namespace.\n\tGetResmgrLabel(string) (string, bool)\n\t// GetAnnotationKeys returns the keys of all pod annotations as a string slice.\n\tGetAnnotationKeys() []string\n\t// GetAnnotation returns the value of the given annotation and whether it was found.\n\tGetAnnotation(key string) (string, bool)\n\t// GetAnnotationObject decodes the value of the given annotation with the given function.\n\tGetAnnotationObject(key string, objPtr interface{},\n\t\tdecode func([]byte, interface{}) error) (bool, error)\n\t// GetResmgrAnnotationKeys returns pod annotation keys (without the\n\t// namespace part) in cri-resource-manager namespace as a string slice.\n\tGetResmgrAnnotationKeys() []string\n\t// GetAnnotation returns the value of a pod annotation from the\n\t// cri-resource-manager namespace and whether it was found.\n\tGetResmgrAnnotation(key string) (string, bool)\n\t// GetResmgrAnnotationObject decodes the value of the given annotation in the\n\t// cri-resource-manager namespace.\n\tGetResmgrAnnotationObject(key string, objPtr interface{},\n\t\tdecode func([]byte, interface{}) error) (bool, error)\n\t// GetEffectiveAnnotation returns the effective annotation for a container.\n\t// For any given key $K and container $C it will look for annotations in\n\t// this order and return the first one found:\n\t//     $K/container.$C\n\t//     $K/pod\n\t//     $K\n\t// and return the value of the first key found.\n\tGetEffectiveAnnotation(key, container string) (string, bool)\n\t// GetCgroupParentDir returns the pods cgroup parent directory.\n\tGetCgroupParentDir() string\n\t// GetPodResourceRequirements returns container resource requirements if the\n\t// necessary associated annotation put in place by the CRI resource manager\n\t// webhook was found.\n\tGetPodResourceRequirements() PodResourceRequirements\n\t// GetContainerAffinity returns the affinity expressions for the named container.\n\tGetContainerAffinity(string) ([]*Affinity, error)\n\t// ScopeExpression returns an affinity expression for defining this pod as the scope.\n\tScopeExpression() *resmgr.Expression\n\n\t// GetProcesses returns the pids of all processes in the pod either excluding\n\t// container processes, if called with false, or including those if called with true.\n\tGetProcesses(bool) ([]string, error)\n\t// GetTasks returns the pids of all threads in the pod either excluding cotnainer\n\t// processes, if called with false, or including those if called with true.\n\tGetTasks(bool) ([]string, error)\n}\n\n// A cached pod.\ntype pod struct {\n\tcache        *cache            // our cache of object\n\tID           string            // pod sandbox runtime id\n\tUID          string            // (k8s) unique id\n\tName         string            // pod sandbox name\n\tNamespace    string            // pod namespace\n\tState        PodState          // ready/not ready\n\tQOSClass     v1.PodQOSClass    // pod QoS class\n\tLabels       map[string]string // pod labels\n\tAnnotations  map[string]string // pod annotations\n\tCgroupParent string            // cgroup parent directory\n\tcontainers   map[string]string // container name to ID map\n\n\tResources *PodResourceRequirements // annotated resource requirements\n\tAffinity  *podContainerAffinity    // annotated container affinity\n}\n\n// ContainerState is the container state in the runtime.\ntype ContainerState int32\n\nconst (\n\t// ContainerStateCreated marks a container created, not running.\n\tContainerStateCreated = ContainerState(int32(criv1.ContainerState_CONTAINER_CREATED))\n\t// ContainerStateRunning marks a container created, running.\n\tContainerStateRunning = ContainerState(int32(criv1.ContainerState_CONTAINER_RUNNING))\n\t// ContainerStateExited marks a container exited.\n\tContainerStateExited = ContainerState(int32(criv1.ContainerState_CONTAINER_EXITED))\n\t// ContainerStateUnknown marks a container to be in an unknown state.\n\tContainerStateUnknown = ContainerState(int32(criv1.ContainerState_CONTAINER_UNKNOWN))\n\t// ContainerStateCreating marks a container as being created.\n\tContainerStateCreating = ContainerState(int32(ContainerStateUnknown) + 1)\n\t// ContainerStateStale marks a container removed.\n\tContainerStateStale = ContainerState(int32(ContainerStateUnknown) + 2)\n)\n\n// Container is the exposed interface from a cached container.\ntype Container interface {\n\tresmgr.Evaluable\n\tfmt.Stringer\n\t// PrettyName returns the user-friendly <podname>:<containername> for the container.\n\tPrettyName() string\n\t// GetPod returns the pod of the container and a boolean indicating if there was one.\n\tGetPod() (Pod, bool)\n\t// GetID returns the ID of the container.\n\tGetID() string\n\t// GetPodID returns the pod ID of the container.\n\tGetPodID() string\n\t// GetCacheID returns the cacheID of the container.\n\tGetCacheID() string\n\t// GetName returns the name of the container.\n\tGetName() string\n\t// GetNamespace returns the namespace of the container.\n\tGetNamespace() string\n\t// UpdateState updates the state of the container.\n\tUpdateState(ContainerState)\n\t// GetState returns the ContainerState of the container.\n\tGetState() ContainerState\n\t// GetQOSClass returns the QoS class the pod would have if this was its only container.\n\tGetQOSClass() v1.PodQOSClass\n\t// GetImage returns the image of the container.\n\tGetImage() string\n\t// GetCommand returns the container command.\n\tGetCommand() []string\n\t// GetArgs returns the container command arguments.\n\tGetArgs() []string\n\t// GetLabelKeys returns the keys of all labels of the container.\n\tGetLabelKeys() []string\n\t// GetLabel returns the value of a container label.\n\tGetLabel(string) (string, bool)\n\t// GetLabels returns a copy of all container labels.\n\tGetLabels() map[string]string\n\t// GetResmgrLabelKeys returns container label keys (without the namespace\n\t// part) in cri-resource-manager namespace.\n\tGetResmgrLabelKeys() []string\n\t// GetResmgrLabel returns the value of a container label from the\n\t// cri-resource-manager namespace.\n\tGetResmgrLabel(string) (string, bool)\n\t// GetAnnotationKeys returns the keys of all annotations of the container.\n\tGetAnnotationKeys() []string\n\t// GetAnnotation returns the value of a container annotation.\n\tGetAnnotation(key string, objPtr interface{}) (string, bool)\n\t// GetResmgrAnnotationKeys returns container annotation keys (without the\n\t// namespace part) in cri-resource-manager namespace.\n\tGetResmgrAnnotationKeys() []string\n\t// GetAnnotation returns the value of a container annotation from the\n\t// cri-resource-manager namespace.\n\tGetResmgrAnnotation(key string, objPtr interface{}) (string, bool)\n\t// GetEffectiveAnnotation returns the effective annotation for the container from the pod.\n\tGetEffectiveAnnotation(key string) (string, bool)\n\t// GetAnnotations returns a copy of all container annotations.\n\tGetAnnotations() map[string]string\n\t// GetEnvKeys returns the keys of all container environment variables.\n\tGetEnvKeys() []string\n\t// GetEnv returns the value of a container environment variable.\n\tGetEnv(string) (string, bool)\n\t// GetMounts returns all the mounts of the container.\n\tGetMounts() []Mount\n\t// GetMountByHost returns the container path corresponding to the host path.\n\t// XXX We should remove this as is might not be unique.\n\tGetMountByHost(string) *Mount\n\t// GetmountByContainer returns the host path mounted to a container path.\n\tGetMountByContainer(string) *Mount\n\t// GetDevices returns the devices of the container.\n\tGetDevices() []Device\n\t// GetDeviceByHost returns the device for a host path.\n\tGetDeviceByHost(string) *Device\n\t// GetDeviceByContainer returns the device for a container path.\n\tGetDeviceByContainer(string) *Device\n\t// GetResourceRequirements returns the webhook-annotated requirements for ths container.\n\tGetResourceRequirements() v1.ResourceRequirements\n\t// GetLinuxResources returns the CRI linux resource request of the container.\n\tGetLinuxResources() *criv1.LinuxContainerResources\n\n\t// SetCommand sets the container command.\n\tSetCommand([]string)\n\t// SetArgs sets the container command arguments.\n\tSetArgs([]string)\n\t// SetLabel sets the value for a container label.\n\tSetLabel(string, string)\n\t// DeleteLabel removes a container label.\n\tDeleteLabel(string)\n\t// SetAnnotation sets the value for a container annotation.\n\tSetAnnotation(string, string)\n\t// DeleteAnnotation removes a container annotation.\n\tDeleteAnnotation(string)\n\t// SetEnv sets a container environment variable.\n\tSetEnv(string, string)\n\t// UnsetEnv unsets a container environment variable.\n\tUnsetEnv(string)\n\t// InsertMount inserts a mount into the container.\n\tInsertMount(*Mount)\n\t// DeleteMount removes a mount from the container.\n\tDeleteMount(string)\n\t// InsertDevice inserts a device into the container.\n\tInsertDevice(*Device)\n\t// DeleteDevice removes a device from the container.\n\tDeleteDevice(string)\n\n\t// Get any attached topology hints.\n\tGetTopologyHints() topology.Hints\n\n\t// GetCPUPeriod gets the CFS CPU period of the container.\n\tGetCPUPeriod() int64\n\t// GetCpuQuota gets the CFS CPU quota of the container.\n\tGetCPUQuota() int64\n\t// GetCPUShares gets the CFS CPU shares of the container.\n\tGetCPUShares() int64\n\t// GetmemoryLimit gets the memory limit in bytes for the container.\n\tGetMemoryLimit() int64\n\t// GetOomScoreAdj gets the OOM score adjustment for the container.\n\tGetOomScoreAdj() int64\n\t// GetCpusetCPUs gets the cgroup cpuset.cpus of the container.\n\tGetCpusetCpus() string\n\t// GetCpusetMems gets the cgroup cpuset.mems of the container.\n\tGetCpusetMems() string\n\n\t// SetLinuxResources sets the Linux-specific resource request of the container.\n\tSetLinuxResources(*criv1.LinuxContainerResources)\n\t// SetCPUPeriod sets the CFS CPU period of the container.\n\tSetCPUPeriod(int64)\n\t// SetCPUQuota sets the CFS CPU quota of the container.\n\tSetCPUQuota(int64)\n\t// SetCPUShares sets the CFS CPU shares of the container.\n\tSetCPUShares(int64)\n\t// SetmemoryLimit sets the memory limit in bytes for the container.\n\tSetMemoryLimit(int64)\n\t// SetOomScoreAdj sets the OOM score adjustment for the container.\n\tSetOomScoreAdj(int64)\n\t// SetCpusetCpu sets the cgroup cpuset.cpus of the container.\n\tSetCpusetCpus(string)\n\t// SetCpusetMems sets the cgroup cpuset.mems of the container.\n\tSetCpusetMems(string)\n\n\t// GetAffinity returns the annotated affinity expressions for this container.\n\tGetAffinity() ([]*Affinity, error)\n\n\t// GetCgroupDir returns the relative path of the cgroup directory for the container.\n\tGetCgroupDir() string\n\n\t// SetRDTClass assigns this container to the given RDT class.\n\tSetRDTClass(string)\n\t// GetRDTClass returns the RDT class for this container.\n\tGetRDTClass() string\n\n\t// SetBlockIOClass assigns this container to the given BlockIO class.\n\tSetBlockIOClass(string)\n\t// GetBlockIOClass returns the BlockIO class for this container.\n\tGetBlockIOClass() string\n\n\t// SetToptierLimit sets the tier memory limit for the container.\n\tSetToptierLimit(int64)\n\t// GetToptierLimit returns the top tier memory limit for the container.\n\tGetToptierLimit() int64\n\n\t// SetPageMigration sets the page migration policy/options for the container.\n\tSetPageMigration(*PageMigrate)\n\t// GetPageMigration returns the current page migration policy/options for the container.\n\tGetPageMigration() *PageMigrate\n\n\t// GetProcesses returns the pids of processes in the container.\n\tGetProcesses() ([]string, error)\n\t// GetTasks returns the pids of threads in the container.\n\tGetTasks() ([]string, error)\n\n\t// SetCRIRequest sets the current pending CRI request of the container.\n\tSetCRIRequest(req interface{}) error\n\t// GetCRIRequest returns the current pending CRI request of the container.\n\tGetCRIRequest() (interface{}, bool)\n\t// ClearCRIRequest clears and returns the current pending CRI request of the container.\n\tClearCRIRequest() (interface{}, bool)\n\n\t// GetCRIEnvs returns container environment variables.\n\tGetCRIEnvs() []*criv1.KeyValue\n\t// GetCRIMounts returns container mounts.\n\tGetCRIMounts() []*criv1.Mount\n\t// GetCRIDevices returns container devices.\n\tGetCRIDevices() []*criv1.Device\n\n\t// GetPending gets the names of the controllers with pending changes.\n\tGetPending() []string\n\t// HasPending checks if the container has pending chanhes for the given controller.\n\tHasPending(string) bool\n\t// ClearPending clears the pending change marker for the given controller.\n\tClearPending(string)\n\n\t// GetTag gets the value of the given tag.\n\tGetTag(string) (string, bool)\n\t// SetTag sets the value of the given tag and returns its previous value..\n\tSetTag(string, string) (string, bool)\n\t// DeleteTag deletes the given tag, returning its deleted value.\n\tDeleteTag(string) (string, bool)\n}\n\n// A cached container.\ntype container struct {\n\tcache         *cache             // our cache of objects\n\tID            string             // container runtime id\n\tPodID         string             // associate pods runtime id\n\tCacheID       string             // our cache id\n\tName          string             // container name\n\tNamespace     string             // container namespace\n\tState         ContainerState     // created/running/exited/unknown\n\tImage         string             // containers image\n\tCommand       []string           // command to run in container\n\tArgs          []string           // arguments for command\n\tLabels        map[string]string  // container labels\n\tAnnotations   map[string]string  // container annotations\n\tEnv           map[string]string  // environment variables\n\tMounts        map[string]*Mount  // mounts\n\tDevices       map[string]*Device // devices\n\tTopologyHints topology.Hints     // Set of topology hints for all containers within Pod\n\tTags          map[string]string  // container tags (local dynamic labels)\n\tAdjustment    string             // name of applicable external adjustment, if any\n\n\tResources v1.ResourceRequirements        // container resources (from webhook annotation)\n\tLinuxReq  *criv1.LinuxContainerResources // used to estimate Resources if we lack annotations\n\treq       *interface{}                   // pending CRI request\n\n\tCgroupDir    string       // cgroup directory relative to a(ny) controller.\n\tRDTClass     string       // RDT class this container is assigned to.\n\tBlockIOClass string       // Block I/O class this container is assigned to.\n\tToptierLimit int64        // Top tier memory limit.\n\tPageMigrate  *PageMigrate // Page migration policy/options for this container.\n\n\tpending map[string]struct{} // controllers with pending changes for this container\n\n\tprettyName string // cached PrettyName()\n}\n\n// MountType is a propagation type.\ntype MountType int32\n\nconst (\n\t// MountPrivate is a private container mount.\n\tMountPrivate MountType = MountType(criv1.MountPropagation_PROPAGATION_PRIVATE)\n\t// MountHostToContainer is a host-to-container mount.\n\tMountHostToContainer MountType = MountType(criv1.MountPropagation_PROPAGATION_HOST_TO_CONTAINER)\n\t// MountBidirectional is a bidirectional mount.\n\tMountBidirectional MountType = MountType(criv1.MountPropagation_PROPAGATION_BIDIRECTIONAL)\n)\n\n// Mount is a filesystem entry mounted inside a container.\ntype Mount struct {\n\t// Container is the path inside the container.\n\tContainer string\n\t// Host is the path on the host.\n\tHost string\n\t// Readonly specifies if the mount is read-only or read-write.\n\tReadonly bool\n\t// Relabels denotes SELinux relabeling.\n\tRelabel bool\n\t// Propagation identifies the mount propagation type.\n\tPropagation MountType\n}\n\n// Device is a device exposed to a container.\ntype Device struct {\n\t// Container is the device path inside the container.\n\tContainer string\n\t// Host is the device path on the host side.\n\tHost string\n\t// Permissions specify the device permissions for the container.\n\tPermissions string\n}\n\n// PageMigrate contains the policy/preferences for container page migration.\ntype PageMigrate struct {\n\tSourceNodes idset.IDSet // idle memory pages on these NUMA nodes\n\tTargetNodes idset.IDSet // should be migrated to these NUMA nodes\n}\n\n// Clone creates a copy of the page migration policy/preferences.\nfunc (pm *PageMigrate) Clone() *PageMigrate {\n\tif pm == nil {\n\t\treturn nil\n\t}\n\tc := &PageMigrate{}\n\tif pm.SourceNodes != nil {\n\t\tc.SourceNodes = pm.SourceNodes.Clone()\n\t}\n\tif pm.TargetNodes != nil {\n\t\tc.TargetNodes = pm.TargetNodes.Clone()\n\t}\n\treturn c\n}\n\n// Cachable is an interface opaque cachable data must implement.\ntype Cachable interface {\n\t// Set value (via a pointer receiver) to the object.\n\tSet(value interface{})\n\t// Get the object that should be cached.\n\tGet() interface{}\n}\n\n// Cache is the primary interface exposed for tracking pods and containers.\n//\n// Cache tracks pods and containers in the runtime, mostly by processing CRI\n// requests and responses which the cache is fed as these are being procesed.\n// Cache also saves its state upon changes to secondary storage and restores\n// itself upon startup.\ntype Cache interface {\n\t// InsertPod inserts a pod into the cache, using a runtime request or reply.\n\tInsertPod(id string, msg interface{}, status *PodStatus) (Pod, error)\n\t// DeletePod deletes a pod from the cache.\n\tDeletePod(id string) Pod\n\t// LookupPod looks up a pod in the cache.\n\tLookupPod(id string) (Pod, bool)\n\t// InsertContainer inserts a container into the cache, using a runtime request or reply.\n\tInsertContainer(msg interface{}) (Container, error)\n\t// UpdateContainerID updates a containers runtime id.\n\tUpdateContainerID(cacheID string, msg interface{}) (Container, error)\n\t// DeleteContainer deletes a container from the cache.\n\tDeleteContainer(id string) Container\n\t// LookupContainer looks up a container in the cache.\n\tLookupContainer(id string) (Container, bool)\n\t// LookupContainerByCgroup looks up a container for the given cgroup path.\n\tLookupContainerByCgroup(path string) (Container, bool)\n\n\t// GetPendingContainers returs all containers with pending changes.\n\tGetPendingContainers() []Container\n\n\t// GetPods returns all the pods known to the cache.\n\tGetPods() []Pod\n\t// GetContainers returns all the containers known to the cache.\n\tGetContainers() []Container\n\n\t// GetContainerCacheIds returns the cache ids of all containers.\n\tGetContainerCacheIds() []string\n\t// GetContaineIds return the ids of all containers.\n\tGetContainerIds() []string\n\n\t// FilterScope returns the containers selected by the scope expression.\n\tFilterScope(*resmgr.Expression) []Container\n\t// EvaluateAffinity evaluates the given affinity against all known in-scope containers\n\tEvaluateAffinity(*Affinity) map[string]int32\n\t// AddImplicitAffinities adds a set of implicit affinities (added to all containers).\n\tAddImplicitAffinities(map[string]ImplicitAffinity) error\n\n\t// GetActivePolicy returns the name of the active policy stored in the cache.\n\tGetActivePolicy() string\n\t// SetActivePolicy updates the name of the active policy stored in the cache.\n\tSetActivePolicy(string) error\n\n\t// ResetActivePolicy clears the active policy any any policy-specific data from the cache.\n\tResetActivePolicy() error\n\n\t// SetPolicyEntry sets the policy entry for a key.\n\tSetPolicyEntry(string, interface{})\n\t// GetPolicyEntry gets the policy entry for a key.\n\tGetPolicyEntry(string, interface{}) bool\n\n\t// SetConfig caches the given configuration.\n\tSetConfig(*config.RawConfig) error\n\t// GetConfig returns the current/cached configuration.\n\tGetConfig() *config.RawConfig\n\t// ResetConfig clears any stored configuration from the cache.\n\tResetConfig() error\n\n\t// SetAdjustment updates external adjustments and containers based this.\n\tSetAdjustment(*config.Adjustment) (bool, map[string]error)\n\n\t// Save requests a cache save.\n\tSave() error\n\n\t// RefreshPods purges/inserts stale/new pods/containers using a pod sandbox list response.\n\tRefreshPods(*criv1.ListPodSandboxResponse, map[string]*PodStatus) ([]Pod, []Pod, []Container)\n\t// RefreshContainers purges/inserts stale/new containers using a container list response.\n\tRefreshContainers(*criv1.ListContainersResponse) ([]Container, []Container)\n\n\t// Get the container (data) directory for a container.\n\tContainerDirectory(string) string\n\t// OpenFile opens the names container data file, creating it if necessary.\n\tOpenFile(string, string, os.FileMode) (*os.File, error)\n\t// WriteFile writes a container data file, creating it if necessary.\n\tWriteFile(string, string, os.FileMode, []byte) error\n}\n\nconst (\n\t// CacheVersion is the running version of the cache.\n\tCacheVersion = \"1\"\n)\n\n// permissions describe preferred/expected ownership and permissions for a file or directory.\ntype permissions struct {\n\tprefer os.FileMode // permissions to create file/directory with\n\treject os.FileMode // bits that cause rejection to use an existing entry\n}\n\n// permissions to create with/check against\nvar (\n\tcacheDirPerm  = &permissions{prefer: 0710, reject: 0022}\n\tcacheFilePerm = &permissions{prefer: 0644, reject: 0022}\n\tdataDirPerm   = &permissions{prefer: 0755, reject: 0022}\n\tdataFilePerm  = &permissions{prefer: 0644, reject: 0022}\n)\n\n// Our cache of objects.\ntype cache struct {\n\tsync.Mutex    `json:\"-\"` // we're lockable\n\tlogger.Logger `json:\"-\"` // cache logger instance\n\tfilePath      string     // where to store to/load from\n\tdataDir       string     // container data directory\n\n\tPods       map[string]*pod       // known/cached pods\n\tContainers map[string]*container // known/cache containers\n\tNextID     uint64                // next container cache id to use\n\n\tCfg        *config.RawConfig      // cached/current configuration\n\tExternal   *config.Adjustment     // cached/current external adjustments\n\tPolicyName string                 // name of the active policy\n\tpolicyData map[string]interface{} // opaque policy data\n\tPolicyJSON map[string]string      // ditto in raw, unmarshaled form\n\n\tpending map[string]struct{} // cache IDs of containers with pending changes\n\n\timplicit map[string]ImplicitAffinity // implicit affinities\n}\n\n// Make sure cache implements Cache.\nvar _ Cache = &cache{}\n\n// Options contains the configurable cache options.\ntype Options struct {\n\t// CacheDir is the directory the cache should save its state in.\n\tCacheDir string\n}\n\n// NewCache instantiates a new cache. Load it from the given path if it exists.\nfunc NewCache(options Options) (Cache, error) {\n\tcch := &cache{\n\t\tfilePath:   filepath.Join(options.CacheDir, \"cache\"),\n\t\tdataDir:    filepath.Join(options.CacheDir, \"containers\"),\n\t\tLogger:     logger.NewLogger(\"cache\"),\n\t\tPods:       make(map[string]*pod),\n\t\tContainers: make(map[string]*container),\n\t\tNextID:     1,\n\t\tpolicyData: make(map[string]interface{}),\n\t\tPolicyJSON: make(map[string]string),\n\t\timplicit:   make(map[string]ImplicitAffinity),\n\t}\n\n\tif _, err := cch.checkPerm(\"cache\", cch.filePath, false, cacheFilePerm); err != nil {\n\t\treturn nil, cacheError(\"refusing to use existing cache file: %v\", err)\n\t}\n\tif err := cch.mkdirAll(\"cache\", options.CacheDir, cacheDirPerm); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := cch.mkdirAll(\"container\", cch.dataDir, dataDirPerm); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := cch.Load(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn cch, nil\n}\n\n// GetActivePolicy returns the name of the active policy stored in the cache.\nfunc (cch *cache) GetActivePolicy() string {\n\treturn cch.PolicyName\n}\n\n// SetActivePolicy updaes the name of the active policy stored in the cache.\nfunc (cch *cache) SetActivePolicy(policy string) error {\n\tcch.PolicyName = policy\n\treturn cch.Save()\n}\n\n// ResetActivePolicy clears the active policy any any policy-specific data from the cache.\nfunc (cch *cache) ResetActivePolicy() error {\n\tcch.Warn(\"clearing all data for active policy (%q) from cache...\",\n\t\tcch.PolicyName)\n\n\tcch.PolicyName = \"\"\n\tcch.policyData = make(map[string]interface{})\n\tcch.PolicyJSON = make(map[string]string)\n\n\treturn cch.Save()\n}\n\n// SetConfig caches the given configuration.\nfunc (cch *cache) SetConfig(cfg *config.RawConfig) error {\n\told := cch.Cfg\n\tcch.Cfg = cfg\n\n\tif err := cch.Save(); err != nil {\n\t\tcch.Cfg = old\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// GetConfig returns the current/cached configuration.\nfunc (cch *cache) GetConfig() *config.RawConfig {\n\treturn cch.Cfg\n}\n\n// ResetConfig clears any stored configuration from the cache.\nfunc (cch *cache) ResetConfig() error {\n\told := cch.Cfg\n\tcch.Cfg = nil\n\n\tif err := cch.Save(); err != nil {\n\t\tcch.Cfg = old\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// SetAdjustment updates external adjustments and containers based on this.\nfunc (cch *cache) SetAdjustment(external *config.Adjustment) (bool, map[string]error) {\n\teffective := map[*container]string{}\n\n\t// collect per container external adjustments, checking for obvious errors\n\terrors := map[string]error{}\n\tfor id, c := range cch.Containers {\n\t\tif id != c.GetCacheID() {\n\t\t\tcontinue\n\t\t}\n\n\t\tadjustments := cch.getApplicableAdjustments(external, c)\n\n\t\tif len(adjustments) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\t// conflict: multiple adjustments per container\n\t\tif len(adjustments) > 1 {\n\t\t\terrors[c.GetID()] = cacheError(\"conflicting adjustments for %s: %s\",\n\t\t\t\tc.PrettyName(), strings.Join(adjustments, \",\"))\n\t\t\tcontinue\n\t\t}\n\n\t\tadjust := external.Adjustments[adjustments[0]]\n\n\t\t// error: trying to override resources for BestEffort container\n\t\tif c.GetQOSClass() == v1.PodQOSBestEffort {\n\t\t\tif adjust.Resources != nil {\n\t\t\t\terrors[c.GetID()] = cacheError(\"%s: can't override resources for BestEffort %s\",\n\t\t\t\t\tadjustments[0], c.PrettyName())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\teffective[c] = adjustments[0]\n\t}\n\tif len(errors) > 0 {\n\t\treturn false, errors\n\t}\n\n\t// update per container external adjustments, mark all containers with pending changes\n\tfor id, c := range cch.Containers {\n\t\tif id != c.GetCacheID() {\n\t\t\tcontinue\n\t\t}\n\n\t\tuptodate := effective[c]\n\t\tprevious := c.setEffectiveAdjustment(uptodate)\n\t\teffective[c] = previous\n\n\t\tif previous != uptodate {\n\t\t\tcch.Info(\"%s effective external adjustment changed from %q to %q\",\n\t\t\t\tc.PrettyName(), previous, uptodate)\n\t\t}\n\n\t\tc.markPending(allControllers...)\n\t}\n\n\tif err := cch.Save(); err != nil {\n\t\tfor id, c := range cch.Containers {\n\t\t\tif id != c.GetCacheID() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.setEffectiveAdjustment(effective[c])\n\t\t}\n\t\treturn false, map[string]error{\"cache\": err}\n\t}\n\n\tcch.External = external\n\treturn true, nil\n}\n\n// Get all external adjustments applicable to the given container.\nfunc (cch *cache) getApplicableAdjustments(ext *config.Adjustment, c *container) []string {\n\tif ext == nil {\n\t\treturn []string{}\n\t}\n\tapplicable := []string{}\n\tfor name, adjust := range ext.Adjustments {\n\t\tif adjust.IsContainerInScope(c) {\n\t\t\tapplicable = append(applicable, name)\n\t\t}\n\t}\n\treturn applicable\n}\n\n// setEffectiveAdjustment updates the effective adjustments of all containers.\nfunc (cch *cache) setEffectiveAdjustment(effective map[*container]string) {\n\tfor id, c := range cch.Containers {\n\t\tif id != c.GetCacheID() {\n\t\t\tcontinue\n\t\t}\n\n\t\tuptodate := effective[c]\n\t\tprevious := c.setEffectiveAdjustment(uptodate)\n\n\t\tif previous != uptodate {\n\t\t\tcch.Info(\"%s effective external adjustment changed from %q to %q\",\n\t\t\t\tc.PrettyName(), previous, uptodate)\n\t\t}\n\n\t\t// we forcibly mark the container as updated in all controller domains\n\t\tfor _, ctrl := range allControllers {\n\t\t\tc.markPending(ctrl)\n\t\t}\n\t}\n}\n\n// Derive cache id using pod uid, or allocate a new unused local cache id.\nfunc (cch *cache) createCacheID(c *container) string {\n\tif pod, ok := c.cache.LookupPod(c.PodID); ok {\n\t\tuid := pod.GetUID()\n\t\tif uid != \"\" {\n\t\t\treturn uid + \":\" + c.Name\n\t\t}\n\t}\n\n\tcch.Warn(\"can't find unique id for pod %s, assigning local cache id\", c.PodID)\n\tid := \"cache:\" + strconv.FormatUint(cch.NextID, 16)\n\tcch.NextID++\n\n\treturn id\n}\n\n// Insert a pod into the cache.\nfunc (cch *cache) InsertPod(id string, msg interface{}, status *PodStatus) (Pod, error) {\n\tvar err error\n\n\tp := &pod{cache: cch, ID: id}\n\n\tswitch msg.(type) {\n\tcase *criv1.RunPodSandboxRequest:\n\t\terr = p.fromRunRequest(msg.(*criv1.RunPodSandboxRequest))\n\tcase *criv1.PodSandbox:\n\t\terr = p.fromListResponse(msg.(*criv1.PodSandbox), status)\n\tdefault:\n\t\terr = fmt.Errorf(\"cannot create pod from message %T\", msg)\n\t}\n\n\tif err != nil {\n\t\tcch.Error(\"failed to insert pod %s: %v\", id, err)\n\t\treturn nil, err\n\t}\n\n\tcch.Pods[p.ID] = p\n\n\tcch.Save()\n\n\treturn p, nil\n}\n\n// Delete a pod from the cache.\nfunc (cch *cache) DeletePod(id string) Pod {\n\tp, ok := cch.Pods[id]\n\tif !ok {\n\t\treturn nil\n\t}\n\n\tcch.Debug(\"removing pod %s (%s)\", p.Name, p.ID)\n\tdelete(cch.Pods, id)\n\n\tcch.Save()\n\n\treturn p\n}\n\n// Look up a pod in the cache.\nfunc (cch *cache) LookupPod(id string) (Pod, bool) {\n\tp, ok := cch.Pods[id]\n\treturn p, ok\n}\n\n// Insert a container into the cache.\nfunc (cch *cache) InsertContainer(msg interface{}) (Container, error) {\n\tvar err error\n\n\tc := &container{\n\t\tcache: cch,\n\t}\n\n\tswitch msg.(type) {\n\tcase *criv1.CreateContainerRequest:\n\t\terr = c.fromCreateRequest(msg.(*criv1.CreateContainerRequest))\n\tcase *criv1.Container:\n\t\terr = c.fromListResponse(msg.(*criv1.Container))\n\tdefault:\n\t\terr = fmt.Errorf(\"cannot create container from message %T\", msg)\n\t}\n\n\tif err != nil {\n\t\treturn nil, cacheError(\"failed to insert container %s: %v\", c.CacheID, err)\n\t}\n\n\tc.CacheID = cch.createCacheID(c)\n\n\tcch.Containers[c.CacheID] = c\n\tif c.ID != \"\" {\n\t\tcch.Containers[c.ID] = c\n\t}\n\n\tcch.createContainerDirectory(c.CacheID)\n\n\tadjustments := cch.getApplicableAdjustments(cch.External, c)\n\tswitch {\n\tcase len(adjustments) > 1:\n\t\tcch.Error(\"conflicting adjustments for %s: %s\",\n\t\t\tc.PrettyName(), strings.Join(adjustments, \",\"))\n\tcase len(adjustments) == 1:\n\t\tc.setEffectiveAdjustment(adjustments[0])\n\t}\n\n\tcch.Save()\n\n\treturn c, nil\n}\n\n// UpdateContainerID updates a containers runtime id.\nfunc (cch *cache) UpdateContainerID(cacheID string, msg interface{}) (Container, error) {\n\tc, ok := cch.Containers[cacheID]\n\tif !ok {\n\t\treturn nil, cacheError(\"%s: failed to update ID, container not found\",\n\t\t\tcacheID)\n\t}\n\n\treply, ok := msg.(*criv1.CreateContainerResponse)\n\tif !ok {\n\t\treturn nil, cacheError(\"%s: failed to update ID from message %T\",\n\t\t\tc.PrettyName(), msg)\n\t}\n\n\tc.ID = reply.ContainerId\n\tcch.Containers[c.ID] = c\n\n\tcch.Save()\n\n\treturn c, nil\n}\n\n// Delete a pod from the cache.\nfunc (cch *cache) DeleteContainer(id string) Container {\n\tc, ok := cch.Containers[id]\n\tif !ok {\n\t\treturn nil\n\t}\n\n\tcch.Debug(\"removing container %s\", c.PrettyName())\n\tcch.removeContainerDirectory(c.CacheID)\n\tdelete(cch.Containers, c.ID)\n\tdelete(cch.Containers, c.CacheID)\n\n\tcch.Save()\n\n\treturn c\n}\n\n// Look up a pod in the cache.\nfunc (cch *cache) LookupContainer(id string) (Container, bool) {\n\tc, ok := cch.Containers[id]\n\treturn c, ok\n}\n\n// LookupContainerByCgroup looks up the container for the given cgroup path.\nfunc (cch *cache) LookupContainerByCgroup(path string) (Container, bool) {\n\tcch.Debug(\"resolving %s to a container...\", path)\n\n\tfor id, c := range cch.Containers {\n\t\tif id != c.CacheID {\n\t\t\tcontinue\n\t\t}\n\n\t\tparent := \"\"\n\t\tif pod, ok := c.GetPod(); ok {\n\t\t\tparent = pod.GetCgroupParentDir()\n\t\t}\n\t\tif parent == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tif !strings.HasPrefix(path, parent+\"/\") {\n\t\t\tcontinue\n\t\t}\n\n\t\tif strings.Index(path, c.GetID()) != -1 {\n\t\t\treturn c, true\n\t\t}\n\t}\n\n\treturn nil, false\n}\n\n// RefreshPods purges/inserts stale/new pods/containers using a pod sandbox list response.\nfunc (cch *cache) RefreshPods(msg *criv1.ListPodSandboxResponse, status map[string]*PodStatus) ([]Pod, []Pod, []Container) {\n\tvalid := make(map[string]struct{})\n\n\tadd := []Pod{}\n\tdel := []Pod{}\n\tcontainers := []Container{}\n\n\tfor _, item := range msg.Items {\n\t\tvalid[item.Id] = struct{}{}\n\t\tif _, ok := cch.Pods[item.Id]; !ok {\n\t\t\tcch.Debug(\"inserting discovered pod %s...\", item.Id)\n\t\t\tpod, err := cch.InsertPod(item.Id, item, status[item.Id])\n\t\t\tif err != nil {\n\t\t\t\tcch.Error(\"failed to insert discovered pod %s to cache: %v\",\n\t\t\t\t\titem.Id, err)\n\t\t\t} else {\n\t\t\t\tadd = append(add, pod)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, pod := range cch.Pods {\n\t\tif _, ok := valid[pod.ID]; !ok {\n\t\t\tcch.Debug(\"purging stale pod %s...\", pod.ID)\n\t\t\tpod.State = PodStateStale\n\t\t\tdel = append(del, cch.DeletePod(pod.ID))\n\t\t}\n\t}\n\n\tfor id, c := range cch.Containers {\n\t\tif _, ok := valid[c.PodID]; !ok {\n\t\t\tcch.Debug(\"purging container %s of stale pod %s...\", c.CacheID, c.PodID)\n\t\t\tcch.DeleteContainer(c.CacheID)\n\t\t\tc.State = ContainerStateStale\n\t\t\tif id == c.CacheID {\n\t\t\t\tcontainers = append(containers, c)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn add, del, containers\n}\n\n// RefreshContainers purges/inserts stale/new containers using a container list response.\nfunc (cch *cache) RefreshContainers(msg *criv1.ListContainersResponse) ([]Container, []Container) {\n\tvalid := make(map[string]struct{})\n\n\tadd := []Container{}\n\tdel := []Container{}\n\n\tfor _, c := range msg.Containers {\n\t\tif ContainerState(c.State) == ContainerStateExited {\n\t\t\tcontinue\n\t\t}\n\n\t\tvalid[c.Id] = struct{}{}\n\t\tif _, ok := cch.Containers[c.Id]; !ok {\n\t\t\tcch.Debug(\"inserting discovered container %s...\", c.Id)\n\t\t\tinserted, err := cch.InsertContainer(c)\n\t\t\tif err != nil {\n\t\t\t\tcch.Error(\"failed to insert discovered container %s to cache: %v\",\n\t\t\t\t\tc.Id, err)\n\t\t\t} else {\n\t\t\t\tadd = append(add, inserted)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor id, c := range cch.Containers {\n\t\tif _, ok := valid[c.ID]; !ok {\n\t\t\tcch.Debug(\"purging stale container %s (state: %v)...\", c.CacheID, c.GetState())\n\t\t\tcch.DeleteContainer(c.CacheID)\n\t\t\tc.State = ContainerStateStale\n\t\t\tif id == c.CacheID {\n\t\t\t\tdel = append(del, c)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn add, del\n}\n\n// Mark a container as having pending changes.\nfunc (cch *cache) markPending(c *container) {\n\tif cch.pending == nil {\n\t\tcch.pending = make(map[string]struct{})\n\t}\n\tcch.pending[c.CacheID] = struct{}{}\n}\n\n// Get all containers with pending changes.\nfunc (cch *cache) GetPendingContainers() []Container {\n\tpending := make([]Container, 0, len(cch.pending))\n\tfor id := range cch.pending {\n\t\tc, ok := cch.LookupContainer(id)\n\t\tif ok {\n\t\t\tpending = append(pending, c)\n\t\t}\n\t}\n\treturn pending\n}\n\n// clear the pending state of the given container.\nfunc (cch *cache) clearPending(c *container) {\n\tdelete(cch.pending, c.CacheID)\n}\n\n// Get the cache ids of all cached containers.\nfunc (cch *cache) GetContainerCacheIds() []string {\n\tids := make([]string, len(cch.Containers))\n\n\tidx := 0\n\tfor id, c := range cch.Containers {\n\t\tif id != c.CacheID {\n\t\t\tcontinue\n\t\t}\n\t\tids[idx] = c.CacheID\n\t\tidx++\n\t}\n\n\treturn ids[0:idx]\n}\n\n// Get the ids of all cached containers.\nfunc (cch *cache) GetContainerIds() []string {\n\tids := make([]string, len(cch.Containers))\n\n\tidx := 0\n\tfor id, c := range cch.Containers {\n\t\tif id == c.CacheID {\n\t\t\tcontinue\n\t\t}\n\t\tids[idx] = c.ID\n\t\tidx++\n\t}\n\n\treturn ids[0:idx]\n}\n\n// GetPods returns all pods present in the cache.\nfunc (cch *cache) GetPods() []Pod {\n\tpods := make([]Pod, 0, len(cch.Pods))\n\tfor _, pod := range cch.Pods {\n\t\tpods = append(pods, pod)\n\t}\n\treturn pods\n}\n\n// GetContainers returns all the containers present in the cache.\nfunc (cch *cache) GetContainers() []Container {\n\tcontainers := make([]Container, 0, len(cch.Containers)/2)\n\tfor id, container := range cch.Containers {\n\t\tif id != container.CacheID {\n\t\t\tcontinue\n\t\t}\n\t\tcontainers = append(containers, container)\n\t}\n\treturn containers\n}\n\n// Set the policy entry for a key.\nfunc (cch *cache) SetPolicyEntry(key string, obj interface{}) {\n\tcch.policyData[key] = obj\n\n\tif cch.DebugEnabled() {\n\t\tif data, err := marshalEntry(obj); err != nil {\n\t\t\tcch.Error(\"marshalling of policy entry '%s' failed: %v\", key, err)\n\t\t} else {\n\t\t\tcch.Debug(\"policy entry '%s' set to '%s'\", key, string(data))\n\t\t}\n\t}\n}\n\n// Get the policy entry for a key.\nfunc (cch *cache) GetPolicyEntry(key string, ptr interface{}) bool {\n\n\t//\n\t// Notes:\n\t//     We try to serve requests from the demarshaled cache (policyData).\n\t//     If that fails (may be a first access since load) we look for the\n\t//     entry in the unmarshaled cache (PolicyJSON), demarshal, and cache\n\t//     the entry if found.\n\t//     Note the quirk: in the latter case we first directly unmarshal to\n\t//     the pointer provided by the caller, only then Get() and cache the\n\t//     result.\n\t//\n\n\tobj, ok := cch.policyData[key]\n\tif !ok {\n\t\tentry, ok := cch.PolicyJSON[key]\n\t\tif !ok {\n\t\t\treturn false\n\t\t}\n\n\t\t// first access to key since startup\n\t\tif err := unmarshalEntry([]byte(entry), ptr); err != nil {\n\t\t\tcch.Fatal(\"failed to unmarshal '%s' policy entry for key '%s' (%T): %v\",\n\t\t\t\tcch.PolicyName, key, ptr, err)\n\t\t}\n\n\t\tif err := cch.cacheEntry(key, ptr); err != nil {\n\t\t\tcch.Fatal(\"failed to cache '%s' policy entry for key '%s': %v\",\n\t\t\t\tcch.PolicyName, key, err)\n\t\t}\n\t} else {\n\t\t// subsequent accesses to key\n\t\tif err := cch.setEntry(ptr, obj); err != nil {\n\t\t\tcch.Fatal(\"failed use cached entry for key '%s' of policy '%s': %v\",\n\t\t\t\tkey, cch.PolicyName, err)\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Marshal an opaque policy entry, special-casing cpusets and maps of cpusets.\nfunc marshalEntry(obj interface{}) ([]byte, error) {\n\tswitch obj.(type) {\n\tcase cpuset.CPUSet:\n\t\treturn []byte(\"\\\"\" + obj.(cpuset.CPUSet).String() + \"\\\"\"), nil\n\tcase map[string]cpuset.CPUSet:\n\t\tdst := make(map[string]string)\n\t\tfor key, cset := range obj.(map[string]cpuset.CPUSet) {\n\t\t\tdst[key] = cset.String()\n\t\t}\n\t\treturn json.Marshal(dst)\n\n\tdefault:\n\t\treturn json.Marshal(obj)\n\t}\n}\n\n// Unmarshal an opaque policy entry, special-casing cpusets and maps of cpusets.\nfunc unmarshalEntry(data []byte, ptr interface{}) error {\n\tswitch ptr.(type) {\n\tcase *cpuset.CPUSet:\n\t\tcset, err := cpuset.Parse(string(data[1 : len(data)-1]))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t*ptr.(*cpuset.CPUSet) = cset\n\t\treturn nil\n\n\tcase *map[string]cpuset.CPUSet:\n\t\tsrc := make(map[string]string)\n\t\tif err := json.Unmarshal([]byte(data), &src); err != nil {\n\t\t\treturn cacheError(\"failed to unmarshal map[string]cpuset.CPUSet: %v\", err)\n\t\t}\n\n\t\tdst := make(map[string]cpuset.CPUSet)\n\t\tfor key, str := range src {\n\t\t\tcset, err := cpuset.Parse(str)\n\t\t\tif err != nil {\n\t\t\t\treturn cacheError(\"failed to unmarshal cpuset.CPUSet '%s': %v\", str, err)\n\t\t\t}\n\t\t\tdst[key] = cset\n\t\t}\n\n\t\t*ptr.(*map[string]cpuset.CPUSet) = dst\n\t\treturn nil\n\n\tdefault:\n\t\terr := json.Unmarshal(data, ptr)\n\t\treturn err\n\t}\n}\n\n// Cache an unmarshaled opaque policy entry, special-casing some simple/common types.\nfunc (cch *cache) cacheEntry(key string, ptr interface{}) error {\n\tif cachable, ok := ptr.(Cachable); ok {\n\t\tcch.policyData[key] = cachable.Get()\n\t\treturn nil\n\t}\n\n\tswitch ptr.(type) {\n\tcase *cpuset.CPUSet:\n\t\tcch.policyData[key] = *ptr.(*cpuset.CPUSet)\n\tcase *map[string]cpuset.CPUSet:\n\t\tcch.policyData[key] = *ptr.(*map[string]cpuset.CPUSet)\n\tcase *map[string]string:\n\t\tcch.policyData[key] = *ptr.(*map[string]string)\n\n\tcase *string:\n\t\tcch.policyData[key] = *ptr.(*string)\n\tcase *bool:\n\t\tcch.policyData[key] = *ptr.(*bool)\n\n\tcase *int32:\n\t\tcch.policyData[key] = *ptr.(*int32)\n\tcase *uint32:\n\t\tcch.policyData[key] = *ptr.(*uint32)\n\tcase *int64:\n\t\tcch.policyData[key] = *ptr.(*int64)\n\tcase *uint64:\n\t\tcch.policyData[key] = *ptr.(*uint64)\n\n\tcase *int:\n\t\tcch.policyData[key] = *ptr.(*int)\n\tcase *uint:\n\t\tcch.policyData[key] = *ptr.(*uint)\n\n\tdefault:\n\t\treturn cacheError(\"can't handle policy data of type %T\", ptr)\n\t}\n\n\treturn nil\n}\n\n// Serve an unmarshaled opaque policy entry, special-casing some simple/common types.\nfunc (cch *cache) setEntry(ptr, obj interface{}) error {\n\tif cachable, ok := ptr.(Cachable); ok {\n\t\tcachable.Set(obj)\n\t\treturn nil\n\t}\n\n\tswitch ptr.(type) {\n\tcase *cpuset.CPUSet:\n\t\t*ptr.(*cpuset.CPUSet) = obj.(cpuset.CPUSet)\n\tcase *map[string]cpuset.CPUSet:\n\t\t*ptr.(*map[string]cpuset.CPUSet) = obj.(map[string]cpuset.CPUSet)\n\tcase *map[string]string:\n\t\t*ptr.(*map[string]string) = obj.(map[string]string)\n\n\tcase *string:\n\t\t*ptr.(*string) = obj.(string)\n\tcase *bool:\n\t\t*ptr.(*bool) = obj.(bool)\n\n\tcase *int32:\n\t\t*ptr.(*int32) = obj.(int32)\n\tcase *uint32:\n\t\t*ptr.(*uint32) = obj.(uint32)\n\tcase *int64:\n\t\t*ptr.(*int64) = obj.(int64)\n\tcase *uint64:\n\t\t*ptr.(*uint64) = obj.(uint64)\n\n\tcase *int:\n\t\t*ptr.(*int) = obj.(int)\n\tcase *uint:\n\t\t*ptr.(*uint) = obj.(uint)\n\n\tdefault:\n\t\treturn cacheError(\"can't handle policy data of type %T\", ptr)\n\t}\n\n\treturn nil\n}\n\n// checkPerm checks permissions of an already existing file or directory.\nfunc (cch *cache) checkPerm(what, path string, isDir bool, p *permissions) (bool, error) {\n\tif isDir {\n\t\twhat += \" directory\"\n\t}\n\n\tinfo, err := os.Stat(path)\n\tif err != nil {\n\t\tif !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn true, cacheError(\"failed to os.Stat() %s %q: %v\", what, path, err)\n\t\t}\n\t\treturn false, nil\n\t}\n\n\t// check expected file type\n\tif isDir {\n\t\tif !info.IsDir() {\n\t\t\treturn true, cacheError(\"%s %q exists, but is not a directory\", what, path)\n\t\t}\n\t} else {\n\t\tif info.Mode()&os.ModeType != 0 {\n\t\t\treturn true, cacheError(\"%s %q exists, but is not a regular file\", what, path)\n\t\t}\n\t}\n\n\texisting := info.Mode().Perm()\n\texpected := p.prefer\n\trejected := p.reject\n\tif ((expected | rejected) &^ os.ModePerm) != 0 {\n\t\tcch.Panic(\"internal error: current permissions check only handles permission bits (rwx)\")\n\t}\n\n\t// check that we don't have any of the rejectable permission bits set\n\tif existing&rejected != 0 {\n\t\treturn true, cacheError(\"existing %s %q has disallowed permissions set: %v\",\n\t\t\twhat, path, existing&rejected)\n\t}\n\n\t// warn if permissions are less strict than the preferred defaults\n\tif (existing | expected) != expected {\n\t\tcch.Warn(\"existing %s %q has less strict permissions %v than expected %v\",\n\t\t\twhat, path, existing, expected)\n\t}\n\n\treturn true, nil\n}\n\n// mkdirAll creates a directory, checking permissions if it already exists.\nfunc (cch *cache) mkdirAll(what, path string, p *permissions) error {\n\texists, err := cch.checkPerm(what, path, true, p)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif exists {\n\t\treturn nil\n\t}\n\n\tif err := os.MkdirAll(path, p.prefer); err != nil {\n\t\treturn cacheError(\"failed to create %s directory %q: %v\", what, path, err)\n\t}\n\n\treturn nil\n}\n\n// snapshot is used to serialize the cache into a saveable/loadable state.\ntype snapshot struct {\n\tVersion    string\n\tPods       map[string]*pod\n\tContainers map[string]*container\n\tNextID     uint64\n\tCfg        *config.RawConfig\n\tPolicyName string\n\tPolicyJSON map[string]string\n}\n\n// Snapshot takes a restorable snapshot of the current state of the cache.\nfunc (cch *cache) Snapshot() ([]byte, error) {\n\ts := snapshot{\n\t\tVersion:    CacheVersion,\n\t\tPods:       make(map[string]*pod),\n\t\tContainers: make(map[string]*container),\n\t\tCfg:        cch.Cfg,\n\t\tNextID:     cch.NextID,\n\t\tPolicyName: cch.PolicyName,\n\t\tPolicyJSON: cch.PolicyJSON,\n\t}\n\n\tfor id, p := range cch.Pods {\n\t\ts.Pods[id] = p\n\t}\n\n\tfor id, c := range cch.Containers {\n\t\tif id == c.CacheID {\n\t\t\ts.Containers[c.CacheID] = c\n\t\t}\n\t}\n\n\tfor key, obj := range cch.policyData {\n\t\tdata, err := marshalEntry(obj)\n\t\tif err != nil {\n\t\t\treturn nil, cacheError(\"failed to marshal policy entry '%s': %v\", key, err)\n\t\t}\n\n\t\ts.PolicyJSON[key] = string(data)\n\t}\n\n\tdata, err := json.Marshal(s)\n\tif err != nil {\n\t\treturn nil, cacheError(\"failed to marshal cache: %v\", err)\n\t}\n\n\treturn data, nil\n}\n\n// Restore restores a previously takes snapshot of the cache.\nfunc (cch *cache) Restore(data []byte) error {\n\ts := snapshot{\n\t\tPods:       make(map[string]*pod),\n\t\tContainers: make(map[string]*container),\n\t\tPolicyJSON: make(map[string]string),\n\t}\n\n\tif err := json.Unmarshal(data, &s); err != nil {\n\t\treturn cacheError(\"failed to unmarshal snapshot data: %v\", err)\n\t}\n\n\tif s.Version != CacheVersion {\n\t\treturn cacheError(\"can't restore snapshot, version '%s' != running version %s\",\n\t\t\ts.Version, CacheVersion)\n\t}\n\n\tcch.Pods = s.Pods\n\tcch.Containers = s.Containers\n\tcch.Cfg = s.Cfg\n\tcch.NextID = s.NextID\n\tcch.PolicyJSON = s.PolicyJSON\n\tcch.PolicyName = s.PolicyName\n\tcch.policyData = make(map[string]interface{})\n\n\tfor _, p := range cch.Pods {\n\t\tp.cache = cch\n\t\tp.containers = make(map[string]string)\n\t}\n\tfor _, c := range cch.Containers {\n\t\tc.cache = cch\n\t\tcch.Containers[c.CacheID] = c\n\t\tif c.ID != \"\" {\n\t\t\tcch.Containers[c.ID] = c\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Save the state of the cache.\nfunc (cch *cache) Save() error {\n\tcch.Debug(\"saving cache to file '%s'...\", cch.filePath)\n\n\tdata, err := cch.Snapshot()\n\tif err != nil {\n\t\treturn cacheError(\"failed to save cache: %v\", err)\n\t}\n\n\ttmpPath := cch.filePath + \".saving\"\n\tif err = os.WriteFile(tmpPath, data, cacheFilePerm.prefer); err != nil {\n\t\treturn cacheError(\"failed to write cache to file %q: %v\", tmpPath, err)\n\t}\n\tif err := os.Rename(tmpPath, cch.filePath); err != nil {\n\t\treturn cacheError(\"failed to rename %q to %q: %v\",\n\t\t\ttmpPath, cch.filePath, err)\n\t}\n\n\treturn nil\n}\n\n// Load loads the last saved state of the cache.\nfunc (cch *cache) Load() error {\n\tcch.Debug(\"loading cache from file '%s'...\", cch.filePath)\n\n\tdata, err := os.ReadFile(cch.filePath)\n\n\tswitch {\n\tcase os.IsNotExist(err):\n\t\tcch.Debug(\"no cache file '%s', nothing to restore\", cch.filePath)\n\t\treturn nil\n\tcase len(data) == 0:\n\t\tcch.Debug(\"empty cache file '%s', nothing to restore\", cch.filePath)\n\t\treturn nil\n\tcase err != nil:\n\t\treturn cacheError(\"failed to load cache from file '%s': %v\", cch.filePath, err)\n\t}\n\n\treturn cch.Restore(data)\n}\n\nfunc (cch *cache) ContainerDirectory(id string) string {\n\tc, ok := cch.Containers[id]\n\tif !ok {\n\t\treturn \"\"\n\t}\n\treturn filepath.Join(cch.dataDir, strings.Replace(c.CacheID, \":\", \"-\", 1))\n}\n\nfunc (cch *cache) createContainerDirectory(id string) error {\n\tdir := cch.ContainerDirectory(id)\n\tif dir == \"\" {\n\t\treturn cacheError(\"failed to determine container directory path for container %s\", id)\n\t}\n\treturn cch.mkdirAll(\"container directory\", dir, dataDirPerm)\n}\n\nfunc (cch *cache) removeContainerDirectory(id string) error {\n\tdir := cch.ContainerDirectory(id)\n\tif dir == \"\" {\n\t\treturn cacheError(\"failed to delete directory for container %s\", id)\n\t}\n\treturn os.RemoveAll(dir)\n}\n\nfunc (cch *cache) OpenFile(id string, name string, perm os.FileMode) (*os.File, error) {\n\tdir := cch.ContainerDirectory(id)\n\tif dir == \"\" {\n\t\treturn nil, cacheError(\"failed to determine data directory for container %s\", id)\n\t}\n\tif err := cch.mkdirAll(\"container directory\", dir, dataDirPerm); err != nil {\n\t\treturn nil, cacheError(\"container %s: can't create data file %q: %v\", id, name, err)\n\t}\n\n\tpath := filepath.Join(dir, name)\n\tif _, err := cch.checkPerm(\"container\", path, false, dataFilePerm); err != nil {\n\t\treturn nil, err\n\t}\n\n\tfile, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)\n\tif err != nil {\n\t\treturn nil, cacheError(\"container %s: can't open data file %q: %v\", id, path, err)\n\t}\n\n\treturn file, nil\n}\n\nfunc (cch *cache) WriteFile(id string, name string, perm os.FileMode, data []byte) error {\n\tfile, err := cch.OpenFile(id, name, perm)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\t_, err = file.Write(data)\n\n\treturn err\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/cache_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n)\n\nvar nextFakePodID = 1\nvar nextFakeContainerID = 1\n\ntype fakePod struct {\n\tname        string\n\tuid         string\n\tid          string\n\tqos         v1.PodQOSClass\n\tlabels      map[string]string\n\tannotations map[string]string\n\tpodCfg      *criv1.PodSandboxConfig\n}\n\ntype fakeContainer struct {\n\tfakePod     *fakePod\n\tname        string\n\tid          string\n\tlabels      map[string]string\n\tannotations map[string]string\n\tresources   criv1.LinuxContainerResources\n}\n\nfunc createTmpCache() (Cache, string, error) {\n\tdir, err := os.MkdirTemp(\"\", \"cache-test\")\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tcch, err := NewCache(Options{CacheDir: dir})\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\treturn cch, dir, nil\n}\n\nfunc removeTmpCache(dir string) {\n\tif dir != \"\" {\n\t\tos.RemoveAll(dir)\n\t}\n}\n\nfunc createFakePod(cch Cache, fp *fakePod) (Pod, error) {\n\tif fp.labels == nil {\n\t\tfp.labels = make(map[string]string)\n\t}\n\tfp.id = fmt.Sprintf(\"pod%4.4d\", nextFakePodID)\n\tfp.uid = fmt.Sprintf(\"poduid%4.4d\", nextFakePodID)\n\tfp.labels[kubernetes.PodUIDLabel] = fp.uid\n\tnextFakePodID++\n\n\tif string(fp.qos) == \"\" {\n\t\tfp.qos = v1.PodQOSBurstable\n\t}\n\n\tcgroupPath := \"\"\n\tif fp.qos != v1.PodQOSGuaranteed {\n\t\tpathClass := \"kubepods-\" + strings.ToLower(string(fp.qos))\n\t\tcgroupPath = \"/kubepods.slice/\" + pathClass + \".slice/\" + pathClass + \"-pod\" + fp.uid\n\t} else {\n\t\tcgroupPath = \"/kubepods.slice/kubepods-pod\" + strings.ReplaceAll(fp.uid, \"-\", \"_\")\n\t}\n\n\treq := &criv1.RunPodSandboxRequest{\n\t\tConfig: &criv1.PodSandboxConfig{\n\t\t\tMetadata: &criv1.PodSandboxMetadata{\n\t\t\t\tName:      fp.name,\n\t\t\t\tUid:       fp.uid,\n\t\t\t\tNamespace: \"default\",\n\t\t\t},\n\t\t\tLabels:      fp.labels,\n\t\t\tAnnotations: fp.annotations,\n\t\t\tLinux: &criv1.LinuxPodSandboxConfig{\n\t\t\t\tCgroupParent: cgroupPath,\n\t\t\t},\n\t\t},\n\t}\n\tfp.podCfg = req.Config\n\n\tcch.(*cache).Debug(\"*** => creating Pod: %+v\\n\", *req)\n\tp, err := cch.InsertPod(fp.id, req, nil)\n\tif err != nil {\n\t\tcch.(*cache).Debug(\"*** <= created Pod FAILED: %+v\\n\", err)\n\t\treturn nil, err\n\t}\n\tcch.(*cache).Debug(\"*** <= created Pod: %+v\\n\", *p.(*pod))\n\treturn p, nil\n}\n\nfunc createFakeContainer(cch Cache, fc *fakeContainer) (Container, error) {\n\tif fc.labels == nil {\n\t\tfc.labels = make(map[string]string)\n\t}\n\tfc.id = fmt.Sprintf(\"container-id-%4.4d\", nextFakeContainerID)\n\tnextFakeContainerID++\n\n\treq := &criv1.CreateContainerRequest{\n\t\tPodSandboxId: fc.fakePod.id,\n\t\tConfig: &criv1.ContainerConfig{\n\t\t\tMetadata: &criv1.ContainerMetadata{\n\t\t\t\tName: fc.name,\n\t\t\t},\n\t\t\tLabels:      fc.labels,\n\t\t\tAnnotations: fc.annotations,\n\t\t\tLinux: &criv1.LinuxContainerConfig{\n\t\t\t\tResources: &fc.resources,\n\t\t\t},\n\t\t},\n\t\tSandboxConfig: fc.fakePod.podCfg,\n\t}\n\n\tcch.(*cache).Debug(\"*** => creating Container: %+v\\n\", *req)\n\tc, err := cch.InsertContainer(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcch.(*cache).Debug(\"*** <= created Container: %+v\\n\", *c.(*container))\n\tupdate := &criv1.CreateContainerResponse{ContainerId: fc.id}\n\tif _, err := cch.UpdateContainerID(c.GetCacheID(), update); err != nil {\n\t\treturn nil, err\n\t}\n\treturn c, nil\n}\n\nfunc TestLookupContainerByCgroup(t *testing.T) {\n\tfakePods := map[string]*fakePod{\n\t\t\"pod1\": {name: \"pod1\"},\n\t\t\"pod2\": {name: \"pod2\"},\n\t\t\"pod3\": {name: \"pod3\"},\n\t}\n\n\tfakePodContainers := map[string][]*fakeContainer{\n\t\t\"pod1\": {{name: \"container1\"}, {name: \"container2\"}, {name: \"err-container3\"}},\n\t\t\"pod2\": {{name: \"err-container4\"}, {name: \"container5\"}, {name: \"err-container6\"}},\n\t\t\"pod3\": {{name: \"container7\"}, {name: \"container8\"}, {name: \"container10\"}},\n\t}\n\n\tcch, dir, err := createTmpCache()\n\tif err != nil {\n\t\tt.Errorf(\"failed: %v\", err)\n\t}\n\tdefer removeTmpCache(dir)\n\n\tfor _, fp := range fakePods {\n\t\t_, err := createFakePod(cch, fp)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"failed to create fake pod: %v\", err)\n\t\t}\n\t}\n\n\tfor podName, fcs := range fakePodContainers {\n\t\tfp, ok := fakePods[podName]\n\t\tif !ok {\n\t\t\tt.Errorf(\"failed to find fake pod '%s'\", podName)\n\t\t}\n\t\tfor _, fc := range fcs {\n\t\t\tfc.fakePod = fp\n\t\t\tif _, err := createFakeContainer(cch, fc); err != nil {\n\t\t\t\tt.Errorf(\"failed to create fake container '%s.%s': %v\", podName, fc.name, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, c := range cch.GetContainers() {\n\t\tp, ok := c.GetPod()\n\t\tif !ok {\n\t\t\tt.Errorf(\"failed to find Pod for Container %s\", c.PrettyName())\n\t\t}\n\t\tpodCgroupDir := p.GetCgroupParentDir()\n\t\tpath := podCgroupDir + \"/container-\" + c.GetID() + \".scope\"\n\n\t\tcch.(*cache).Info(\"=> %s: testing lookup by cgroup path %s...\", c.PrettyName(), path)\n\t\tchk, ok := cch.LookupContainerByCgroup(path)\n\t\tif !ok {\n\t\t\tt.Errorf(\"failed to look up container %s by cgroup path %s (pod parent cgroup: %s)\",\n\t\t\t\tc.PrettyName(), path, podCgroupDir)\n\t\t}\n\t\tcch.(*cache).Info(\"<= %s\", chk.PrettyName())\n\n\t\tif strings.HasPrefix(c.GetName(), \"err-\") {\n\t\t\tpath := podCgroupDir + \"-another/container-\" + c.GetID() + \".scope\"\n\n\t\t\tcch.(*cache).Info(\"=> %s: testing lookup failure by cgroup path %s...\",\n\t\t\t\tc.PrettyName(), path)\n\t\t\tchk, ok := cch.LookupContainerByCgroup(path)\n\t\t\tif ok {\n\t\t\t\tt.Errorf(\"look up of container %s by path %s should have failed, but gave %s\",\n\t\t\t\t\tc.PrettyName(), path, chk.PrettyName())\n\t\t\t}\n\t\t\tcch.(*cache).Info(\"<= OK (not found as expected)\")\n\t\t}\n\n\t\tif chk.GetID() != c.GetID() {\n\t\t\tt.Errorf(\"found container %s is not the expected %s\", chk.GetID(), c.GetID())\n\t\t}\n\t}\n}\n\nfunc TestDefaultRDTAndBlockIOClasses(t *testing.T) {\n\tfakePods := map[string]*fakePod{\n\t\t\"pod1\": {\n\t\t\tname: \"pod1\",\n\t\t\tqos:  v1.PodQOSBestEffort,\n\t\t\tannotations: map[string]string{\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/pod\": \"Pod1RDT\",\n\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.container1\":     \"RDT1\",\n\t\t\t\t\"blockioclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.container1\": \"BLKIO1\",\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.container2\":     \"RDT2\",\n\t\t\t\t\"blockioclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.container2\": \"BLKIO2\",\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.container3\":     \"RDT3\",\n\t\t\t\t\"blockioclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.container4\": \"BLKIO4\",\n\t\t\t},\n\t\t},\n\t\t\"pod2\": {\n\t\t\tname: \"pod2\",\n\t\t\tqos:  v1.PodQOSBurstable,\n\t\t\tannotations: map[string]string{\n\t\t\t\t\"blockioclass.\" + kubernetes.ResmgrKeyNamespace: \"Pod2BLKIO\",\n\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.3\":     \"RDT3\",\n\t\t\t\t\"blockioclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.3\": \"BLKIO3\",\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.4\":     \"RDT4\",\n\t\t\t\t\"rdtclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.1\":     \"RDT1\",\n\t\t\t\t\"blockioclass.\" + kubernetes.ResmgrKeyNamespace + \"/container.2\": \"BLKIO2\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfakePodContainers := map[string][]*fakeContainer{\n\t\t\"pod1\": {\n\t\t\t{name: \"container1\"},\n\t\t\t{name: \"container2\"},\n\t\t\t{name: \"container3\"},\n\t\t\t{name: \"container4\"},\n\t\t},\n\t}\n\n\ttype classes struct {\n\t\tRDT     string\n\t\tBlockIO string\n\t}\n\n\texpected := map[string]map[string]classes{\n\t\t\"pod1\": {\n\t\t\t\"container1\": {\n\t\t\t\tRDT:     \"RDT1\",\n\t\t\t\tBlockIO: \"BLKIO1\",\n\t\t\t},\n\t\t\t\"container2\": {\n\t\t\t\tRDT:     \"RDT2\",\n\t\t\t\tBlockIO: \"BLKIO2\",\n\t\t\t},\n\t\t\t\"container3\": {\n\t\t\t\tRDT:     \"RDT3\",\n\t\t\t\tBlockIO: string(fakePods[\"pod1\"].qos),\n\t\t\t},\n\t\t\t\"container4\": {\n\t\t\t\tRDT:     \"Pod1RDT\",\n\t\t\t\tBlockIO: \"BLKIO4\",\n\t\t\t},\n\t\t},\n\t\t\"pod2\": {\n\t\t\t\"container1\": {\n\t\t\t\tRDT:     \"RDT1\",\n\t\t\t\tBlockIO: \"Pod2BLKIO\",\n\t\t\t},\n\t\t\t\"container2\": {\n\t\t\t\tRDT:     string(fakePods[\"pod2\"].qos),\n\t\t\t\tBlockIO: \"BLKIO2\",\n\t\t\t},\n\t\t\t\"container3\": {\n\t\t\t\tRDT:     \"RDT3\",\n\t\t\t\tBlockIO: \"BLKIO3\",\n\t\t\t},\n\t\t\t\"container4\": {\n\t\t\t\tRDT:     \"RDT4\",\n\t\t\t\tBlockIO: \"Pod2BLKIO\",\n\t\t\t},\n\t\t},\n\t}\n\n\tcch, dir, err := createTmpCache()\n\tif err != nil {\n\t\tt.Errorf(\"failed: %v\", err)\n\t}\n\tdefer removeTmpCache(dir)\n\n\tfor _, fp := range fakePods {\n\t\t_, err := createFakePod(cch, fp)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"failed to create fake pod: %v\", err)\n\t\t}\n\t}\n\n\tfor podName, fcs := range fakePodContainers {\n\t\tfp, ok := fakePods[podName]\n\t\tif !ok {\n\t\t\tt.Errorf(\"failed to find fake pod '%s'\", podName)\n\t\t}\n\t\tfor _, fc := range fcs {\n\t\t\tfc.fakePod = fp\n\t\t\tif _, err := createFakeContainer(cch, fc); err != nil {\n\t\t\t\tt.Errorf(\"failed to create fake container '%s.%s': %v\", podName, fc.name, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, c := range cch.GetContainers() {\n\t\tpod, ok := c.GetPod()\n\t\tif !ok {\n\t\t\tt.Errorf(\"failed to find Pod for Container %s\", c.PrettyName())\n\t\t}\n\n\t\texp, ok := expected[pod.GetName()][c.GetName()]\n\t\tif !ok {\n\t\t\tt.Errorf(\"failed to find expected results Container %s\", c.PrettyName())\n\t\t}\n\n\t\tif c.GetRDTClass() != exp.RDT {\n\t\t\tt.Errorf(\"container %s: RDT class %s, expected %s\", c.PrettyName(),\n\t\t\t\tc.GetRDTClass(), exp.RDT)\n\t\t}\n\n\t\tif c.GetBlockIOClass() != exp.BlockIO {\n\t\t\tt.Errorf(\"container %s: BlockIO class %s, expected %s\", c.PrettyName(),\n\t\t\t\tc.GetBlockIOClass(), exp.BlockIO)\n\t\t}\n\t}\n}\n\nconst (\n\t// anything below 2 millicpus will yield 0 as an estimate\n\tminNonZeroRequest = 2\n\t// check CPU request/limit estimate accuracy up to this many CPU cores\n\tmaxCPU = (kubernetes.MaxShares / kubernetes.SharesPerCPU) * kubernetes.MilliCPUToCPU\n\t// we expect our estimates to be within 1 millicpu from the real ones\n\texpectedAccuracy = 1\n)\n\nfunc TestCPURequestCalculationAccuracy(t *testing.T) {\n\tfor request := 0; request < maxCPU; request++ {\n\t\tshares := MilliCPUToShares(int64(request))\n\t\testimate := SharesToMilliCPU(int64(shares))\n\n\t\tdiff := int64(request) - estimate\n\t\tif diff > expectedAccuracy || diff < -expectedAccuracy {\n\t\t\tif diff < 0 {\n\t\t\t\tdiff = -diff\n\t\t\t}\n\t\t\tif request > minNonZeroRequest {\n\t\t\t\tt.Errorf(\"CPU request %v: estimate %v, unexpected inaccuracy %v > %v\",\n\t\t\t\t\trequest, estimate, diff, expectedAccuracy)\n\t\t\t} else {\n\t\t\t\tt.Logf(\"CPU request %v: estimate %v, inaccuracy %v > %v (OK, this was expected)\",\n\t\t\t\t\trequest, estimate, diff, expectedAccuracy)\n\t\t\t}\n\t\t}\n\n\t\t// fail if our estimates are not accurate for full CPUs worth of millicpus\n\t\tif (request%1000) == 0 && diff != 0 {\n\t\t\tt.Errorf(\"CPU request %v != estimate %v (diff %v)\", request, estimate, diff)\n\t\t}\n\t}\n}\n\nfunc TestCPULimitCalculationAccuracy(t *testing.T) {\n\tfor limit := int64(0); limit < int64(maxCPU); limit++ {\n\t\tquota, period := MilliCPUToQuota(limit)\n\t\testimate := QuotaToMilliCPU(quota, period)\n\n\t\tdiff := limit - estimate\n\t\tif diff > expectedAccuracy || diff < -expectedAccuracy {\n\t\t\tif diff < 0 {\n\t\t\t\tdiff = -diff\n\t\t\t}\n\t\t\tif quota != kubernetes.MinQuotaPeriod {\n\t\t\t\tt.Errorf(\"CPU limit %v: estimate %v, unexpected inaccuracy %v > %v\",\n\t\t\t\t\tlimit, estimate, diff, expectedAccuracy)\n\t\t\t} else {\n\t\t\t\tt.Logf(\"CPU limit %v: estimate %v, inaccuracy %v > %v (OK, this was expected)\",\n\t\t\t\t\tlimit, estimate, diff, expectedAccuracy)\n\t\t\t}\n\t\t}\n\n\t\t// fail if our estimates are not accurate for full CPUs worth of millicpus\n\t\tif (limit%1000) == 0 && diff != 0 {\n\t\t\tt.Errorf(\"CPU limit %v != estimate %v (diff %v)\", limit, estimate, diff)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/container.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"encoding/json\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\textapi \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n)\n\n// Create a container for a create request.\nfunc (c *container) fromCreateRequest(req *criv1.CreateContainerRequest) error {\n\tc.PodID = req.PodSandboxId\n\n\tpod, ok := c.cache.Pods[c.PodID]\n\tif !ok {\n\t\treturn cacheError(\"can't find cached pod %s for container to create\", c.PodID)\n\t}\n\n\tcfg := req.Config\n\tif cfg == nil {\n\t\treturn cacheError(\"container of pod %s has no config\", c.PodID)\n\t}\n\tmeta := cfg.Metadata\n\tif meta == nil {\n\t\treturn cacheError(\"container of pod %s has no request metadata\", c.PodID)\n\t}\n\tpodCfg := req.SandboxConfig\n\tif podCfg == nil {\n\t\treturn cacheError(\"container of pod %s has no request pod config data\", c.PodID)\n\t}\n\tpodMeta := podCfg.Metadata\n\tif podMeta == nil {\n\t\treturn cacheError(\"container of pod %s has no request pod metadata\", c.PodID)\n\t}\n\n\tc.Name = meta.Name\n\tc.Namespace = podMeta.Namespace\n\tc.State = ContainerStateCreating\n\tc.Image = cfg.GetImage().GetImage()\n\tc.Command = cfg.Command\n\tc.Args = cfg.Args\n\tc.Labels = cfg.Labels\n\tc.Annotations = cfg.Annotations\n\n\tc.Env = make(map[string]string)\n\tfor _, kv := range cfg.Envs {\n\t\tc.Env[kv.Key] = kv.Value\n\t}\n\n\tgenHints := true\n\tif hintSetting, ok := c.GetEffectiveAnnotation(TopologyHintsKey); ok {\n\t\tpreference, err := strconv.ParseBool(hintSetting)\n\t\tif err != nil {\n\t\t\tc.cache.Error(\"invalid annotation %q=%q: %v\", TopologyHintsKey, hintSetting, err)\n\t\t} else {\n\t\t\tgenHints = preference\n\t\t}\n\t}\n\tc.cache.Info(\"automatic topology hint generation %s for %q\",\n\t\tmap[bool]string{false: \"disabled\", true: \"enabled\"}[genHints], c.PrettyName())\n\n\tc.Mounts = make(map[string]*Mount)\n\tfor _, m := range cfg.Mounts {\n\t\tc.Mounts[m.ContainerPath] = &Mount{\n\t\t\tContainer:   m.ContainerPath,\n\t\t\tHost:        m.HostPath,\n\t\t\tReadonly:    m.Readonly,\n\t\t\tRelabel:     m.SelinuxRelabel,\n\t\t\tPropagation: MountType(m.Propagation),\n\t\t}\n\n\t\tif genHints {\n\t\t\tif hints := getTopologyHints(m.HostPath, m.ContainerPath, m.Readonly); len(hints) > 0 {\n\t\t\t\tc.TopologyHints = topology.MergeTopologyHints(c.TopologyHints, hints)\n\t\t\t}\n\t\t}\n\t}\n\n\tc.Devices = make(map[string]*Device)\n\tfor _, d := range cfg.Devices {\n\t\tc.Devices[d.ContainerPath] = &Device{\n\t\t\tContainer:   d.ContainerPath,\n\t\t\tHost:        d.HostPath,\n\t\t\tPermissions: d.Permissions,\n\t\t}\n\t\tif genHints {\n\t\t\tif hints := getTopologyHints(d.HostPath, d.ContainerPath, strings.IndexAny(d.Permissions, \"wm\") == -1); len(hints) > 0 {\n\t\t\t\tc.TopologyHints = topology.MergeTopologyHints(c.TopologyHints, hints)\n\t\t\t}\n\t\t}\n\t}\n\n\tc.Tags = make(map[string]string)\n\n\tc.LinuxReq = cfg.GetLinux().GetResources()\n\n\tif pod.Resources != nil {\n\t\tif r, ok := pod.Resources.InitContainers[c.Name]; ok {\n\t\t\tc.Resources = r\n\t\t} else if r, ok := pod.Resources.Containers[c.Name]; ok {\n\t\t\tc.Resources = r\n\t\t}\n\t}\n\n\tif len(c.Resources.Requests) == 0 && len(c.Resources.Limits) == 0 {\n\t\tc.Resources = estimateComputeResources(c.LinuxReq, pod.CgroupParent)\n\t}\n\n\tc.TopologyHints = topology.MergeTopologyHints(c.TopologyHints, getKubeletHint(c.GetCpusetCpus(), c.GetCpusetMems()))\n\n\tif err := c.setDefaults(); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Create container from a container list response.\nfunc (c *container) fromListResponse(lrc *criv1.Container) error {\n\tc.PodID = lrc.PodSandboxId\n\n\tpod, ok := c.cache.Pods[c.PodID]\n\tif !ok {\n\t\treturn cacheError(\"can't find cached pod %s for listed container\", c.PodID)\n\t}\n\n\tmeta := lrc.Metadata\n\tif meta == nil {\n\t\treturn cacheError(\"listed container of pod %s has no metadata\", c.PodID)\n\t}\n\n\tc.ID = lrc.Id\n\tc.Name = meta.Name\n\tc.Namespace = pod.Namespace\n\tc.State = ContainerState(int32(lrc.State))\n\tc.Image = lrc.GetImage().GetImage()\n\tc.Labels = lrc.Labels\n\tc.Annotations = lrc.Annotations\n\tc.Tags = make(map[string]string)\n\n\tif pod.Resources != nil {\n\t\tif r, ok := pod.Resources.InitContainers[c.Name]; ok {\n\t\t\tc.Resources = r\n\t\t} else if r, ok := pod.Resources.Containers[c.Name]; ok {\n\t\t\tc.Resources = r\n\t\t}\n\t}\n\n\tif len(c.Resources.Requests) == 0 && len(c.Resources.Limits) == 0 {\n\t\tc.Resources = estimateComputeResources(c.LinuxReq, pod.CgroupParent)\n\t}\n\n\tif err := c.setDefaults(); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (c *container) setDefaults() error {\n\tclass, ok := c.GetEffectiveAnnotation(RDTClassKey)\n\tif !ok {\n\t\tclass = RDTClassPodQoS\n\t}\n\tc.SetRDTClass(class)\n\n\tclass, ok = c.GetEffectiveAnnotation(BlockIOClassKey)\n\tif !ok {\n\t\tclass = string(c.GetQOSClass())\n\t}\n\tc.SetBlockIOClass(class)\n\n\tlimit, ok := c.GetEffectiveAnnotation(ToptierLimitKey)\n\tif !ok {\n\t\tc.ToptierLimit = ToptierLimitUnset\n\t} else {\n\t\tqty, err := resapi.ParseQuantity(limit)\n\t\tif err != nil {\n\t\t\treturn cacheError(\"%q: failed to parse top tier limit annotation %q (%q): %v\",\n\t\t\t\tc.PrettyName(), ToptierLimitKey, limit, err)\n\t\t}\n\t\tc.SetToptierLimit(qty.Value())\n\t}\n\n\treturn nil\n}\n\nfunc (c *container) PrettyName() string {\n\tif c.prettyName != \"\" {\n\t\treturn c.prettyName\n\t}\n\tif pod, ok := c.GetPod(); !ok {\n\t\tc.prettyName = c.PodID + \":\" + c.Name\n\t} else {\n\t\tc.prettyName = pod.GetName() + \":\" + c.Name\n\t}\n\treturn c.prettyName\n}\n\nfunc (c *container) GetPod() (Pod, bool) {\n\tpod, found := c.cache.Pods[c.PodID]\n\treturn pod, found\n}\n\nfunc (c *container) GetID() string {\n\treturn c.ID\n}\n\nfunc (c *container) GetPodID() string {\n\treturn c.PodID\n}\n\nfunc (c *container) GetCacheID() string {\n\treturn c.CacheID\n}\n\nfunc (c *container) GetName() string {\n\treturn c.Name\n}\n\nfunc (c *container) GetNamespace() string {\n\treturn c.Namespace\n}\n\nfunc (c *container) UpdateState(state ContainerState) {\n\tc.State = state\n}\n\nfunc (c *container) GetState() ContainerState {\n\treturn c.State\n}\n\nfunc (c *container) GetQOSClass() v1.PodQOSClass {\n\tvar qos v1.PodQOSClass\n\n\tif pod, found := c.GetPod(); found {\n\t\tqos = pod.GetQOSClass()\n\t}\n\n\treturn qos\n}\n\nfunc (c *container) GetImage() string {\n\treturn c.Image\n}\n\nfunc (c *container) GetCommand() []string {\n\tcommand := make([]string, len(c.Command))\n\tcopy(command, c.Command)\n\treturn command\n}\n\nfunc (c *container) GetArgs() []string {\n\targs := make([]string, len(c.Args))\n\tcopy(args, c.Args)\n\treturn args\n}\n\nfunc keysInNamespace(m map[string]string, namespace string) []string {\n\tkeys := make([]string, 0, len(m))\n\n\tfor key := range m {\n\t\tsplit := strings.SplitN(key, \"/\", 2)\n\t\tif len(split) == 2 && split[0] == namespace {\n\t\t\tkeys = append(keys, split[1])\n\t\t} else if len(split) == 1 && len(namespace) == 0 {\n\t\t\tkeys = append(keys, split[0])\n\t\t}\n\t}\n\n\treturn keys\n}\n\nfunc (c *container) GetLabelKeys() []string {\n\tkeys := make([]string, len(c.Labels))\n\n\tidx := 0\n\tfor key := range c.Labels {\n\t\tkeys[idx] = key\n\t\tidx++\n\t}\n\n\treturn keys\n}\n\nfunc (c *container) GetLabel(key string) (string, bool) {\n\tvalue, ok := c.Labels[key]\n\treturn value, ok\n}\n\nfunc (c *container) GetResmgrLabelKeys() []string {\n\treturn keysInNamespace(c.Labels, kubernetes.ResmgrKeyNamespace)\n}\n\nfunc (c *container) GetResmgrLabel(key string) (string, bool) {\n\tvalue, ok := c.Labels[kubernetes.ResmgrKey(key)]\n\treturn value, ok\n}\n\nfunc (c *container) GetLabels() map[string]string {\n\tif c.Labels == nil {\n\t\treturn nil\n\t}\n\tlabels := make(map[string]string, len(c.Labels))\n\tfor key, value := range c.Labels {\n\t\tlabels[key] = value\n\t}\n\treturn labels\n}\n\nfunc (c *container) GetAnnotationKeys() []string {\n\tkeys := make([]string, len(c.Annotations))\n\n\tidx := 0\n\tfor key := range c.Annotations {\n\t\tkeys[idx] = key\n\t\tidx++\n\t}\n\n\treturn keys\n}\n\nfunc (c *container) GetAnnotation(key string, objPtr interface{}) (string, bool) {\n\tjsonStr, ok := c.Annotations[key]\n\tif !ok {\n\t\treturn \"\", false\n\t}\n\n\tif objPtr != nil {\n\t\tif err := json.Unmarshal([]byte(jsonStr), objPtr); err != nil {\n\t\t\tc.cache.Error(\"failed to unmarshal annotation %s (%s) of pod %s into %T\",\n\t\t\t\tkey, jsonStr, c.ID, objPtr)\n\t\t\treturn \"\", false\n\t\t}\n\t}\n\n\treturn jsonStr, true\n}\n\nfunc (c *container) GetResmgrAnnotationKeys() []string {\n\treturn keysInNamespace(c.Annotations, kubernetes.ResmgrKeyNamespace)\n}\n\nfunc (c *container) GetResmgrAnnotation(key string, objPtr interface{}) (string, bool) {\n\treturn c.GetAnnotation(kubernetes.ResmgrKey(key), objPtr)\n}\n\nfunc (c *container) GetEffectiveAnnotation(key string) (string, bool) {\n\tpod, ok := c.GetPod()\n\tif !ok {\n\t\treturn \"\", false\n\t}\n\treturn pod.GetEffectiveAnnotation(key, c.Name)\n}\n\nfunc (c *container) GetAnnotations() map[string]string {\n\tif c.Annotations == nil {\n\t\treturn nil\n\t}\n\tannotations := make(map[string]string, len(c.Annotations))\n\tfor key, value := range c.Annotations {\n\t\tannotations[key] = value\n\t}\n\treturn annotations\n}\n\nfunc (c *container) GetEnvKeys() []string {\n\tkeys := make([]string, len(c.Env))\n\n\tidx := 0\n\tfor key := range c.Env {\n\t\tkeys[idx] = key\n\t\tidx++\n\t}\n\n\treturn keys\n}\n\nfunc (c *container) GetEnv(key string) (string, bool) {\n\tvalue, ok := c.Env[key]\n\treturn value, ok\n}\n\nfunc (c *container) GetMounts() []Mount {\n\tmounts := make([]Mount, len(c.Mounts))\n\n\tidx := 0\n\tfor _, m := range c.Mounts {\n\t\tmounts[idx] = *m\n\t\tidx++\n\t}\n\n\treturn mounts\n}\n\nfunc (c *container) GetMountByHost(path string) *Mount {\n\tfor _, m := range c.Mounts {\n\t\tif m.Host == path {\n\t\t\treturn &(*m)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (c *container) GetMountByContainer(path string) *Mount {\n\tm, ok := c.Mounts[path]\n\tif !ok {\n\t\treturn nil\n\t}\n\n\treturn &(*m)\n}\n\nfunc (c *container) GetDevices() []Device {\n\tdevices := make([]Device, len(c.Devices))\n\n\tidx := 0\n\tfor _, d := range c.Devices {\n\t\tdevices[idx] = *d\n\t\tidx++\n\t}\n\n\treturn devices\n}\n\nfunc (c *container) GetDeviceByHost(path string) *Device {\n\tfor _, d := range c.Devices {\n\t\tif d.Host == path {\n\t\t\treturn &(*d)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (c *container) GetDeviceByContainer(path string) *Device {\n\td, ok := c.Devices[path]\n\tif !ok {\n\t\treturn nil\n\t}\n\n\treturn &(*d)\n}\n\nfunc (c *container) GetResourceRequirements() v1.ResourceRequirements {\n\tif adjust, _ := c.getEffectiveAdjustment(); adjust != nil {\n\t\tif resources, ok := adjust.GetResourceRequirements(); ok {\n\t\t\treturn resources\n\t\t}\n\t}\n\treturn c.Resources\n}\n\nfunc (c *container) GetLinuxResources() *criv1.LinuxContainerResources {\n\tif c.LinuxReq == nil {\n\t\treturn nil\n\t}\n\n\treturn &(*c.LinuxReq)\n}\n\nfunc (c *container) setEffectiveAdjustment(name string) string {\n\tprevious := c.Adjustment\n\tc.Adjustment = name\n\treturn previous\n}\n\nfunc (c *container) getEffectiveAdjustment() (*extapi.AdjustmentSpec, string) {\n\tif c.Adjustment == \"\" {\n\t\treturn nil, \"\"\n\t}\n\tif c.cache.External != nil {\n\t\treturn c.cache.External.Adjustments[c.Adjustment], c.Adjustment\n\t}\n\treturn nil, c.Adjustment\n}\n\nfunc (c *container) SetCommand(value []string) {\n\tc.Command = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetArgs(value []string) {\n\tc.Args = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetLabel(key, value string) {\n\tif c.Labels == nil {\n\t\tc.Labels = make(map[string]string)\n\t}\n\tc.Labels[key] = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) DeleteLabel(key string) {\n\tif _, ok := c.Labels[key]; ok {\n\t\tdelete(c.Labels, key)\n\t\tc.markPending(CRI)\n\t}\n}\n\nfunc (c *container) SetAnnotation(key, value string) {\n\tif c.Annotations == nil {\n\t\tc.Annotations = make(map[string]string)\n\t}\n\tc.Annotations[key] = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) DeleteAnnotation(key string) {\n\tif _, ok := c.Annotations[key]; ok {\n\t\tdelete(c.Annotations, key)\n\t\tc.markPending(CRI)\n\t}\n}\n\nfunc (c *container) SetEnv(key, value string) {\n\tif c.Env == nil {\n\t\tc.Env = make(map[string]string)\n\t}\n\tc.Env[key] = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) UnsetEnv(key string) {\n\tif _, ok := c.Env[key]; ok {\n\t\tdelete(c.Env, key)\n\t\tc.markPending(CRI)\n\t}\n}\n\nfunc (c *container) InsertMount(m *Mount) {\n\tif c.Mounts == nil {\n\t\tc.Mounts = make(map[string]*Mount)\n\t}\n\tc.Mounts[m.Container] = m\n\tc.markPending(CRI)\n}\n\nfunc (c *container) DeleteMount(path string) {\n\tif _, ok := c.Mounts[path]; ok {\n\t\tdelete(c.Mounts, path)\n\t\tc.markPending(CRI)\n\t}\n}\n\nfunc (c *container) InsertDevice(d *Device) {\n\tif c.Devices == nil {\n\t\tc.Devices = make(map[string]*Device)\n\t}\n\tc.Devices[d.Container] = d\n\tc.markPending(CRI)\n}\n\nfunc (c *container) DeleteDevice(path string) {\n\tif _, ok := c.Devices[path]; ok {\n\t\tdelete(c.Devices, path)\n\t\tc.markPending(CRI)\n\t}\n}\n\nfunc (c *container) GetTopologyHints() topology.Hints {\n\treturn c.TopologyHints\n}\n\nfunc (c *container) GetCPUPeriod() int64 {\n\tif c.LinuxReq == nil {\n\t\treturn 0\n\t}\n\treturn c.LinuxReq.CpuPeriod\n}\n\nfunc (c *container) GetCPUQuota() int64 {\n\tif c.LinuxReq == nil {\n\t\treturn 0\n\t}\n\treturn c.LinuxReq.CpuQuota\n}\n\nfunc (c *container) GetCPUShares() int64 {\n\tif c.LinuxReq == nil {\n\t\treturn 0\n\t}\n\treturn c.LinuxReq.CpuShares\n}\n\nfunc (c *container) GetMemoryLimit() int64 {\n\tif c.LinuxReq == nil {\n\t\treturn 0\n\t}\n\treturn c.LinuxReq.MemoryLimitInBytes\n}\n\nfunc (c *container) GetOomScoreAdj() int64 {\n\tif c.LinuxReq == nil {\n\t\treturn 0\n\t}\n\treturn c.LinuxReq.OomScoreAdj\n}\n\nfunc (c *container) GetCpusetCpus() string {\n\tif c.LinuxReq == nil {\n\t\treturn \"\"\n\t}\n\treturn c.LinuxReq.CpusetCpus\n}\n\nfunc (c *container) GetCpusetMems() string {\n\tif c.LinuxReq == nil {\n\t\treturn \"\"\n\t}\n\treturn c.LinuxReq.CpusetMems\n}\n\nfunc (c *container) SetLinuxResources(req *criv1.LinuxContainerResources) {\n\tc.LinuxReq = req\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetCPUPeriod(value int64) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.CpuPeriod = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetCPUQuota(value int64) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.CpuQuota = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetCPUShares(value int64) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.CpuShares = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetMemoryLimit(value int64) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.MemoryLimitInBytes = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetOomScoreAdj(value int64) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.OomScoreAdj = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetCpusetCpus(value string) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.CpusetCpus = value\n\tc.markPending(CRI)\n}\n\nfunc (c *container) SetCpusetMems(value string) {\n\tif c.LinuxReq == nil {\n\t\tc.LinuxReq = &criv1.LinuxContainerResources{}\n\t}\n\tc.LinuxReq.CpusetMems = value\n\tc.markPending(CRI)\n}\n\nfunc getTopologyHints(hostPath, containerPath string, readOnly bool) topology.Hints {\n\n\tif readOnly {\n\t\t// if device or path is read-only, assume it as non-important for now\n\t\t// TODO: determine topology hint, but use it with low priority\n\t\treturn topology.Hints{}\n\t}\n\n\t// ignore topology information for small files in /etc, service files in /var/lib/kubelet and host libraries mounts\n\tignoredTopologyPaths := []string{\"/.cri-resmgr\", \"/etc/\", \"/dev/termination-log\", \"/lib/\", \"/lib64/\", \"/usr/lib/\", \"/usr/lib32/\", \"/usr/lib64/\"}\n\n\tfor _, path := range ignoredTopologyPaths {\n\t\tif strings.HasPrefix(hostPath, path) || strings.HasPrefix(containerPath, path) {\n\t\t\treturn topology.Hints{}\n\t\t}\n\t}\n\n\t// More complext rules, for Kubelet secrets and config maps\n\tignoredTopologyPathRegexps := []*regexp.Regexp{\n\t\t// Kubelet directory can be different, but we can detect it by structure inside of it.\n\t\t// For now, we can safely ignore exposed config maps and secrets for topology hints.\n\t\tregexp.MustCompile(`(kubelet)?/pods/[[:xdigit:]-]+/volumes/kubernetes\\.io~(configmap|secret)/`),\n\t}\n\tfor _, re := range ignoredTopologyPathRegexps {\n\t\tif re.MatchString(hostPath) || re.MatchString(containerPath) {\n\t\t\treturn topology.Hints{}\n\t\t}\n\t}\n\n\tif devPath, err := topology.FindSysFsDevice(hostPath); err == nil {\n\t\t// errors are ignored\n\t\tif hints, err := topology.NewTopologyHints(devPath); err == nil && len(hints) > 0 {\n\t\t\treturn hints\n\t\t}\n\t}\n\n\treturn topology.Hints{}\n}\n\nfunc getKubeletHint(cpus, mems string) (ret topology.Hints) {\n\tif cpus != \"\" || mems != \"\" {\n\t\tret = topology.Hints{\n\t\t\ttopology.ProviderKubelet: topology.Hint{\n\t\t\t\tProvider: topology.ProviderKubelet,\n\t\t\t\tCPUs:     cpus,\n\t\t\t\tNUMAs:    mems}}\n\t}\n\treturn\n}\n\nfunc (c *container) GetAffinity() ([]*Affinity, error) {\n\tpod, ok := c.GetPod()\n\tif !ok {\n\t\tc.cache.Error(\"internal error: can't find Pod for container %s\", c.PrettyName())\n\t}\n\taffinity, err := pod.GetContainerAffinity(c.GetName())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\taffinity = append(affinity, c.implicitAffinities(len(affinity) > 0)...)\n\tc.cache.Debug(\"affinity for container %s:\", c.PrettyName())\n\tfor _, a := range affinity {\n\t\tc.cache.Debug(\"  - %s\", a.String())\n\t}\n\n\treturn affinity, nil\n}\n\nfunc (c *container) GetCgroupDir() string {\n\tif c.CgroupDir != \"\" {\n\t\treturn c.CgroupDir\n\t}\n\tif pod, ok := c.GetPod(); ok {\n\t\tparent, _ := pod.GetCgroupParentDir(), pod.GetID()\n\t\tID := c.GetID()\n\t\tc.CgroupDir = findContainerDir(parent, ID)\n\t}\n\treturn c.CgroupDir\n}\n\nfunc (c *container) SetRDTClass(class string) {\n\tc.RDTClass = class\n\tc.markPending(RDT)\n}\n\nfunc (c *container) GetRDTClass() string {\n\tif adjust, _ := c.getEffectiveAdjustment(); adjust != nil {\n\t\tif class, ok := adjust.GetRDTClass(); ok {\n\t\t\treturn class\n\t\t}\n\t}\n\treturn c.RDTClass\n}\n\nfunc (c *container) SetBlockIOClass(class string) {\n\tc.BlockIOClass = class\n\tc.markPending(BlockIO)\n}\n\nfunc (c *container) GetBlockIOClass() string {\n\tif adjust, _ := c.getEffectiveAdjustment(); adjust != nil {\n\t\tif class, ok := adjust.GetBlockIOClass(); ok {\n\t\t\treturn class\n\t\t}\n\t}\n\treturn c.BlockIOClass\n}\n\nfunc (c *container) SetToptierLimit(limit int64) {\n\tc.ToptierLimit = limit\n\tc.markPending(Memory)\n}\n\nfunc (c *container) GetToptierLimit() int64 {\n\tif adjust, _ := c.getEffectiveAdjustment(); adjust != nil {\n\t\tif adjust.ToptierLimit != nil {\n\t\t\treturn adjust.ToptierLimit.Value()\n\t\t}\n\t}\n\treturn c.ToptierLimit\n}\n\nfunc (c *container) SetPageMigration(p *PageMigrate) {\n\tc.PageMigrate = p\n\tc.markPending(PageMigration)\n}\n\nfunc (c *container) GetPageMigration() *PageMigrate {\n\treturn c.PageMigrate\n}\n\nfunc (c *container) GetProcesses() ([]string, error) {\n\tdir := c.GetCgroupDir()\n\tif dir == \"\" {\n\t\treturn nil, cacheError(\"%s: unknown cgroup directory\", c.PrettyName())\n\t}\n\treturn cgroups.Cpu.Group(dir).GetProcesses()\n}\n\nfunc (c *container) GetTasks() ([]string, error) {\n\tdir := c.GetCgroupDir()\n\tif dir == \"\" {\n\t\treturn nil, cacheError(\"%s: unknown cgroup directory\", c.PrettyName())\n\t}\n\treturn cgroups.Cpu.Group(dir).GetTasks()\n}\n\nfunc (c *container) SetCRIRequest(req interface{}) error {\n\tif c.req != nil {\n\t\treturn cacheError(\"can't set pending container request: another pending\")\n\t}\n\tc.req = &req\n\treturn nil\n}\n\nfunc (c *container) GetCRIRequest() (interface{}, bool) {\n\tif c.req == nil {\n\t\treturn nil, false\n\t}\n\n\treturn *c.req, true\n}\n\nfunc (c *container) ClearCRIRequest() (interface{}, bool) {\n\treq, ok := c.GetCRIRequest()\n\tc.req = nil\n\treturn req, ok\n}\n\nfunc (c *container) GetCRIEnvs() []*criv1.KeyValue {\n\tenvs := make([]*criv1.KeyValue, len(c.Env), len(c.Env))\n\tidx := 0\n\tfor k, v := range c.Env {\n\t\tenvs[idx] = &criv1.KeyValue{\n\t\t\tKey:   k,\n\t\t\tValue: v,\n\t\t}\n\t\tidx++\n\t}\n\treturn envs\n}\n\nfunc (c *container) GetCRIMounts() []*criv1.Mount {\n\tif c.Mounts == nil {\n\t\treturn nil\n\t}\n\tmounts := make([]*criv1.Mount, len(c.Mounts), len(c.Mounts))\n\tidx := 0\n\tfor _, m := range c.Mounts {\n\t\tmounts[idx] = &criv1.Mount{\n\t\t\tContainerPath:  m.Container,\n\t\t\tHostPath:       m.Host,\n\t\t\tReadonly:       m.Readonly,\n\t\t\tSelinuxRelabel: m.Relabel,\n\t\t\tPropagation:    criv1.MountPropagation(m.Propagation),\n\t\t}\n\t\tidx++\n\t}\n\treturn mounts\n}\n\nfunc (c *container) GetCRIDevices() []*criv1.Device {\n\tif c.Devices == nil {\n\t\treturn nil\n\t}\n\tdevices := make([]*criv1.Device, len(c.Devices), len(c.Devices))\n\tidx := 0\n\tfor _, d := range c.Devices {\n\t\tdevices[idx] = &criv1.Device{\n\t\t\tContainerPath: d.Container,\n\t\t\tHostPath:      d.Host,\n\t\t\tPermissions:   d.Permissions,\n\t\t}\n\t\tidx++\n\t}\n\treturn devices\n}\n\nfunc (c *container) markPending(controllers ...string) {\n\tif c.pending == nil {\n\t\tc.pending = make(map[string]struct{})\n\t}\n\tfor _, ctrl := range controllers {\n\t\tc.pending[ctrl] = struct{}{}\n\t\tc.cache.markPending(c)\n\t}\n}\n\nfunc (c *container) ClearPending(controller string) {\n\tdelete(c.pending, controller)\n\tif len(c.pending) == 0 {\n\t\tc.cache.clearPending(c)\n\t}\n}\n\nfunc (c *container) GetPending() []string {\n\tif c.pending == nil {\n\t\treturn nil\n\t}\n\tpending := make([]string, 0, len(c.pending))\n\tfor controller := range c.pending {\n\t\tpending = append(pending, controller)\n\t}\n\tsort.Strings(pending)\n\treturn pending\n}\n\nfunc (c *container) HasPending(controller string) bool {\n\tif c.pending == nil {\n\t\treturn false\n\t}\n\t_, pending := c.pending[controller]\n\treturn pending\n}\n\nfunc (c *container) GetTag(key string) (string, bool) {\n\tvalue, ok := c.Tags[key]\n\treturn value, ok\n}\n\nfunc (c *container) SetTag(key string, value string) (string, bool) {\n\tprev, ok := c.Tags[key]\n\tc.Tags[key] = value\n\treturn prev, ok\n}\n\nfunc (c *container) DeleteTag(key string) (string, bool) {\n\tvalue, ok := c.Tags[key]\n\tdelete(c.Tags, key)\n\treturn value, ok\n}\n\nfunc (c *container) implicitAffinities(hasExplicit bool) []*Affinity {\n\taffinities := []*Affinity{}\n\tfor name, generate := range c.cache.implicit {\n\t\timplicit := generate(c, hasExplicit)\n\t\tif implicit == nil {\n\t\t\tc.cache.Debug(\"no implicit affinity %s for container %s\",\n\t\t\t\tname, c.PrettyName())\n\t\t\tcontinue\n\t\t}\n\n\t\tc.cache.Debug(\"using implicit affinity %s for %s\", name, c.PrettyName())\n\t\taffinities = append(affinities, implicit)\n\t}\n\treturn affinities\n}\n\nfunc (c *container) String() string {\n\treturn c.PrettyName()\n}\n\nfunc (c *container) Eval(key string) interface{} {\n\tswitch key {\n\tcase resmgr.KeyPod:\n\t\tpod, ok := c.GetPod()\n\t\tif !ok {\n\t\t\treturn cacheError(\"%s: failed to find pod %s\", c.PrettyName(), c.PodID)\n\t\t}\n\t\treturn pod\n\tcase resmgr.KeyName:\n\t\treturn c.Name\n\tcase resmgr.KeyNamespace:\n\t\treturn c.Namespace\n\tcase resmgr.KeyQOSClass:\n\t\treturn c.GetQOSClass()\n\tcase resmgr.KeyLabels:\n\t\treturn c.Labels\n\tcase resmgr.KeyTags:\n\t\treturn c.Tags\n\tcase resmgr.KeyID:\n\t\treturn c.ID\n\tdefault:\n\t\treturn cacheError(\"%s: Container cannot evaluate of %q\", c.PrettyName(), key)\n\t}\n}\n\n// CompareContainersFn compares two containers by some arbitrary property.\n// It returns a negative integer, 0, or a positive integer, depending on\n// whether the first container is considered smaller, equal, or larger than\n// the second.\ntype CompareContainersFn func(Container, Container) int\n\n// SortContainers sorts a slice of containers using the given comparison functions.\n// If the containers are otherwise equal they are sorted by pod and container name.\n// If the comparison functions are omitted, containers are compared by QoS class,\n// memory and cpuset size.\nfunc SortContainers(containers []Container, compareFns ...CompareContainersFn) {\n\tif len(compareFns) == 0 {\n\t\tcompareFns = CompareByQOSMemoryCPU\n\t}\n\tsort.Slice(containers, func(i, j int) bool {\n\t\tci, cj := containers[i], containers[j]\n\t\tfor _, cmpFn := range compareFns {\n\t\t\tswitch diff := cmpFn(ci, cj); {\n\t\t\tcase diff < 0:\n\t\t\t\treturn true\n\t\t\tcase diff > 0:\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\t// If two containers are otherwise equal they are sorted by pod and container name.\n\t\tif pi, ok := ci.GetPod(); ok {\n\t\t\tif pj, ok := cj.GetPod(); ok {\n\t\t\t\tni, nj := pi.GetName(), pj.GetName()\n\t\t\t\tif ni != nj {\n\t\t\t\t\treturn ni < nj\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn ci.GetName() < cj.GetName()\n\t})\n}\n\n// CompareByQOSMemoryCPU is a slice for comparing container by QOS, memory, and CPU.\nvar CompareByQOSMemoryCPU = []CompareContainersFn{CompareQOS, CompareMemory, CompareCPU}\n\n// CompareQOS compares containers by QOS class.\nfunc CompareQOS(ci, cj Container) int {\n\tqosi, qosj := ci.GetQOSClass(), cj.GetQOSClass()\n\tswitch {\n\tcase qosi == v1.PodQOSGuaranteed && qosj != v1.PodQOSGuaranteed:\n\t\treturn -1\n\tcase qosj == v1.PodQOSGuaranteed && qosi != v1.PodQOSGuaranteed:\n\t\treturn +1\n\tcase qosi == v1.PodQOSBurstable && qosj == v1.PodQOSBestEffort:\n\t\treturn -1\n\tcase qosj == v1.PodQOSBurstable && qosi == v1.PodQOSBestEffort:\n\t\treturn +1\n\t}\n\treturn 0\n}\n\n// CompareMemory compares containers by memory requests and limits.\nfunc CompareMemory(ci, cj Container) int {\n\tvar reqi, limi, reqj, limj int64\n\n\tresi := ci.GetResourceRequirements()\n\tif qty, ok := resi.Requests[v1.ResourceMemory]; ok {\n\t\treqi = qty.Value()\n\t}\n\tif qty, ok := resi.Limits[v1.ResourceMemory]; ok {\n\t\tlimi = qty.Value()\n\t}\n\tresj := cj.GetResourceRequirements()\n\tif qty, ok := resj.Requests[v1.ResourceMemory]; ok {\n\t\treqj = qty.Value()\n\t}\n\tif qty, ok := resj.Limits[v1.ResourceMemory]; ok {\n\t\tlimj = qty.Value()\n\t}\n\n\tswitch diff := reqj - reqi; {\n\tcase diff < 0:\n\t\treturn -1\n\tcase diff > 0:\n\t\treturn +1\n\t}\n\tswitch diff := limj - limi; {\n\tcase diff < 0:\n\t\treturn -1\n\tcase diff > 0:\n\t\treturn +1\n\t}\n\treturn 0\n}\n\n// CompareCPU compares containers by CPU requests and limits.\nfunc CompareCPU(ci, cj Container) int {\n\tvar reqi, limi, reqj, limj int64\n\n\tresi := ci.GetResourceRequirements()\n\tif qty, ok := resi.Requests[v1.ResourceCPU]; ok {\n\t\treqi = qty.MilliValue()\n\t}\n\tif qty, ok := resi.Limits[v1.ResourceCPU]; ok {\n\t\tlimi = qty.MilliValue()\n\t}\n\tresj := cj.GetResourceRequirements()\n\tif qty, ok := resj.Requests[v1.ResourceCPU]; ok {\n\t\treqj = qty.MilliValue()\n\t}\n\tif qty, ok := resj.Limits[v1.ResourceCPU]; ok {\n\t\tlimj = qty.MilliValue()\n\t}\n\n\tswitch diff := reqj - reqi; {\n\tcase diff < 0:\n\t\treturn -1\n\tcase diff > 0:\n\t\treturn +1\n\t}\n\tswitch diff := limj - limi; {\n\tcase diff < 0:\n\t\treturn -1\n\tcase diff > 0:\n\t\treturn +1\n\t}\n\treturn 0\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/container_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\t\"github.com/google/go-cmp/cmp/cmpopts\"\n)\n\nfunc TestGetKubeletHint(t *testing.T) {\n\ttype T struct {\n\t\tname        string\n\t\tcpus        string\n\t\tmems        string\n\t\texpectedLen int\n\t}\n\n\tcases := []T{\n\t\t{\n\t\t\tname:        \"empty\",\n\t\t\tcpus:        \"\",\n\t\t\tmems:        \"\",\n\t\t\texpectedLen: 0,\n\t\t},\n\t\t{\n\t\t\tname:        \"cpus\",\n\t\t\tcpus:        \"0-9\",\n\t\t\tmems:        \"\",\n\t\t\texpectedLen: 1,\n\t\t},\n\t\t{\n\t\t\tname:        \"mems\",\n\t\t\tcpus:        \"\",\n\t\t\tmems:        \"0,1\",\n\t\t\texpectedLen: 1,\n\t\t},\n\t\t{\n\t\t\tname:        \"both\",\n\t\t\tcpus:        \"0-9\",\n\t\t\tmems:        \"0,1\",\n\t\t\texpectedLen: 1,\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\toutput := getKubeletHint(tc.cpus, tc.mems)\n\t\t\tif len(output) != tc.expectedLen {\n\t\t\t\tt.Errorf(\"expected len of hints: %d, got: %d, hints: %+v\", tc.expectedLen, len(output), output)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestGetTopologyHints(t *testing.T) {\n\ttype T struct {\n\t\tname          string\n\t\thostPath      string\n\t\tcontainerPath string\n\t\treadOnly      bool\n\t\texpectedLen   int\n\t}\n\n\tcases := []T{\n\t\t{\n\t\t\tname:          \"read-only\",\n\t\t\thostPath:      \"/something\",\n\t\t\tcontainerPath: \"/something\",\n\t\t\treadOnly:      true,\n\t\t},\n\t\t{\n\t\t\tname:          \"host /etc\",\n\t\t\thostPath:      \"/etc/something\",\n\t\t\tcontainerPath: \"/data/something\",\n\t\t},\n\t\t{\n\t\t\tname:          \"container /etc\",\n\t\t\thostPath:      \"/var/lib/kubelet/pods/0c9bcfc4-c51b-11e9-ac9a-b8aeed7c7427/etc-hosts\",\n\t\t\tcontainerPath: \"/etc/hosts\",\n\t\t},\n\t\t{\n\t\t\tname:          \"ConfigMap\",\n\t\t\tcontainerPath: \"/var/lib/kube-proxy\",\n\t\t\thostPath:      \"/var/lib/kubelet/pods/0c9bcfc4-c51b-11e9-ac9a-b8aeed7c7427/volumes/kubernetes.io~configmap/kube-proxy\",\n\t\t},\n\t\t{\n\t\t\tname:          \"secret\",\n\t\t\tcontainerPath: \"/var/run/secrets/kubernetes.io/serviceaccount\",\n\t\t\thostPath:      \"/var/lib/kubelet/pods/0c9bcfc4-c51b-11e9-ac9a-b8aeed7c7427/volumes/kubernetes.io~secret/kube-proxy-token-d9slz\",\n\t\t},\n\t\t{\n\t\t\tname:          \"dev null\",\n\t\t\thostPath:      \"/dev/null\",\n\t\t\tcontainerPath: \"/dev/null\",\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\toutput := getTopologyHints(tc.hostPath, tc.containerPath, tc.readOnly)\n\t\t\tif len(output) != tc.expectedLen {\n\t\t\t\tt.Errorf(\"expected len of hints: %d, got: %d, hints: %+v\", tc.expectedLen, len(output), output)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestKeysInNamespace(t *testing.T) {\n\ttestMap := map[string]string{\n\t\t\"no-namespace\":               \"\",\n\t\t\"my.name.space\":              \"\",\n\t\t\"my.name.space/key-1\":        \"\",\n\t\t\"my.name.space/key-2\":        \"\",\n\t\t\"other.name.space/other-key\": \"\",\n\t}\n\ttcases := []struct {\n\t\tname          string\n\t\tcollectionMap map[string]string\n\t\tnamespace     string\n\t\texpectedKeys  []string\n\t}{\n\t\t{\n\t\t\tname: \"empty map should return nothing for empty namespace\",\n\t\t},\n\t\t{\n\t\t\tname:      \"empty map should return nothing\",\n\t\t\tnamespace: \"my.name.space\",\n\t\t},\n\t\t{\n\t\t\tname:          \"keys with no namespace\",\n\t\t\tcollectionMap: testMap,\n\t\t\texpectedKeys:  []string{\"my.name.space\", \"no-namespace\"},\n\t\t},\n\t\t{\n\t\t\tname:          \"keys in namespace\",\n\t\t\tcollectionMap: testMap,\n\t\t\tnamespace:     \"my.name.space\",\n\t\t\texpectedKeys:  []string{\"key-1\", \"key-2\"},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tkeys := keysInNamespace(tc.collectionMap, tc.namespace)\n\t\t\tsort.Strings(keys)\n\t\t\tif !cmp.Equal(keys, tc.expectedKeys, cmpopts.EquateEmpty()) {\n\t\t\t\tt.Errorf(\"Expected %v, received %v\", tc.expectedKeys, keys)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/error.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"fmt\"\n)\n\nfunc cacheError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"cache: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/pod.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"encoding/json\"\n\t\"strconv\"\n\t\"strings\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n)\n\nconst (\n\t// KeyResourceAnnotation is the annotation key our webhook uses.\n\tKeyResourceAnnotation = \"intel.com/resources\"\n)\n\n// Create a pod from a run request.\nfunc (p *pod) fromRunRequest(req *criv1.RunPodSandboxRequest) error {\n\tcfg := req.Config\n\tif cfg == nil {\n\t\treturn cacheError(\"pod %s has no config\", p.ID)\n\t}\n\tmeta := cfg.Metadata\n\tif meta == nil {\n\t\treturn cacheError(\"pod %s has no request metadata\", p.ID)\n\t}\n\n\tp.containers = make(map[string]string)\n\tp.UID = meta.Uid\n\tp.Name = meta.Name\n\tp.Namespace = meta.Namespace\n\tp.State = PodState(int32(PodStateReady))\n\tp.Labels = cfg.Labels\n\tp.Annotations = cfg.Annotations\n\tp.CgroupParent = cfg.GetLinux().GetCgroupParent()\n\n\tif err := p.discoverQOSClass(); err != nil {\n\t\tp.cache.Error(\"%v\", err)\n\t}\n\n\tp.parseResourceAnnotations()\n\n\treturn nil\n}\n\n// Create a pod from a list response.\nfunc (p *pod) fromListResponse(pod *criv1.PodSandbox, status *PodStatus) error {\n\tmeta := pod.Metadata\n\tif meta == nil {\n\t\treturn cacheError(\"pod %s has no reply metadata\", p.ID)\n\t}\n\n\tp.containers = make(map[string]string)\n\tp.UID = meta.Uid\n\tp.Name = meta.Name\n\tp.Namespace = meta.Namespace\n\tp.State = PodState(int32(pod.State))\n\tp.Labels = pod.Labels\n\tp.Annotations = pod.Annotations\n\n\tif status == nil {\n\t\tp.cache.Error(\"pod %s has no associated status query data\", p.ID)\n\t} else {\n\t\tp.CgroupParent = status.CgroupParent\n\t}\n\n\tif err := p.discoverQOSClass(); err != nil {\n\t\tp.cache.Error(\"%v\", err)\n\t}\n\n\tp.parseResourceAnnotations()\n\n\treturn nil\n}\n\n// Get the init containers of a pod.\nfunc (p *pod) GetInitContainers() []Container {\n\tif p.Resources == nil {\n\t\treturn nil\n\t}\n\n\tcontainers := []Container{}\n\n\tfor id, c := range p.cache.Containers {\n\t\tif id != c.CacheID {\n\t\t\tcontinue\n\t\t}\n\t\tif _, ok := p.Resources.InitContainers[c.ID]; ok {\n\t\t\tcontainers = append(containers, c)\n\t\t}\n\t}\n\n\treturn containers\n}\n\n// Get the normal containers of a pod.\nfunc (p *pod) GetContainers() []Container {\n\tcontainers := []Container{}\n\n\tfor id, c := range p.cache.Containers {\n\t\tif c.PodID != p.ID || id != c.CacheID {\n\t\t\tcontinue\n\t\t}\n\t\tif p.Resources != nil {\n\t\t\tif _, ok := p.Resources.InitContainers[c.ID]; ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tcontainers = append(containers, c)\n\t}\n\n\treturn containers\n}\n\n// Get container pointer by its name.\nfunc (p *pod) getContainer(name string) *container {\n\tvar found *container\n\n\tif id, ok := p.containers[name]; ok {\n\t\treturn p.cache.Containers[id]\n\t}\n\n\tfor _, c := range p.GetContainers() {\n\t\tcptr := c.(*container)\n\t\tp.containers[cptr.Name] = cptr.ID\n\t\tif cptr.Name == name {\n\t\t\tfound = cptr\n\t\t}\n\t}\n\n\treturn found\n}\n\n// Get container by its name.\nfunc (p *pod) GetContainer(name string) (Container, bool) {\n\tc := p.getContainer(name)\n\n\treturn c, c != nil\n}\n\n// Get the id of a pod.\nfunc (p *pod) GetID() string {\n\treturn p.ID\n}\n\n// Get the (k8s) unique id of a pod.\nfunc (p *pod) GetUID() string {\n\treturn p.UID\n}\n\n// Get the name of a pod.\nfunc (p *pod) GetName() string {\n\treturn p.Name\n}\n\n// Get the namespace of a pod.\nfunc (p *pod) GetNamespace() string {\n\treturn p.Namespace\n}\n\n// Get the PodState of a pod.\nfunc (p *pod) GetState() PodState {\n\treturn p.State\n}\n\n// Get the keys of all labels of a pod.\nfunc (p *pod) GetLabelKeys() []string {\n\tkeys := make([]string, len(p.Labels))\n\n\tidx := 0\n\tfor key := range p.Labels {\n\t\tkeys[idx] = key\n\t\tidx++\n\t}\n\n\treturn keys\n}\n\n// Get the label for a key of a pod.\nfunc (p *pod) GetLabel(key string) (string, bool) {\n\tvalue, ok := p.Labels[key]\n\treturn value, ok\n}\n\n// Get all label keys in the cri-resource-manager namespace.\nfunc (p *pod) GetResmgrLabelKeys() []string {\n\treturn keysInNamespace(p.Labels, kubernetes.ResmgrKeyNamespace)\n}\n\n// Get the label for the given key in the cri-resource-manager namespace.\nfunc (p *pod) GetResmgrLabel(key string) (string, bool) {\n\tvalue, ok := p.Labels[kubernetes.ResmgrKey(key)]\n\treturn value, ok\n}\n\n// Get the keys of all annotations of a pod.\nfunc (p *pod) GetAnnotationKeys() []string {\n\tkeys := make([]string, len(p.Annotations))\n\n\tidx := 0\n\tfor key := range p.Annotations {\n\t\tkeys[idx] = key\n\t\tidx++\n\t}\n\n\treturn keys\n}\n\n// Get pod annotation for the given key.\nfunc (p *pod) GetAnnotation(key string) (string, bool) {\n\tvalue, ok := p.Annotations[key]\n\treturn value, ok\n}\n\n// Get and decode/unmarshal pod annotation for the given key.\nfunc (p *pod) GetAnnotationObject(key string, objPtr interface{},\n\tdecode func([]byte, interface{}) error) (bool, error) {\n\tvar err error\n\n\tvalue, ok := p.GetAnnotation(key)\n\tif !ok {\n\t\treturn false, nil\n\t}\n\n\t// decode with decoder function, if given\n\tif decode != nil {\n\t\terr = decode([]byte(value), objPtr)\n\t\treturn true, err\n\t}\n\n\t// decode with type-specific default decoder\n\tswitch objPtr.(type) {\n\tcase *string:\n\t\t*objPtr.(*string) = value\n\tcase *bool:\n\t\t*objPtr.(*bool), err = strconv.ParseBool(value)\n\tcase *int:\n\t\tvar i int64\n\t\ti, err = strconv.ParseInt(value, 0, 0)\n\t\t*objPtr.(*int) = int(i)\n\tcase *uint:\n\t\tvar i uint64\n\t\ti, err = strconv.ParseUint(value, 0, 0)\n\t\t*objPtr.(*uint) = uint(i)\n\tcase *int64:\n\t\t*objPtr.(*int64), err = strconv.ParseInt(value, 0, 64)\n\tcase *uint64:\n\t\t*objPtr.(*uint64), err = strconv.ParseUint(value, 0, 64)\n\tdefault:\n\t\terr = json.Unmarshal([]byte(value), objPtr)\n\t}\n\n\tif err != nil {\n\t\tp.cache.Error(\"failed to decode annotation %s (%s): %v\", key, value, err)\n\t}\n\n\treturn true, err\n}\n\n// Get the keys of all annotation in the cri-resource-manager namespace.\nfunc (p *pod) GetResmgrAnnotationKeys() []string {\n\treturn keysInNamespace(p.Annotations, kubernetes.ResmgrKeyNamespace)\n}\n\n// Get the value of the given annotation in the cri-resource-manager namespace.\nfunc (p *pod) GetResmgrAnnotation(key string) (string, bool) {\n\treturn p.GetAnnotation(kubernetes.ResmgrKey(key))\n}\n\n// Get and decode the pod annotation for the key in the cri-resource-manager namespace..\nfunc (p *pod) GetResmgrAnnotationObject(key string, objPtr interface{},\n\tdecode func([]byte, interface{}) error) (bool, error) {\n\treturn p.GetAnnotationObject(kubernetes.ResmgrKey(key), objPtr, decode)\n}\n\n// Get the effective annotation for the container.\nfunc (p *pod) GetEffectiveAnnotation(key, container string) (string, bool) {\n\tif v, ok := p.Annotations[key+\"/container.\"+container]; ok {\n\t\treturn v, true\n\t}\n\tif v, ok := p.Annotations[key+\"/pod\"]; ok {\n\t\treturn v, true\n\t}\n\tv, ok := p.Annotations[key]\n\treturn v, ok\n}\n\n// Get the cgroup parent directory of a pod, if known.\nfunc (p *pod) GetCgroupParentDir() string {\n\treturn p.CgroupParent\n}\n\n// discover a pod's QoS class by parsing the cgroup parent directory.\nfunc (p *pod) discoverQOSClass() error {\n\tif p.CgroupParent == \"\" {\n\t\tp.QOSClass = v1.PodQOSBestEffort\n\t\treturn cacheError(\"%s: unknown cgroup parent/QoS class\", p.ID)\n\t}\n\n\tdirs := strings.Split(p.CgroupParent[1:], \"/\")\n\tif len(dirs) < 1 {\n\t\treturn cacheError(\"%s: failed to parse %q for QoS class\",\n\t\t\tp.ID, p.CgroupParent)\n\n\t}\n\n\t// consume any potential --cgroup-root passed to kubelet\n\tif dirs[0] != \"kubepods.slice\" && dirs[0] != \"kubepods\" {\n\t\tdirs = dirs[1:]\n\t}\n\tif len(dirs) < 1 {\n\t\treturn cacheError(\"%s: failed to parse %q for QoS class\",\n\t\t\tp.ID, p.CgroupParent)\n\t}\n\n\t// consume potential kubepods[.slice]\n\tif dirs[0] == \"kubepods.slice\" || dirs[0] == \"kubepods\" {\n\t\tdirs = dirs[1:]\n\t}\n\tif len(dirs) < 1 {\n\t\treturn cacheError(\"%s: failed to parse %q for QoS class\",\n\t\t\tp.ID, p.CgroupParent)\n\t}\n\n\t// check for besteffort, burstable, or lack thereof indicating guaranteed\n\tswitch dir := dirs[0]; {\n\tcase dir == \"kubepods-besteffort.slice\" || dir == \"besteffort\":\n\t\tp.QOSClass = v1.PodQOSBestEffort\n\t\treturn nil\n\tcase dir == \"kubepods-burstable.slice\" || dir == \"burstable\":\n\t\tp.QOSClass = v1.PodQOSBurstable\n\t\treturn nil\n\tcase strings.HasPrefix(dir, \"kubepods-pod\") || strings.HasPrefix(dir, \"pod\"):\n\t\tp.QOSClass = v1.PodQOSGuaranteed\n\t\treturn nil\n\t}\n\n\treturn cacheError(\"%s: failed to parse %q for QoS class\",\n\t\tp.ID, p.CgroupParent)\n}\n\n// Get the resource requirements of a pod.\nfunc (p *pod) GetPodResourceRequirements() PodResourceRequirements {\n\tif p.Resources == nil {\n\t\treturn PodResourceRequirements{}\n\t}\n\n\treturn *p.Resources\n}\n\n// Parse per container resource requirements from webhook annotations.\nfunc (p *pod) parseResourceAnnotations() {\n\tp.Resources = &PodResourceRequirements{}\n\tp.GetAnnotationObject(KeyResourceAnnotation, p.Resources, nil)\n}\n\n// Determine the QoS class of the pod.\nfunc (p *pod) GetQOSClass() v1.PodQOSClass {\n\treturn p.QOSClass\n}\n\n// GetContainerAffinity returns the annotated affinity for the named container.\nfunc (p *pod) GetContainerAffinity(name string) ([]*Affinity, error) {\n\tif p.Affinity != nil {\n\t\treturn (*p.Affinity)[name], nil\n\t}\n\n\taffinity := &podContainerAffinity{}\n\n\tvalue, ok := p.GetResmgrAnnotation(keyAffinity)\n\tif ok {\n\t\tweight := DefaultWeight\n\t\tif !affinity.parseSimple(p, value, weight) {\n\t\t\tif err := affinity.parseFull(p, value, weight); err != nil {\n\t\t\t\tp.cache.Error(\"%v\", err)\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\tvalue, ok = p.GetResmgrAnnotation(keyAntiAffinity)\n\tif ok {\n\t\tweight := -DefaultWeight\n\t\tif !affinity.parseSimple(p, value, weight) {\n\t\t\tif err := affinity.parseFull(p, value, weight); err != nil {\n\t\t\t\tp.cache.Error(\"%v\", err)\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif p.cache.DebugEnabled() {\n\t\tp.cache.Debug(\"Pod container affinity for %s:\", p.GetName())\n\t\tfor id, ca := range *affinity {\n\t\t\tp.cache.Debug(\"  - container %s:\", id)\n\t\t\tfor _, a := range ca {\n\t\t\t\tp.cache.Debug(\"    * %s\", a.String())\n\t\t\t}\n\t\t}\n\t}\n\n\tp.Affinity = affinity\n\n\treturn (*p.Affinity)[name], nil\n}\n\n// ScopeExpression returns an affinity expression for defining this pod as the scope.\nfunc (p *pod) ScopeExpression() *resmgr.Expression {\n\treturn &resmgr.Expression{\n\t\t//      Domain: LabelsDomain,\n\t\tKey:    kubernetes.PodNameLabel,\n\t\tOp:     resmgr.Equals,\n\t\tValues: []string{p.GetName()},\n\t}\n}\n\n// String returns a string representation of pod.\nfunc (p *pod) String() string {\n\treturn p.Name\n}\n\n// Eval returns the value of a key for expression evaluation.\nfunc (p *pod) Eval(key string) interface{} {\n\tswitch key {\n\tcase resmgr.KeyName:\n\t\treturn p.Name\n\tcase resmgr.KeyNamespace:\n\t\treturn p.Namespace\n\tcase resmgr.KeyQOSClass:\n\t\treturn p.GetQOSClass()\n\tcase resmgr.KeyLabels:\n\t\treturn p.Labels\n\tcase resmgr.KeyID:\n\t\treturn p.ID\n\tcase resmgr.KeyUID:\n\t\treturn p.UID\n\tdefault:\n\t\treturn cacheError(\"Pod cannot evaluate of %q\", key)\n\t}\n}\n\n// GetProcesses returns the pids of processes in a pod.\nfunc (p *pod) GetProcesses(recursive bool) ([]string, error) {\n\treturn p.getTasks(recursive, true)\n}\n\n// GetTasks returns the pids of threads in a pod.\nfunc (p *pod) GetTasks(recursive bool) ([]string, error) {\n\treturn p.getTasks(recursive, false)\n}\n\n// getTasks returns the pids of processes or threads in a pod.\nfunc (p *pod) getTasks(recursive, processes bool) ([]string, error) {\n\tvar pids, childPids []string\n\tvar err error\n\n\tdir := p.GetCgroupParentDir()\n\tif dir == \"\" {\n\t\treturn nil, cacheError(\"%s: unknown cgroup parent directory\", p.Name)\n\t}\n\n\tif processes {\n\t\tpids, err = cgroups.Cpu.Group(dir).GetProcesses()\n\t} else {\n\t\tpids, err = cgroups.Cpu.Group(dir).GetTasks()\n\t}\n\tif err != nil {\n\t\treturn nil, cacheError(\"%s: failed to read pids: %v\", p.Name, err)\n\t}\n\n\tif !recursive {\n\t\treturn pids, nil\n\t}\n\n\tfor _, c := range append(p.GetInitContainers(), p.GetContainers()...) {\n\t\tif c.GetState() == ContainerStateRunning {\n\t\t\tif processes {\n\t\t\t\tchildPids, err = c.GetProcesses()\n\t\t\t} else {\n\t\t\t\tchildPids, err = c.GetTasks()\n\t\t\t}\n\t\t\tif err == nil {\n\t\t\t\tpids = append(pids, childPids...)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tp.cache.Error(\"%s: failed to read pids of %s: %v\", p.Name,\n\t\t\t\tc.PrettyName(), err)\n\t\t}\n\t}\n\n\treturn pids, nil\n}\n\n// ParsePodStatus parses a PodSandboxStatusResponse into a PodStatus.\nfunc ParsePodStatus(response *criv1.PodSandboxStatusResponse) (*PodStatus, error) {\n\tvar name string\n\n\ttype infoRuntimeSpec struct {\n\t\tAnnotations map[string]string `json:\"annotations\"`\n\t}\n\ttype infoConfig struct {\n\t\tLinux *struct {\n\t\t\tCgroupParent string `json:\"cgroup_parent\"`\n\t\t} `json:\"linux\"`\n\t}\n\ttype statusInfo struct {\n\t\tRuntimeSpec *infoRuntimeSpec `json:\"runtimeSpec\"`\n\t\tConfig      *infoConfig      `json:\"config\"`\n\t}\n\n\tif response.Status.Metadata != nil {\n\t\tname = response.Status.Metadata.Name\n\t} else {\n\t\tname = response.Status.Id\n\t}\n\n\tblob, ok := response.Info[\"info\"]\n\tif !ok {\n\t\treturn nil, cacheError(\"%s: missing info in pod status response\", name)\n\t}\n\tinfo := statusInfo{}\n\tif err := json.Unmarshal([]byte(blob), &info); err != nil {\n\t\treturn nil, cacheError(\"%s: failed to extract pod status info: %v\",\n\t\t\tname, err)\n\t}\n\n\tps := &PodStatus{}\n\n\tif info.Config != nil { // containerd\n\t\t// CgroupParent: Info[\"config\"][\"linux\"][\"cgroup_parent\"]\n\t\tps.CgroupParent = info.Config.Linux.CgroupParent\n\t} else if info.RuntimeSpec != nil { // cri-o\n\t\t// CgroupParent: Info[\"info\"][\"runtimeSpec\"][\"annotations\"][crioCgroupParent]\n\t\tconst (\n\t\t\tcrioCgroupParent = \"io.kubernetes.cri-o.CgroupParent\"\n\t\t)\n\n\t\tps.CgroupParent = info.RuntimeSpec.Annotations[crioCgroupParent]\n\t}\n\n\tif ps.CgroupParent == \"\" {\n\t\treturn nil, cacheError(\"%s: failed to extract cgroup parent from pod status\",\n\t\t\tname)\n\t}\n\n\treturn ps, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/cache/utils.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cache\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"strconv\"\n\t\"strings\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\t\"k8s.io/apimachinery/pkg/util/sets\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n)\n\nvar (\n\tmemoryCapacity   int64\n\tSharesToMilliCPU = kubernetes.SharesToMilliCPU\n\tQuotaToMilliCPU  = kubernetes.QuotaToMilliCPU\n\tMilliCPUToShares = kubernetes.MilliCPUToShares\n\tMilliCPUToQuota  = kubernetes.MilliCPUToQuota\n)\n\n// IsPodQOSClassName returns true if the given class is one of the Pod QOS classes.\nfunc IsPodQOSClassName(class string) bool {\n\tswitch corev1.PodQOSClass(class) {\n\tcase corev1.PodQOSBestEffort, corev1.PodQOSBurstable, corev1.PodQOSGuaranteed:\n\t\treturn true\n\t}\n\treturn false\n}\n\n// estimateComputeResources calculates resource requests/limits from a CRI request.\nfunc estimateComputeResources(lnx *criv1.LinuxContainerResources, cgroupParent string) corev1.ResourceRequirements {\n\tvar qos corev1.PodQOSClass\n\n\tresources := corev1.ResourceRequirements{\n\t\tRequests: corev1.ResourceList{},\n\t\tLimits:   corev1.ResourceList{},\n\t}\n\n\tif lnx == nil {\n\t\treturn resources\n\t}\n\n\tif cgroupParent != \"\" {\n\t\tqos = cgroupParentToQOS(cgroupParent)\n\t}\n\n\t// calculate CPU request\n\tif value := SharesToMilliCPU(lnx.CpuShares); value > 0 {\n\t\tqty := resapi.NewMilliQuantity(value, resapi.DecimalSI)\n\t\tresources.Requests[corev1.ResourceCPU] = *qty\n\t}\n\n\t// get memory limit\n\tif value := lnx.MemoryLimitInBytes; value > 0 {\n\t\tqty := resapi.NewQuantity(value, resapi.DecimalSI)\n\t\tresources.Limits[corev1.ResourceMemory] = *qty\n\t}\n\n\t// set or calculate CPU limit, set memory request if known\n\tif qos == corev1.PodQOSGuaranteed {\n\t\tresources.Limits[corev1.ResourceCPU] = resources.Requests[corev1.ResourceCPU]\n\t\tresources.Requests[corev1.ResourceMemory] = resources.Limits[corev1.ResourceMemory]\n\t} else {\n\t\tif value := QuotaToMilliCPU(lnx.CpuQuota, lnx.CpuPeriod); value > 0 {\n\t\t\tqty := resapi.NewMilliQuantity(value, resapi.DecimalSI)\n\t\t\tresources.Limits[corev1.ResourceCPU] = *qty\n\t\t}\n\t}\n\n\treturn resources\n}\n\n// getMemoryCapacity parses memory capacity from /proc/meminfo (mimicking cAdvisor).\nfunc getMemoryCapacity() int64 {\n\tvar data []byte\n\tvar err error\n\n\tif memoryCapacity > 0 {\n\t\treturn memoryCapacity\n\t}\n\n\tif data, err = os.ReadFile(\"/proc/meminfo\"); err != nil {\n\t\treturn -1\n\t}\n\n\tfor _, line := range strings.Split(string(data), \"\\n\") {\n\t\tkeyval := strings.Split(line, \":\")\n\t\tif len(keyval) != 2 || keyval[0] != \"MemTotal\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tvalunit := strings.Split(strings.TrimSpace(keyval[1]), \" \")\n\t\tif len(valunit) != 2 || valunit[1] != \"kB\" {\n\t\t\treturn -1\n\t\t}\n\n\t\tmemoryCapacity, err = strconv.ParseInt(valunit[0], 10, 64)\n\t\tif err != nil {\n\t\t\treturn -1\n\t\t}\n\n\t\tmemoryCapacity *= 1024\n\t\tbreak\n\t}\n\n\treturn memoryCapacity\n}\n\n// cgroupParentToQOS tries to map Pod cgroup parent to QOS class.\nfunc cgroupParentToQOS(dir string) corev1.PodQOSClass {\n\tvar qos corev1.PodQOSClass\n\n\t// The parent directory naming scheme depends on the cgroup driver in use.\n\t// Thus, rely on substring matching\n\tsplit := strings.Split(strings.TrimPrefix(dir, \"/\"), \"/\")\n\tswitch {\n\tcase len(split) < 2:\n\t\tqos = corev1.PodQOSClass(\"\")\n\tcase strings.Index(split[1], strings.ToLower(string(corev1.PodQOSBurstable))) != -1:\n\t\tqos = corev1.PodQOSBurstable\n\tcase strings.Index(split[1], strings.ToLower(string(corev1.PodQOSBestEffort))) != -1:\n\t\tqos = corev1.PodQOSBestEffort\n\tdefault:\n\t\tqos = corev1.PodQOSGuaranteed\n\t}\n\n\treturn qos\n}\n\n// resourcesToQOS tries to map Pod container resources (from annotation) to QOS class.\nfunc resourcesToQOS(podResources *PodResourceRequirements) corev1.PodQOSClass {\n\tvar qos corev1.PodQOSClass\n\n\tif podResources == nil {\n\t\treturn qos\n\t}\n\n\trequests := corev1.ResourceList{}\n\tlimits := corev1.ResourceList{}\n\tzeroQuantity := resapi.MustParse(\"0\")\n\tisGuaranteed := true\n\tfor _, resources := range podResources.Containers {\n\t\t// process requests\n\t\tfor name, quantity := range resources.Requests {\n\t\t\tif !isSupportedQoSComputeResource(name) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif quantity.Cmp(zeroQuantity) == 1 {\n\t\t\t\tdelta := quantity.DeepCopy()\n\t\t\t\tif _, exists := requests[name]; !exists {\n\t\t\t\t\trequests[name] = delta\n\t\t\t\t} else {\n\t\t\t\t\tdelta.Add(requests[name])\n\t\t\t\t\trequests[name] = delta\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// process limits\n\t\tqosLimitsFound := sets.NewString()\n\t\tfor name, quantity := range resources.Limits {\n\t\t\tif !isSupportedQoSComputeResource(name) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif quantity.Cmp(zeroQuantity) == 1 {\n\t\t\t\tqosLimitsFound.Insert(string(name))\n\t\t\t\tdelta := quantity.DeepCopy()\n\t\t\t\tif _, exists := limits[name]; !exists {\n\t\t\t\t\tlimits[name] = delta\n\t\t\t\t} else {\n\t\t\t\t\tdelta.Add(limits[name])\n\t\t\t\t\tlimits[name] = delta\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif !qosLimitsFound.HasAll(string(corev1.ResourceMemory), string(corev1.ResourceCPU)) {\n\t\t\tisGuaranteed = false\n\t\t}\n\t}\n\tif len(requests) == 0 && len(limits) == 0 {\n\t\treturn corev1.PodQOSBestEffort\n\t}\n\t// Check is requests match limits for all resources.\n\tif isGuaranteed {\n\t\tfor name, req := range requests {\n\t\t\tif lim, exists := limits[name]; !exists || lim.Cmp(req) != 0 {\n\t\t\t\tisGuaranteed = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\tif isGuaranteed &&\n\t\tlen(requests) == len(limits) {\n\t\treturn corev1.PodQOSGuaranteed\n\t}\n\treturn corev1.PodQOSBurstable\n}\n\n// findContainerDir brute-force searches for a container cgroup dir.\nfunc findContainerDir(podCgroupDir, ID string) string {\n\tvar dirs []string\n\n\tif podCgroupDir == \"\" {\n\t\treturn \"\"\n\t}\n\n\tcpusetDir := cgroups.Cpuset.Path()\n\n\tdirs = []string{\n\t\tpath.Join(cpusetDir, podCgroupDir, ID),\n\t\t// containerd, systemd\n\t\tpath.Join(cpusetDir, podCgroupDir, \"cri-containerd-\"+ID+\".scope\"),\n\t\t// containerd, cgroupfs\n\t\tpath.Join(cpusetDir, podCgroupDir, \"cri-containerd-\"+ID),\n\t\t// crio, systemd\n\t\tpath.Join(cpusetDir, podCgroupDir, \"crio-\"+ID+\".scope\"),\n\t\t// crio, cgroupfs\n\t\tpath.Join(cpusetDir, podCgroupDir, \"crio-\"+ID),\n\t}\n\n\tfor _, dir := range dirs {\n\t\tif info, err := os.Stat(dir); err == nil {\n\t\t\tif info.Mode().IsDir() {\n\t\t\t\treturn strings.TrimPrefix(dir, cpusetDir)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn \"\"\n}\n\nfunc isSupportedQoSComputeResource(name corev1.ResourceName) bool {\n\treturn name == corev1.ResourceCPU || name == corev1.ResourceMemory\n}\n\nfunc init() {\n\t// TODO: get rid of this eventually, use pkg/sysfs instead...\n\tgetMemoryCapacity()\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/config/api/v1/api.pb.go",
    "content": "//\n//Copyright 2019 Intel Corporation\n//\n//Licensed under the Apache License, Version 2.0 (the \"License\");\n//you may not use this file except in compliance with the License.\n//You may obtain a copy of the License at\n//\n//http://www.apache.org/licenses/LICENSE-2.0\n//\n//Unless required by applicable law or agreed to in writing, software\n//distributed under the License is distributed on an \"AS IS\" BASIS,\n//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n//See the License for the specific language governing permissions and\n//limitations under the License.\n\n// Code generated by protoc-gen-go. DO NOT EDIT.\n// versions:\n// \tprotoc-gen-go v1.28.0\n// \tprotoc        v3.20.1\n// source: pkg/cri/resource-manager/config/api/v1/api.proto\n\npackage v1\n\nimport (\n\tprotoreflect \"google.golang.org/protobuf/reflect/protoreflect\"\n\tprotoimpl \"google.golang.org/protobuf/runtime/protoimpl\"\n\treflect \"reflect\"\n\tsync \"sync\"\n)\n\nconst (\n\t// Verify that this generated code is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)\n\t// Verify that runtime/protoimpl is sufficiently up-to-date.\n\t_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)\n)\n\ntype SetConfigRequest struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\t// node_name is node name used to acquire this configuration.\n\tNodeName string `protobuf:\"bytes,1,opt,name=node_name,json=nodeName,proto3\" json:\"node_name,omitempty\"`\n\t// config is the ConfigMap data.\n\tConfig map[string]string `protobuf:\"bytes,2,rep,name=config,proto3\" json:\"config,omitempty\" protobuf_key:\"bytes,1,opt,name=key,proto3\" protobuf_val:\"bytes,2,opt,name=value,proto3\"`\n}\n\nfunc (x *SetConfigRequest) Reset() {\n\t*x = SetConfigRequest{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[0]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *SetConfigRequest) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SetConfigRequest) ProtoMessage() {}\n\nfunc (x *SetConfigRequest) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[0]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SetConfigRequest.ProtoReflect.Descriptor instead.\nfunc (*SetConfigRequest) Descriptor() ([]byte, []int) {\n\treturn file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescGZIP(), []int{0}\n}\n\nfunc (x *SetConfigRequest) GetNodeName() string {\n\tif x != nil {\n\t\treturn x.NodeName\n\t}\n\treturn \"\"\n}\n\nfunc (x *SetConfigRequest) GetConfig() map[string]string {\n\tif x != nil {\n\t\treturn x.Config\n\t}\n\treturn nil\n}\n\ntype SetConfigReply struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\t// If not empty, indicate an error that happened while trying to apply new configuration.\n\tError string `protobuf:\"bytes,1,opt,name=error,proto3\" json:\"error,omitempty\"`\n}\n\nfunc (x *SetConfigReply) Reset() {\n\t*x = SetConfigReply{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[1]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *SetConfigReply) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SetConfigReply) ProtoMessage() {}\n\nfunc (x *SetConfigReply) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[1]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SetConfigReply.ProtoReflect.Descriptor instead.\nfunc (*SetConfigReply) Descriptor() ([]byte, []int) {\n\treturn file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescGZIP(), []int{1}\n}\n\nfunc (x *SetConfigReply) GetError() string {\n\tif x != nil {\n\t\treturn x.Error\n\t}\n\treturn \"\"\n}\n\ntype SetAdjustmentRequest struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\t// node_name is node name used to acquire this configuration.\n\tNodeName string `protobuf:\"bytes,1,opt,name=node_name,json=nodeName,proto3\" json:\"node_name,omitempty\"`\n\t// Serialized map of all adjustment CRDs, name as key, CRD as value.\n\tAdjustment string `protobuf:\"bytes,2,opt,name=adjustment,proto3\" json:\"adjustment,omitempty\"`\n}\n\nfunc (x *SetAdjustmentRequest) Reset() {\n\t*x = SetAdjustmentRequest{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[2]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *SetAdjustmentRequest) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SetAdjustmentRequest) ProtoMessage() {}\n\nfunc (x *SetAdjustmentRequest) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[2]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SetAdjustmentRequest.ProtoReflect.Descriptor instead.\nfunc (*SetAdjustmentRequest) Descriptor() ([]byte, []int) {\n\treturn file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescGZIP(), []int{2}\n}\n\nfunc (x *SetAdjustmentRequest) GetNodeName() string {\n\tif x != nil {\n\t\treturn x.NodeName\n\t}\n\treturn \"\"\n}\n\nfunc (x *SetAdjustmentRequest) GetAdjustment() string {\n\tif x != nil {\n\t\treturn x.Adjustment\n\t}\n\treturn \"\"\n}\n\ntype SetAdjustmentReply struct {\n\tstate         protoimpl.MessageState\n\tsizeCache     protoimpl.SizeCache\n\tunknownFields protoimpl.UnknownFields\n\n\t// If not empty, indicates that errors happened while trying to apply the adjustments.\n\tErrors map[string]string `protobuf:\"bytes,1,rep,name=errors,proto3\" json:\"errors,omitempty\" protobuf_key:\"bytes,1,opt,name=key,proto3\" protobuf_val:\"bytes,2,opt,name=value,proto3\"`\n}\n\nfunc (x *SetAdjustmentReply) Reset() {\n\t*x = SetAdjustmentReply{}\n\tif protoimpl.UnsafeEnabled {\n\t\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[3]\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tms.StoreMessageInfo(mi)\n\t}\n}\n\nfunc (x *SetAdjustmentReply) String() string {\n\treturn protoimpl.X.MessageStringOf(x)\n}\n\nfunc (*SetAdjustmentReply) ProtoMessage() {}\n\nfunc (x *SetAdjustmentReply) ProtoReflect() protoreflect.Message {\n\tmi := &file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[3]\n\tif protoimpl.UnsafeEnabled && x != nil {\n\t\tms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))\n\t\tif ms.LoadMessageInfo() == nil {\n\t\t\tms.StoreMessageInfo(mi)\n\t\t}\n\t\treturn ms\n\t}\n\treturn mi.MessageOf(x)\n}\n\n// Deprecated: Use SetAdjustmentReply.ProtoReflect.Descriptor instead.\nfunc (*SetAdjustmentReply) Descriptor() ([]byte, []int) {\n\treturn file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescGZIP(), []int{3}\n}\n\nfunc (x *SetAdjustmentReply) GetErrors() map[string]string {\n\tif x != nil {\n\t\treturn x.Errors\n\t}\n\treturn nil\n}\n\nvar File_pkg_cri_resource_manager_config_api_v1_api_proto protoreflect.FileDescriptor\n\nvar file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDesc = []byte{\n\t0x0a, 0x30, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x72, 0x69, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72,\n\t0x63, 0x65, 0x2d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69,\n\t0x67, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x72, 0x6f,\n\t0x74, 0x6f, 0x12, 0x02, 0x76, 0x31, 0x22, 0xa4, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x43, 0x6f,\n\t0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6e,\n\t0x6f, 0x64, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,\n\t0x6e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66,\n\t0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65,\n\t0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43,\n\t0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66,\n\t0x69, 0x67, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72,\n\t0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,\n\t0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,\n\t0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x26, 0x0a,\n\t0x0e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12,\n\t0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,\n\t0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x53, 0x0a, 0x14, 0x53, 0x65, 0x74, 0x41, 0x64, 0x6a, 0x75,\n\t0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a,\n\t0x09, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,\n\t0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x64,\n\t0x6a, 0x75, 0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,\n\t0x61, 0x64, 0x6a, 0x75, 0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x12, 0x53,\n\t0x65, 0x74, 0x41, 0x64, 0x6a, 0x75, 0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c,\n\t0x79, 0x12, 0x3a, 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,\n\t0x0b, 0x32, 0x22, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x41, 0x64, 0x6a, 0x75, 0x73, 0x74,\n\t0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73,\n\t0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x1a, 0x39, 0x0a,\n\t0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,\n\t0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,\n\t0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,\n\t0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0x86, 0x01, 0x0a, 0x06, 0x43, 0x6f, 0x6e,\n\t0x66, 0x69, 0x67, 0x12, 0x37, 0x0a, 0x09, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,\n\t0x12, 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,\n\t0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x43,\n\t0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0d,\n\t0x53, 0x65, 0x74, 0x41, 0x64, 0x6a, 0x75, 0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x2e,\n\t0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x41, 0x64, 0x6a, 0x75, 0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74,\n\t0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74,\n\t0x41, 0x64, 0x6a, 0x75, 0x73, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22,\n\t0x00, 0x42, 0x07, 0x5a, 0x05, 0x2e, 0x2e, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,\n\t0x6f, 0x33,\n}\n\nvar (\n\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescOnce sync.Once\n\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescData = file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDesc\n)\n\nfunc file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescGZIP() []byte {\n\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescOnce.Do(func() {\n\t\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescData)\n\t})\n\treturn file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDescData\n}\n\nvar file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes = make([]protoimpl.MessageInfo, 6)\nvar file_pkg_cri_resource_manager_config_api_v1_api_proto_goTypes = []interface{}{\n\t(*SetConfigRequest)(nil),     // 0: v1.SetConfigRequest\n\t(*SetConfigReply)(nil),       // 1: v1.SetConfigReply\n\t(*SetAdjustmentRequest)(nil), // 2: v1.SetAdjustmentRequest\n\t(*SetAdjustmentReply)(nil),   // 3: v1.SetAdjustmentReply\n\tnil,                          // 4: v1.SetConfigRequest.ConfigEntry\n\tnil,                          // 5: v1.SetAdjustmentReply.ErrorsEntry\n}\nvar file_pkg_cri_resource_manager_config_api_v1_api_proto_depIdxs = []int32{\n\t4, // 0: v1.SetConfigRequest.config:type_name -> v1.SetConfigRequest.ConfigEntry\n\t5, // 1: v1.SetAdjustmentReply.errors:type_name -> v1.SetAdjustmentReply.ErrorsEntry\n\t0, // 2: v1.Config.SetConfig:input_type -> v1.SetConfigRequest\n\t2, // 3: v1.Config.SetAdjustment:input_type -> v1.SetAdjustmentRequest\n\t1, // 4: v1.Config.SetConfig:output_type -> v1.SetConfigReply\n\t3, // 5: v1.Config.SetAdjustment:output_type -> v1.SetAdjustmentReply\n\t4, // [4:6] is the sub-list for method output_type\n\t2, // [2:4] is the sub-list for method input_type\n\t2, // [2:2] is the sub-list for extension type_name\n\t2, // [2:2] is the sub-list for extension extendee\n\t0, // [0:2] is the sub-list for field type_name\n}\n\nfunc init() { file_pkg_cri_resource_manager_config_api_v1_api_proto_init() }\nfunc file_pkg_cri_resource_manager_config_api_v1_api_proto_init() {\n\tif File_pkg_cri_resource_manager_config_api_v1_api_proto != nil {\n\t\treturn\n\t}\n\tif !protoimpl.UnsafeEnabled {\n\t\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*SetConfigRequest); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*SetConfigReply); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*SetAdjustmentRequest); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {\n\t\t\tswitch v := v.(*SetAdjustmentReply); i {\n\t\t\tcase 0:\n\t\t\t\treturn &v.state\n\t\t\tcase 1:\n\t\t\t\treturn &v.sizeCache\n\t\t\tcase 2:\n\t\t\t\treturn &v.unknownFields\n\t\t\tdefault:\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\ttype x struct{}\n\tout := protoimpl.TypeBuilder{\n\t\tFile: protoimpl.DescBuilder{\n\t\t\tGoPackagePath: reflect.TypeOf(x{}).PkgPath(),\n\t\t\tRawDescriptor: file_pkg_cri_resource_manager_config_api_v1_api_proto_rawDesc,\n\t\t\tNumEnums:      0,\n\t\t\tNumMessages:   6,\n\t\t\tNumExtensions: 0,\n\t\t\tNumServices:   1,\n\t\t},\n\t\tGoTypes:           file_pkg_cri_resource_manager_config_api_v1_api_proto_goTypes,\n\t\tDependencyIndexes: file_pkg_cri_resource_manager_config_api_v1_api_proto_depIdxs,\n\t\tMessageInfos:      file_pkg_cri_resource_manager_config_api_v1_api_proto_msgTypes,\n\t}.Build()\n\tFile_pkg_cri_resource_manager_config_api_v1_api_proto = out.File\n\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_rawDesc = nil\n\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_goTypes = nil\n\tfile_pkg_cri_resource_manager_config_api_v1_api_proto_depIdxs = nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/config/api/v1/api.proto",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\nsyntax = \"proto3\";\n\npackage v1;\noption go_package = \"../v1\";\n\nservice Config{\n    rpc SetConfig(SetConfigRequest) returns (SetConfigReply) {}\n    rpc SetAdjustment(SetAdjustmentRequest) returns (SetAdjustmentReply) {}\n}\n\nmessage SetConfigRequest {\n    // node_name is node name used to acquire this configuration.\n    string node_name = 1;\n    // config is the ConfigMap data.\n    map<string, string> config = 2;\n}\n\nmessage SetConfigReply {\n     // If not empty, indicate an error that happened while trying to apply new configuration.\n    string error = 1;\n}\n\nmessage SetAdjustmentRequest {\n    // node_name is node name used to acquire this configuration.\n    string node_name = 1;\n    // Serialized map of all adjustment CRDs, name as key, CRD as value.\n    string adjustment = 2;\n}\n\nmessage SetAdjustmentReply {\n    // If not empty, indicates that errors happened while trying to apply the adjustments.\n    map<string, string> errors = 1;\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/config/api/v1/api_grpc.pb.go",
    "content": "// Code generated by protoc-gen-go-grpc. DO NOT EDIT.\n// versions:\n// - protoc-gen-go-grpc v1.2.0\n// - protoc             v3.20.1\n// source: pkg/cri/resource-manager/config/api/v1/api.proto\n\npackage v1\n\nimport (\n\tcontext \"context\"\n\tgrpc \"google.golang.org/grpc\"\n\tcodes \"google.golang.org/grpc/codes\"\n\tstatus \"google.golang.org/grpc/status\"\n)\n\n// This is a compile-time assertion to ensure that this generated file\n// is compatible with the grpc package it is being compiled against.\n// Requires gRPC-Go v1.32.0 or later.\nconst _ = grpc.SupportPackageIsVersion7\n\n// ConfigClient is the client API for Config service.\n//\n// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.\ntype ConfigClient interface {\n\tSetConfig(ctx context.Context, in *SetConfigRequest, opts ...grpc.CallOption) (*SetConfigReply, error)\n\tSetAdjustment(ctx context.Context, in *SetAdjustmentRequest, opts ...grpc.CallOption) (*SetAdjustmentReply, error)\n}\n\ntype configClient struct {\n\tcc grpc.ClientConnInterface\n}\n\nfunc NewConfigClient(cc grpc.ClientConnInterface) ConfigClient {\n\treturn &configClient{cc}\n}\n\nfunc (c *configClient) SetConfig(ctx context.Context, in *SetConfigRequest, opts ...grpc.CallOption) (*SetConfigReply, error) {\n\tout := new(SetConfigReply)\n\terr := c.cc.Invoke(ctx, \"/v1.Config/SetConfig\", in, out, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn out, nil\n}\n\nfunc (c *configClient) SetAdjustment(ctx context.Context, in *SetAdjustmentRequest, opts ...grpc.CallOption) (*SetAdjustmentReply, error) {\n\tout := new(SetAdjustmentReply)\n\terr := c.cc.Invoke(ctx, \"/v1.Config/SetAdjustment\", in, out, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn out, nil\n}\n\n// ConfigServer is the server API for Config service.\n// All implementations must embed UnimplementedConfigServer\n// for forward compatibility\ntype ConfigServer interface {\n\tSetConfig(context.Context, *SetConfigRequest) (*SetConfigReply, error)\n\tSetAdjustment(context.Context, *SetAdjustmentRequest) (*SetAdjustmentReply, error)\n\tmustEmbedUnimplementedConfigServer()\n}\n\n// UnimplementedConfigServer must be embedded to have forward compatible implementations.\ntype UnimplementedConfigServer struct {\n}\n\nfunc (UnimplementedConfigServer) SetConfig(context.Context, *SetConfigRequest) (*SetConfigReply, error) {\n\treturn nil, status.Errorf(codes.Unimplemented, \"method SetConfig not implemented\")\n}\nfunc (UnimplementedConfigServer) SetAdjustment(context.Context, *SetAdjustmentRequest) (*SetAdjustmentReply, error) {\n\treturn nil, status.Errorf(codes.Unimplemented, \"method SetAdjustment not implemented\")\n}\nfunc (UnimplementedConfigServer) mustEmbedUnimplementedConfigServer() {}\n\n// UnsafeConfigServer may be embedded to opt out of forward compatibility for this service.\n// Use of this interface is not recommended, as added methods to ConfigServer will\n// result in compilation errors.\ntype UnsafeConfigServer interface {\n\tmustEmbedUnimplementedConfigServer()\n}\n\nfunc RegisterConfigServer(s grpc.ServiceRegistrar, srv ConfigServer) {\n\ts.RegisterService(&Config_ServiceDesc, srv)\n}\n\nfunc _Config_SetConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {\n\tin := new(SetConfigRequest)\n\tif err := dec(in); err != nil {\n\t\treturn nil, err\n\t}\n\tif interceptor == nil {\n\t\treturn srv.(ConfigServer).SetConfig(ctx, in)\n\t}\n\tinfo := &grpc.UnaryServerInfo{\n\t\tServer:     srv,\n\t\tFullMethod: \"/v1.Config/SetConfig\",\n\t}\n\thandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\treturn srv.(ConfigServer).SetConfig(ctx, req.(*SetConfigRequest))\n\t}\n\treturn interceptor(ctx, in, info, handler)\n}\n\nfunc _Config_SetAdjustment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {\n\tin := new(SetAdjustmentRequest)\n\tif err := dec(in); err != nil {\n\t\treturn nil, err\n\t}\n\tif interceptor == nil {\n\t\treturn srv.(ConfigServer).SetAdjustment(ctx, in)\n\t}\n\tinfo := &grpc.UnaryServerInfo{\n\t\tServer:     srv,\n\t\tFullMethod: \"/v1.Config/SetAdjustment\",\n\t}\n\thandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\treturn srv.(ConfigServer).SetAdjustment(ctx, req.(*SetAdjustmentRequest))\n\t}\n\treturn interceptor(ctx, in, info, handler)\n}\n\n// Config_ServiceDesc is the grpc.ServiceDesc for Config service.\n// It's only intended for direct use with grpc.RegisterService,\n// and not to be introspected or modified (even as a copy)\nvar Config_ServiceDesc = grpc.ServiceDesc{\n\tServiceName: \"v1.Config\",\n\tHandlerType: (*ConfigServer)(nil),\n\tMethods: []grpc.MethodDesc{\n\t\t{\n\t\t\tMethodName: \"SetConfig\",\n\t\t\tHandler:    _Config_SetConfig_Handler,\n\t\t},\n\t\t{\n\t\t\tMethodName: \"SetAdjustment\",\n\t\t\tHandler:    _Config_SetAdjustment_Handler,\n\t\t},\n\t},\n\tStreams:  []grpc.StreamDesc{},\n\tMetadata: \"pkg/cri/resource-manager/config/api/v1/api.proto\",\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/config/config.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage config\n\nimport (\n\textapi \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n)\n\n// RawConfig represents the resource manager config data in unparsed form, as\n// received from the agent.\ntype RawConfig struct {\n\t// NodeName is the node name the agent used to acquire configuration.\n\tNodeName string\n\t// Data is the raw ConfigMap data for this node.\n\tData map[string]string\n}\n\n// Adjustment represents external adjustments for this node.\ntype Adjustment struct {\n\t// Adjustments contains all adjustment CRDs for this node.\n\tAdjustments map[string]*extapi.AdjustmentSpec\n}\n\n// HasIdenticalData returns true if RawConfig has identical data to the supplied one.\nfunc (c *RawConfig) HasIdenticalData(data map[string]string) bool {\n\tif c == nil && data == nil {\n\t\treturn true\n\t}\n\tif c == nil || data == nil {\n\t\treturn false\n\t}\n\n\tif len(c.Data) != len(data) {\n\t\treturn false\n\t}\n\n\tfor k, v := range c.Data {\n\t\tif dv, found := data[k]; !found || dv != v {\n\t\t\treturn false\n\t\t}\n\t}\n\n\tfor dk, dv := range data {\n\t\tif v, found := c.Data[dk]; !found || v != dv {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/config/server.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage config\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\n\t\"google.golang.org/grpc\"\n\n\tv1 \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/config/api/v1\"\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n\n\t\"encoding/json\"\n\n\textapi \"github.com/intel/cri-resource-manager/pkg/apis/resmgr/v1alpha1\"\n)\n\nconst (\n\tSocketDisabled = \"disabled\"\n)\n\n// SetConfigCb is a callback function for a SetConfig request.\ntype SetConfigCb func(*RawConfig) error\n\n// SetAdjustmentCb is a callback function for a SetAdjustment request.\ntype SetAdjustmentCb func(*Adjustment) map[string]error\n\n// Server is the interface for our gRPC server.\ntype Server interface {\n\tStart(string) error\n\tStop()\n}\n\n// server implements Server.\ntype server struct {\n\tv1.UnimplementedConfigServer\n\tlog.Logger\n\tsocket          string          // configured socket\n\tsync.Mutex                      // lock for concurrent per-request goroutines.\n\tserver          *grpc.Server    // gRPC server instance\n\tsetConfigCb     SetConfigCb     // configuration update notification callback\n\tsetAdjustmentCb SetAdjustmentCb // extneral adjustment update notification callback\n}\n\n// NewConfigServer creates new Server instance.\nfunc NewConfigServer(configCb SetConfigCb, adjustmentCb SetAdjustmentCb) (Server, error) {\n\ts := &server{\n\t\tLogger:          log.NewLogger(\"config-server\"),\n\t\tsetConfigCb:     configCb,\n\t\tsetAdjustmentCb: adjustmentCb,\n\t}\n\treturn s, nil\n}\n\n// Start runs server instance.\nfunc (s *server) Start(socket string) error {\n\tif socket == SocketDisabled || socket == \"\" {\n\t\ts.Info(\"config-server is disabled...,\")\n\t\treturn nil\n\t}\n\n\t// Make sure we have a directory for the socket\n\tif err := os.MkdirAll(filepath.Dir(socket), 0700); err != nil {\n\t\treturn serverError(\"failed to create directory for socket %s: %v\",\n\t\t\tsocket, err)\n\t}\n\n\t// Remove socket file if it exists\n\tif err := os.Remove(socket); err != nil && !os.IsNotExist(err) {\n\t\treturn serverError(\"failed to unlink socket file: %s\", err)\n\t}\n\n\t// Create server listening for local unix domain socket\n\tlis, err := net.Listen(\"unix\", socket)\n\tif err != nil {\n\t\treturn serverError(\"failed to listen to socket: %v\", err)\n\t}\n\n\tserverOpts := []grpc.ServerOption{}\n\ts.server = grpc.NewServer(serverOpts...)\n\tv1.RegisterConfigServer(s.server, s)\n\n\ts.Info(\"starting config-server at socket %s...\", socket)\n\tgo func() {\n\t\tdefer lis.Close()\n\t\terr := s.server.Serve(lis)\n\t\tif err != nil {\n\t\t\ts.Fatal(\"config-server died: %v\", err)\n\t\t}\n\t}()\n\treturn nil\n\n}\n\n// Stop Server instance\nfunc (s *server) Stop() {\n\tif s.server != nil {\n\t\ts.server.Stop()\n\t\ts.server = nil\n\t}\n}\n\n// SetConfig pushes a configuration update to the server.\nfunc (s *server) SetConfig(_ context.Context, req *v1.SetConfigRequest) (*v1.SetConfigReply, error) {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\ts.Debug(\"SetConfig request: %+v\", req)\n\n\treply := &v1.SetConfigReply{}\n\terr := s.setConfigCb(&RawConfig{NodeName: req.NodeName, Data: req.Config})\n\tif err != nil {\n\t\treply.Error = fmt.Sprintf(\"failed to apply configuration: %v\", err)\n\t}\n\n\treturn reply, nil\n}\n\n// SetAdjustment pushes updated external policies to the server.\nfunc (s *server) SetAdjustment(_ context.Context, req *v1.SetAdjustmentRequest) (*v1.SetAdjustmentReply, error) {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\ts.Debug(\"SetAdjustment request: %+v\", req)\n\n\terrors := map[string]error{}\n\tspecs := map[string]*extapi.AdjustmentSpec{}\n\n\tif err := json.Unmarshal([]byte(req.Adjustment), &specs); err != nil {\n\t\treturn nil, serverError(\"failed to decode SetAdjustment request: %v\", err)\n\t}\n\n\tfor name, spec := range specs {\n\t\tif err := spec.Verify(); err != nil {\n\t\t\terrors[name] = err\n\t\t}\n\t}\n\n\tif len(errors) == 0 {\n\t\terrors = s.setAdjustmentCb(&Adjustment{Adjustments: specs})\n\t}\n\n\treply := &v1.SetAdjustmentReply{Errors: make(map[string]string)}\n\tfor str, err := range errors {\n\t\treply.Errors[str] = err.Error()\n\t}\n\treturn reply, nil\n}\n\nfunc serverError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/blockio/blockio.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage blockio\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/blockio\"\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// BlockIOController is the name of the block I/O controller.\n\tBlockIOController = cache.BlockIO\n)\n\n// blockio encapsulates the runtime state of our block I/O enforcement/controller.\ntype blockioctl struct {\n\tcache cache.Cache // resource manager cache\n\tidle  *bool       // true if we run without any classes configured\n}\n\n// Our logger instance.\nvar log logger.Logger = logger.NewLogger(BlockIOController)\n\n// Our singleton block I/O controller instance.\nvar singleton *blockioctl\n\n// getBlockIOController returns our singleton block I/O controller instance.\nfunc getBlockIOController() *blockioctl {\n\tif singleton == nil {\n\t\tsingleton = &blockioctl{}\n\t}\n\treturn singleton\n}\n\n// Start initializes the controller for enforcing decisions.\nfunc (ctl *blockioctl) Start(cache cache.Cache, _ client.Client) error {\n\tctl.cache = cache\n\tctl.reconfigureRunningContainers()\n\treturn nil\n}\n\n// Stop shuts down the controller.\nfunc (ctl *blockioctl) Stop() {\n}\n\n// PreCreateHook is the block I/O controller pre-create hook.\nfunc (ctl *blockioctl) PreCreateHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PreStartHook is the block I/O controller pre-start hook.\nfunc (ctl *blockioctl) PreStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostStartHook is the block I/O controller post-start hook.\nfunc (ctl *blockioctl) PostStartHook(c cache.Container) error {\n\tif !c.HasPending(BlockIOController) {\n\t\treturn nil\n\t}\n\n\tif err := ctl.assign(c); err != nil {\n\t\treturn err\n\t}\n\n\tc.ClearPending(BlockIOController)\n\n\treturn nil\n}\n\n// PostUpdateHook is the block I/O controller post-update hook.\nfunc (ctl *blockioctl) PostUpdateHook(c cache.Container) error {\n\tif !c.HasPending(BlockIOController) {\n\t\treturn nil\n\t}\n\n\tif err := ctl.assign(c); err != nil {\n\t\treturn err\n\t}\n\n\tc.ClearPending(BlockIOController)\n\n\treturn nil\n}\n\n// PostStop is the block I/O controller post-stop hook.\nfunc (ctl *blockioctl) PostStopHook(_ cache.Container) error {\n\treturn nil\n}\n\n// isImplicitlyDisabled checks if we run without any classes confiured\nfunc (ctl *blockioctl) isImplicitlyDisabled() bool {\n\tif ctl.idle != nil {\n\t\treturn *ctl.idle\n\t}\n\n\tidle := len(blockio.GetClasses()) == 0\n\tif idle {\n\t\tlog.Warn(\"controller implictly disabled (no configured classes)\")\n\t}\n\tctl.idle = &idle\n\n\treturn *ctl.idle\n}\n\n// assign assigns the container to the given block I/O class.\nfunc (ctl *blockioctl) assign(c cache.Container) error {\n\tclass := c.GetBlockIOClass()\n\tif class == \"\" {\n\t\treturn nil\n\t}\n\n\tif ctl.isImplicitlyDisabled() && cache.IsPodQOSClassName(class) {\n\t\treturn nil\n\t}\n\n\tif err := blockio.SetContainerClass(c, class); err != nil {\n\t\treturn blockioError(\"%q: failed to assign to class %q: %w\", c.PrettyName(), class, err)\n\t}\n\n\tlog.Info(\"%q: assigned to class %q\", c.PrettyName(), class)\n\n\treturn nil\n}\n\n// configNotify is blockio class mapping and class definition configuration callback\nfunc (ctl *blockioctl) configNotify(event config.Event, _ config.Source) error {\n\tignoreErrors := (event == config.RevertEvent)\n\terr := blockio.UpdateOciConfig(ignoreErrors)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Possible errors in reconfiguring running containers are not errors in\n\t// the updated configuration, therefore silently ignored.\n\tctl.reconfigureRunningContainers()\n\n\t// We'll re-check idleness at next operation/request.\n\tctl.idle = nil\n\n\treturn nil\n}\n\n// reconfigureRunningContainers force setting current blockio configuration to all containers running on the node\nfunc (ctl *blockioctl) reconfigureRunningContainers() error {\n\terrs := []error{}\n\tif ctl.cache == nil {\n\t\treturn nil\n\t}\n\tfor _, c := range ctl.cache.GetContainers() {\n\t\tclass := c.GetBlockIOClass()\n\t\tlog.Debug(\"%q: configure blockio class %q\", c.PrettyName(), class)\n\t\terr := blockio.SetContainerClass(c, class)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\treturn errors.Join(errs...)\n}\n\n// blockioError creates a block I/O-controller-specific formatted error message.\nfunc blockioError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"blockio: \"+format, args...)\n}\n\n// init registers this controller and sets configuration change handling.\nfunc init() {\n\tcontrol.Register(BlockIOController, \"Block I/O controller\", getBlockIOController())\n\tconfig.GetModule(blockio.ConfigModuleName).AddNotify(getBlockIOController().configNotify)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/control.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage control\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\n// Control is the interface for triggering controller-/domain-specific post-decision actions.\ntype Control interface {\n\t// StartStopControllers starts/stops all controllers according to configuration.\n\tStartStopControllers(cache.Cache, client.Client) error\n\t// PreCreateHooks runs the pre-create hooks of all registered controllers.\n\tRunPreCreateHooks(cache.Container) error\n\t// RunPreStartHooks runs the pre-start hooks of all registered controllers.\n\tRunPreStartHooks(cache.Container) error\n\t// RunPostStartHooks runs the post-start hooks of all registered controllers.\n\tRunPostStartHooks(cache.Container) error\n\t// RunPostUpdateHooks runs the post-update hooks of all registered controllers.\n\tRunPostUpdateHooks(cache.Container) error\n\t// RunPostStopHooks runs the post-stop hooks of all registered controllers.\n\tRunPostStopHooks(cache.Container) error\n}\n\n// Controller is the interface all resource controllers must implement.\ntype Controller interface {\n\t// Start prepares the controller for resource control/decision enforcement.\n\tStart(cache.Cache, client.Client) error\n\t// Stop shuts down the controller.\n\tStop()\n\t// PreCreateHook is the controller's pre-create hook.\n\tPreCreateHook(cache.Container) error\n\t// PreStartHook is the controller's pre-start hook.\n\tPreStartHook(cache.Container) error\n\t// PostStartHook is the controller's post-start hook.\n\tPostStartHook(cache.Container) error\n\t// PostUpdateHook is the controller's post-update hook.\n\tPostUpdateHook(cache.Container) error\n\t// PostStopHook is the controller's post-stop hook.\n\tPostStopHook(cache.Container) error\n}\n\n// control encapsulates our controller-agnostic runtime state.\ntype control struct {\n\tcache       cache.Cache   // resource manager cache\n\tclient      client.Client // resource manager CRI client\n\tcontrollers []*controller // active controllers\n}\n\n// controller represents a single registered controller.\ntype controller struct {\n\tname        string     // controller name\n\tdescription string     // controller description\n\tc           Controller // controller interface\n\tmode        mode       // controller mode\n\trunning     bool       // whether the controller is running\n}\n\n// our hook names\nconst (\n\tprecreate  = \"pre-create\"\n\tprestart   = \"pre-start\"\n\tpoststart  = \"post-start\"\n\tpostupdate = \"post-update\"\n\tpoststop   = \"post-stop\"\n)\n\n// All registered controllers.\nvar controllers = make(map[string]*controller)\n\n// Our logger instance.\nvar log logger.Logger = logger.NewLogger(\"resource-control\")\n\n// NewControl creates a new controller-agnostic instance.\nfunc NewControl() (Control, error) {\n\tc := &control{}\n\n\tfor _, controller := range controllers {\n\t\tc.controllers = append(c.controllers, controller)\n\t}\n\tsort.Slice(c.controllers,\n\t\tfunc(i, j int) bool {\n\t\t\treturn strings.Compare(c.controllers[i].name, c.controllers[j].name) < 0\n\t\t})\n\n\treturn c, nil\n}\n\n// StartStopController starts/stops all controllers according to configuration.\nfunc (c *control) StartStopControllers(cache cache.Cache, client client.Client) error {\n\tc.cache = cache\n\tc.client = client\n\n\tlog.Info(\"syncing controllers with configuration...\")\n\n\tfor _, controller := range c.controllers {\n\t\tif controller.mode == Disabled {\n\t\t\tif controller.running {\n\t\t\t\tcontroller.c.Stop()\n\t\t\t\tcontroller.running = false\n\t\t\t}\n\t\t\tlog.Info(\"controller %s: disabled\", controller.name)\n\t\t\tcontinue\n\t\t}\n\n\t\tif controller.running {\n\t\t\tlog.Info(\"controller %s: running\", controller.name)\n\t\t\tcontinue\n\t\t}\n\n\t\terr := controller.c.Start(cache, client)\n\n\t\tif err != nil {\n\t\t\tlog.Error(\"controller %s: failed to start: %v\", controller.name, err)\n\t\t\tcontroller.running = false\n\t\t\tswitch controller.mode {\n\t\t\tcase Required:\n\t\t\t\treturn controlError(\"%s failed to start: %v\", controller.name, err)\n\t\t\tcase Optional, Relaxed:\n\t\t\t\tlog.Warn(\"disabling %s, failed to start: %v\", controller.name, err)\n\t\t\t\tcontroller.mode = Disabled\n\t\t\t}\n\t\t} else {\n\t\t\tcontroller.running = true\n\t\t\tif controller.mode == Optional {\n\t\t\t\tcontroller.mode = Required\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, controller := range c.controllers {\n\t\tstate := map[bool]string{false: \"inactive\", true: \"running\"}\n\t\tlog.Info(\"controller %s is now %s, mode %s\",\n\t\t\tcontroller.name, state[controller.running], controller.mode)\n\t}\n\n\treturn nil\n}\n\n// RunPreCreateHooks runs all registered controllers' PreCreate hooks.\nfunc (c *control) RunPreCreateHooks(container cache.Container) error {\n\tfor _, controller := range c.controllers {\n\t\tif err := c.runhook(controller, precreate, container); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// RunPreStartHooks runs all registered controllers' PreStart hooks.\nfunc (c *control) RunPreStartHooks(container cache.Container) error {\n\tfor _, controller := range c.controllers {\n\t\tif err := c.runhook(controller, prestart, container); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// RunPostStartHooks runs all registered controllers' PostStart hooks.\nfunc (c *control) RunPostStartHooks(container cache.Container) error {\n\tfor _, controller := range c.controllers {\n\t\tif err := c.runhook(controller, poststart, container); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// RunPostUpdateHooks runs all registered controllers' PostUpdate hooks.\nfunc (c *control) RunPostUpdateHooks(container cache.Container) error {\n\tfor _, controller := range c.controllers {\n\t\tif err := c.runhook(controller, postupdate, container); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// RunPostStopHooks runs all registered controllers' PostStop hooks.\nfunc (c *control) RunPostStopHooks(container cache.Container) error {\n\tfor _, controller := range c.controllers {\n\t\tif err := c.runhook(controller, poststop, container); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// runhook executes the given container hook according to the controller settings\nfunc (c *control) runhook(controller *controller, hook string, container cache.Container) error {\n\tif controller.mode == Disabled || !controller.running {\n\t\treturn nil\n\t}\n\n\tvar fn func(cache.Container) error\n\n\tswitch hook {\n\tcase precreate:\n\t\tfn = controller.c.PreCreateHook\n\tcase prestart:\n\t\tfn = controller.c.PreStartHook\n\tcase poststart:\n\t\tfn = controller.c.PostStartHook\n\tcase postupdate:\n\t\tfn = controller.c.PostUpdateHook\n\tcase poststop:\n\t\tfn = controller.c.PostStopHook\n\t}\n\n\tlog.Debug(\"running %s %s hook for container %s\", controller.name, hook, container.PrettyName())\n\n\tif err := fn(container); err != nil {\n\t\tif controller.mode == Required {\n\t\t\treturn controlError(\"%s %s hook failed: %v\", controller.name, hook, err)\n\t\t}\n\t\tlog.Error(\"%s %s hook failed: %v\", controller.name, hook, err)\n\t}\n\n\treturn nil\n}\n\n// Register registers a new controller.\nfunc Register(name, description string, c Controller) error {\n\tlog.Info(\"registering controller %s...\", name)\n\n\tif oc, ok := controllers[name]; ok {\n\t\treturn controlError(\"controller %s (%s) already registered.\", oc.name, oc.description)\n\t}\n\n\tcontrollers[name] = &controller{\n\t\tname:        name,\n\t\tdescription: description,\n\t\tc:           c,\n\t}\n\n\treturn nil\n}\n\n// controlError returns a controller-specific formatted error.\nfunc controlError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"control: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/cpu/api.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpu\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// GetClasses returns all available CPU classes.\nfunc GetClasses() map[string]Class {\n\treturn getCPUController().config.getClasses()\n}\n\n// Assign assigns a set of cpus to a class.\n//\n// TODO: Drop this function. Don't store cpu class in policy data but implement\n// controller-specific data store in cache.\nfunc Assign(c cache.Cache, class string, cpus ...int) error {\n\t// NOTE: no locking implemented anywhere around -> we don't expect multiple parallel callers\n\n\t// Store the class assignment. Assign cpus to a class and remove them from\n\t// other classes\n\tassignments := *getClassAssignments(c)\n\n\tif this, ok := assignments[class]; !ok {\n\t\tassignments[class] = utils.NewIDSetFromIntSlice(cpus...)\n\t} else {\n\t\tthis.Add(cpus...)\n\t}\n\n\tfor k, v := range assignments {\n\t\tif k != class {\n\t\t\tv.Del(cpus...)\n\n\t\t\t// Don't store empty classes, serves as a garbage collector, too\n\t\t\tif v.Size() == 0 {\n\t\t\t\tdelete(assignments, k)\n\t\t\t}\n\t\t}\n\t}\n\n\tsetClassAssignments(c, &assignments)\n\n\tif getCPUController().started {\n\t\t// We don't want to try to enforce until the controller has been fully\n\t\t// started. Enforcement of all assignments happens on StarT(), anyway.\n\t\tctl := getCPUController()\n\t\tif err := ctl.enforceCpufreq(class, cpus...); err != nil {\n\t\t\tlog.Error(\"cpufreq enforcement failed: %v\", err)\n\t\t}\n\t\tif err := ctl.enforceUncore(assignments, cpus...); err != nil {\n\t\t\tlog.Error(\"uncore frequency enforcement failed: %v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/cpu/cache.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpu\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\tcacheKeyCPUAssignments = \"CPUClassAssignments\"\n)\n\n// cpuClassAssignments contains the information about how cpus are assigned to\n// classes\ntype cpuClassAssignments map[string]utils.IDSet\n\n// Get the state of CPU class assignments from cache\nfunc getClassAssignments(c cache.Cache) *cpuClassAssignments {\n\ta := &cpuClassAssignments{}\n\n\tif !c.GetPolicyEntry(cacheKeyCPUAssignments, a) {\n\t\tlog.Error(\"no cached state of CPU class assignments found\")\n\t}\n\n\treturn a\n}\n\n// Save the state of CPU class assignments in cache\nfunc setClassAssignments(c cache.Cache, a *cpuClassAssignments) {\n\tc.SetPolicyEntry(cacheKeyCPUAssignments, cache.Cachable(a))\n}\n\n// Set the value of cached cpuClassAssignments\nfunc (c *cpuClassAssignments) Set(value interface{}) {\n\tswitch value.(type) {\n\tcase cpuClassAssignments:\n\t\t*c = value.(cpuClassAssignments)\n\tcase *cpuClassAssignments:\n\t\tcp := value.(*cpuClassAssignments)\n\t\t*c = *cp\n\t}\n}\n\n// Get cached cpuClassAssignments\nfunc (c *cpuClassAssignments) Get() interface{} {\n\treturn *c\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/cpu/cpu.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpu\n\nimport (\n\t\"fmt\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// ConfigModuleName is the configuration section for the CPU controller.\n\tConfigModuleName = \"cpu\"\n\n\t// CPUController is the name of the CPU controller.\n\tCPUController = cache.CPU\n)\n\n// cpuctl encapsulates the runtime state of our CPU enforcement/controller.\ntype cpuctl struct {\n\tcache   cache.Cache  // resource manager cache\n\tsystem  sysfs.System // system topology\n\tconfig  *config\n\tstarted bool\n}\n\ntype config struct {\n\tClasses map[string]Class `json:\"classes\"`\n\n\t// Private field for storing info if we need to care about uncore\n\tuncoreEnabled bool\n}\n\ntype Class struct {\n\tMinFreq                     uint `json:\"minFreq\"`\n\tMaxFreq                     uint `json:\"maxFreq\"`\n\tEnergyPerformancePreference uint `json:\"energyPerformancePreference\"`\n\tUncoreMinFreq               uint `json:\"uncoreMinFreq\"`\n\tUncoreMaxFreq               uint `json:\"uncoreMaxFreq\"`\n}\n\nvar log logger.Logger = logger.NewLogger(CPUController)\n\n// Ccontroller singleton instance.\nvar singleton *cpuctl\n\n// getCPUController returns the (singleton) CPU controller instance.\nfunc getCPUController() *cpuctl {\n\tif singleton == nil {\n\t\tsingleton = &cpuctl{}\n\t\tsingleton.config = singleton.defaultOptions().(*config)\n\t}\n\treturn singleton\n}\n\n// Start initializes the controller for enforcing decisions.\nfunc (ctl *cpuctl) Start(cache cache.Cache, _ client.Client) error {\n\tsys, err := sysfs.DiscoverSystem()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to discover system topology: %w\", err)\n\t}\n\n\tctl.system = sys\n\tctl.cache = cache\n\n\t// DEBUG: dump the class assignments we have stored in the cache\n\tlog.Debug(\"retrieved cpu class assignments from cache:\\n%s\", utils.DumpJSON(getClassAssignments(ctl.cache)))\n\n\tif err := ctl.configure(); err != nil {\n\t\t// Just print an error. A config update later on may be valid.\n\t\tlog.Error(\"failed apply /cpuinitial configuration: %v\", err)\n\t}\n\n\t// TODO: We probably could just remove this and the hooks if they are not used\n\tpkgcfg.GetModule(ConfigModuleName).AddNotify(getCPUController().configNotify)\n\n\tctl.started = true\n\n\treturn nil\n}\n\n// Stop shuts down the controller.\nfunc (ctl *cpuctl) Stop() {\n}\n\n// PreCreateHook handler for the CPU controller.\nfunc (ctl *cpuctl) PreCreateHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PreStartHook handler for the CPU controller.\nfunc (ctl *cpuctl) PreStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostStartHook handler for the CPU controller.\nfunc (ctl *cpuctl) PostStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostUpdateHook handler for the CPU controller.\nfunc (ctl *cpuctl) PostUpdateHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostStopHook handler for the CPU controller.\nfunc (ctl *cpuctl) PostStopHook(_ cache.Container) error {\n\treturn nil\n}\n\n// enforceCpufreq enforces a class-specific cpufreq configuration to a cpuset\nfunc (ctl *cpuctl) enforceCpufreq(class string, cpus ...int) error {\n\tif _, ok := ctl.config.Classes[class]; !ok {\n\t\treturn fmt.Errorf(\"non-existent cpu class %q\", class)\n\t}\n\n\tmin := int(ctl.config.Classes[class].MinFreq)\n\tmax := int(ctl.config.Classes[class].MaxFreq)\n\tlog.Debug(\"enforcing cpu frequency limits {%d, %d} from class %q on %v\", min, max, class, cpus)\n\n\tif err := utils.SetCPUsScalingMinFreq(cpus, min); err != nil {\n\t\treturn fmt.Errorf(\"Cannot set min freq %d: %w\", min, err)\n\t}\n\n\tif err := utils.SetCPUsScalingMaxFreq(cpus, max); err != nil {\n\t\treturn fmt.Errorf(\"Cannot set max freq %d: %w\", max, err)\n\t}\n\n\treturn nil\n}\n\n// enforceUncore enforces uncore frequency limits\nfunc (ctl *cpuctl) enforceUncore(assignments cpuClassAssignments, affectedCPUs ...int) error {\n\tif !ctl.config.uncoreEnabled {\n\t\treturn nil\n\t}\n\n\tcpus := cpuset.New(affectedCPUs...)\n\n\tfor _, cpuPkgID := range ctl.system.PackageIDs() {\n\t\tcpuPkg := ctl.system.Package(cpuPkgID)\n\t\tfor _, cpuDieID := range cpuPkg.DieIDs() {\n\t\t\tdieCPUs := cpuPkg.DieCPUSet(cpuDieID)\n\n\t\t\t// Check if this die is affected by the specified cpuset\n\t\t\tif cpus.Size() == 0 || dieCPUs.Intersection(cpus).Size() > 0 {\n\t\t\t\tmin, max, minCls, maxCls := effectiveUncoreFreqs(utils.NewIDSet(dieCPUs.List()...), ctl.config.Classes, assignments)\n\n\t\t\t\tif min == 0 && max == 0 {\n\t\t\t\t\tlog.Debug(\"no uncore frequency limits for cpu package/die %d/%d\", cpuPkgID, cpuDieID)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tlog.Debug(\"enforcing uncore min freq to %d (class %q), max freq to %d (class %q) on cpu package/die %d/%d\", min, minCls, max, maxCls, cpuPkgID, cpuDieID)\n\t\t\t\tif min > 0 {\n\t\t\t\t\tif max > 0 && min > max {\n\t\t\t\t\t\tlog.Warn(\"uncore frequency limit min > max (%d > %d) on cpu package/die %d/%d\", min, max, cpuPkgID, cpuDieID)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := utils.SetUncoreMinFreq(cpuPkgID, cpuDieID, int(min)); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif max > 0 {\n\t\t\t\t\tif err := utils.SetUncoreMaxFreq(cpuPkgID, cpuDieID, int(max)); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// effectiveUncoreClasses resolves the effective classes for setting the uncore\n// frequency limits for a cpu package/die. It has \"performance preference\" so\n// that the highest value (for both min and max) of the cpu classes effective\n// on the die is selected.\nfunc effectiveUncoreFreqs(cpus utils.IDSet, classes map[string]Class, assignments cpuClassAssignments) (minFreq, maxFreq uint, minCls, maxCls string) {\n\tfor className, assignedCPUs := range assignments {\n\t\t// Check if this class is \"effective\" on the specified cpuset\n\t\tif idSetIntersects(cpus, assignedCPUs) {\n\t\t\tclass := classes[className]\n\t\t\tif class.UncoreMinFreq > minFreq {\n\t\t\t\tminCls = className\n\t\t\t\tminFreq = class.UncoreMinFreq\n\t\t\t}\n\t\t\tif class.UncoreMaxFreq > maxFreq {\n\t\t\t\tmaxCls = className\n\t\t\t\tmaxFreq = class.UncoreMaxFreq\n\t\t\t}\n\t\t}\n\t}\n\treturn minFreq, maxFreq, minCls, maxCls\n}\n\nfunc idSetIntersects(a, b utils.IDSet) bool {\n\t// Try to optimize the search for unbalanced idsets\n\tif len(a) < len(b) {\n\t\tfor id := range a {\n\t\t\tif _, ok := b[id]; ok {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t} else {\n\t\tfor id := range b {\n\t\t\tif _, ok := a[id]; ok {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (ctl *cpuctl) configure() error {\n\t// Re-configure CPUs that are assigned to some known class\n\tassignments := *getClassAssignments(ctl.cache)\n\n\t// DEBUG: dump the class assignments we have stored in the cache\n\tlog.Debug(\"applying cpu controller configuration:\\n%s\", utils.DumpJSON(ctl.config))\n\n\t// Sanity check\n\tuncoreAvailable := utils.UncoreFreqAvailable()\n\tfor name, conf := range ctl.config.Classes {\n\t\tif conf.UncoreMinFreq != 0 || conf.UncoreMaxFreq != 0 {\n\t\t\tif !uncoreAvailable {\n\t\t\t\treturn fmt.Errorf(\"uncore limits set in cpu class %q but uncore driver not available in the system, make sure that the intel_uncore_frequency driver is loaded\", name)\n\t\t\t}\n\t\t\tctl.config.uncoreEnabled = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Configure the system\n\tfor class, cpus := range assignments {\n\t\tif _, ok := ctl.config.Classes[class]; ok {\n\t\t\t// Re-configure cpus (sysfs) according to new class parameters\n\t\t\tif err := ctl.enforceCpufreq(class, cpus.SortedMembers()...); err != nil {\n\t\t\t\tlog.Error(\"cpufreq enforcement on re-configure failed: %v\", err)\n\t\t\t}\n\t\t} else {\n\t\t\t// TODO: what should we really do with classes that do not exist in\n\t\t\t// the configuration anymore? Now we remember the CPUs assigned to\n\t\t\t// them. A further config update might re-introduce the class in\n\t\t\t// which case the CPUs will be reconfigured.\n\t\t\tlog.Warn(\"class %q with cpus %v missing from the configuration\", class, cpus)\n\t\t}\n\t}\n\tif err := ctl.enforceUncore(assignments); err != nil {\n\t\tlog.Error(\"uncore frequency enforcement on re-configure failed: %v\", err)\n\t}\n\n\tlog.Debug(\"cpu controller configured\")\n\n\treturn nil\n}\n\n// Callback for runtime configuration notifications.\nfunc (ctl *cpuctl) configNotify(_ pkgcfg.Event, _ pkgcfg.Source) error {\n\tif !ctl.started {\n\t\t// We don't want to configure until the controller has been fully\n\t\t// started and initialized. We will configure on Start(), anyway.\n\t\treturn nil\n\t}\n\n\tlog.Info(\"configuration update, applying new config\")\n\treturn ctl.configure()\n}\n\nfunc (ctl *cpuctl) defaultOptions() interface{} {\n\treturn &config{}\n}\n\nfunc (c *config) getClasses() map[string]Class {\n\tret := make(map[string]Class, len(c.Classes))\n\tfor k, v := range c.Classes {\n\t\tret[k] = v\n\t}\n\treturn ret\n}\n\n// Register us as a controller.\nfunc init() {\n\tcontrol.Register(CPUController, \"CPU controller\", getCPUController())\n\tpkgcfg.Register(ConfigModuleName, \"CPU control\", getCPUController().config, getCPUController().defaultOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/cri/cri.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cri\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// CRIController is the name of this controller.\n\tCRIController = cache.CRI\n)\n\n// crictl encapsulated the runtime state of our CRI enforcement/controller.\ntype crictl struct {\n\tcache  cache.Cache\n\tclient client.Client\n}\n\n// Our logger instance.\nvar log logger.Logger = logger.NewLogger(CRIController)\n\n// Our CRI controller singleton instance.\nvar singleton *crictl\n\n// getCRIController returns our singleton CRI controller instance.\nfunc getCRIController() control.Controller {\n\tif singleton == nil {\n\t\tsingleton = &crictl{}\n\t}\n\treturn singleton\n}\n\n// Start initializes the controller for enforcing decisions.\nfunc (ctl *crictl) Start(cache cache.Cache, client client.Client) error {\n\tctl.cache = cache\n\tctl.client = client\n\n\treturn nil\n}\n\n// Stop shuts down the controller.\nfunc (ctl *crictl) Stop() {\n}\n\n// PreCreateHook is the CRI controller pre-create hook.\nfunc (ctl *crictl) PreCreateHook(c cache.Container) error {\n\tif !c.HasPending(CRIController) {\n\t\tlog.Debug(\"pre-create hook: no pending changes for %s\", c.PrettyName())\n\t\treturn nil\n\t}\n\n\tlog.Debug(\"pre-create hook: updating %s\", c.PrettyName())\n\n\trequest, ok := c.GetCRIRequest()\n\tif !ok {\n\t\treturn criError(\"pre-create hook: no pending CRI request\")\n\t}\n\tcreate, ok := request.(*criv1.CreateContainerRequest)\n\tif !ok {\n\t\treturn criError(\"pre-create hook: pending CRI request of wrong type (%T)\", request)\n\t}\n\n\tcreate.Config.Command = c.GetCommand()\n\tcreate.Config.Args = c.GetArgs()\n\tcreate.Config.Labels = c.GetLabels()\n\tcreate.Config.Annotations = c.GetAnnotations()\n\tcreate.Config.Envs = c.GetCRIEnvs()\n\tcreate.Config.Mounts = c.GetCRIMounts()\n\tcreate.Config.Devices = c.GetCRIDevices()\n\tif create.Config.Linux == nil {\n\t\tcreate.Config.Linux = &criv1.LinuxContainerConfig{}\n\t}\n\tcreate.Config.Linux.Resources = c.GetLinuxResources()\n\n\tc.ClearPending(CRIController)\n\n\treturn nil\n}\n\n// PreStartHook is the CRI controller pre-start hook.\nfunc (ctl *crictl) PreStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostStartHook is the CRI controller post-start hook.\nfunc (ctl *crictl) PostStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostUpdateHook is the CRI controller post-update hook.\nfunc (ctl *crictl) PostUpdateHook(c cache.Container) error {\n\tvar update *criv1.UpdateContainerResourcesRequest\n\n\tif !c.HasPending(CRIController) {\n\t\tlog.Debug(\"post-update hook: no changes for %s\", c.PrettyName())\n\t\treturn nil\n\t}\n\n\tlog.Debug(\"post-update hook: updating %s\", c.PrettyName())\n\n\tresources := c.GetLinuxResources()\n\tif resources == nil {\n\t\treturn nil\n\t}\n\trequest, ok := c.GetCRIRequest()\n\tif !ok {\n\t\tupdate = &criv1.UpdateContainerResourcesRequest{\n\t\t\tContainerId: c.GetID(),\n\t\t}\n\t\tc.SetCRIRequest(update)\n\t} else {\n\t\tif update, ok = request.(*criv1.UpdateContainerResourcesRequest); !ok {\n\t\t\treturn criError(\"post-update hook: CRI request of wrong type (%T)\", request)\n\t\t}\n\t}\n\tupdate.Linux = resources\n\n\tc.ClearPending(CRIController)\n\n\treturn nil\n}\n\n// PostStop is the CRI controller post-stop hook.\nfunc (ctl *crictl) PostStopHook(_ cache.Container) error {\n\treturn nil\n}\n\n// criError creates an CRI-controller-specific formatted error message.\nfunc criError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"cri: \"+format, args...)\n}\n\n// Register us as a controller.\nfunc init() {\n\tcontrol.Register(CRIController, \"CRI controller\", getCRIController())\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/flags.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage control\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// Options captures our runtime configuration.\ntype options struct {\n\tControllers map[string]mode\n}\n\n// Our runtime configuration.\nvar opt = defaultOptions().(*options)\n\n// mode describes how errors for the controller should be treated.\ntype mode int\n\nconst (\n\t// Disabled controller are stopped, hooks are not run.\n\tDisabled mode = iota\n\t// Required controllers must start, hooks must succeed.\n\tRequired\n\t// Optional controllers are Disabled if they can't start, otherwise they are Required.\n\tOptional\n\t// Relaxed controllers are Disabled if they can't start, hook failures are not errors.\n\tRelaxed\n\t// Default mode is Relaxed.\n\tDefault = Relaxed\n)\n\n// ControllerMode returns the current mode for the given controller.\nfunc (o *options) ControllerMode(name string) mode {\n\tif m, ok := o.Controllers[name]; ok {\n\t\treturn m\n\t}\n\n\treturn Default\n}\n\n// configNotify is our configuration update notification callback.\nfunc (o *options) configNotify(_ config.Event, _ config.Source) error {\n\tlog.Info(\"configuration updated\")\n\tfor name, controller := range controllers {\n\t\tcontroller.mode = o.ControllerMode(name)\n\t}\n\treturn nil\n}\n\n// String returns the string representation of a mode.\nfunc (m mode) String() string {\n\tswitch m {\n\tcase Disabled:\n\t\treturn \"disabled\"\n\tcase Required:\n\t\treturn \"required\"\n\tcase Optional:\n\t\treturn \"optional\"\n\tcase Relaxed:\n\t\treturn \"relaxed\"\n\tdefault:\n\t\treturn fmt.Sprintf(\"<unknown mode %d>\", m)\n\t}\n}\n\n// MarshalJSON is the JSON marshaller for mode.\nfunc (m mode) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(m.String())\n}\n\n// UnmarshalJSON is the JSON unmarshaller for mode.\nfunc (m *mode) UnmarshalJSON(raw []byte) error {\n\tvar str string\n\n\tif err := json.Unmarshal(raw, &str); err != nil {\n\t\treturn controlError(\"failed to unmarshal mode: %v\", err)\n\t}\n\n\tswitch strings.ToLower(str) {\n\tcase \"disabled\", \"disable\":\n\t\t*m = Disabled\n\tcase \"required\", \"mandatory\":\n\t\t*m = Required\n\tcase \"optional\":\n\t\t*m = Optional\n\tcase \"relaxed\":\n\t\t*m = Relaxed\n\tdefault:\n\t\treturn controlError(\"invalid mode %s\", str)\n\t}\n\treturn nil\n}\n\n// defaultOptions returns a new options instance, all initialized to defaults.\nfunc defaultOptions() interface{} {\n\treturn &options{Controllers: make(map[string]mode)}\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tconfig.Register(\"resource-manager.control\", \"Resource control.\", opt, defaultOptions,\n\t\tconfig.WithNotify(opt.configNotify))\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/memory/memory.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage memory\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// MemoryController is the name of the memory controller.\n\tMemoryController = cache.Memory\n\n\t// memoryCgroupPath is the path to the root of the memory cgroup.\n\tmemoryCgroupPath = \"/sys/fs/cgroup/memory\"\n\t// toptierSoftLimitControl is the memory cgroup entry to set top tier soft limit.\n\ttoptierSoftLimitControl = \"memory.toptier_soft_limit_in_bytes\"\n)\n\n// memctl encapsulates the runtime state of our memory enforcement/controller.\ntype memctl struct {\n\tcache    cache.Cache // resource manager cache\n\tdisabled bool        // true, if kernel lacks the necessary cgroup controls\n}\n\n// Our logger instance.\nvar log logger.Logger = logger.NewLogger(MemoryController)\n\n// Our singleton memory controller instance.\nvar singleton *memctl\n\n// getMemoryController returns our singleton memory controller instance.\nfunc getMemoryController() *memctl {\n\tif singleton == nil {\n\t\tsingleton = &memctl{}\n\t}\n\treturn singleton\n}\n\n// Start initializes the controller for enforcing decisions.\nfunc (ctl *memctl) Start(cache cache.Cache, _ client.Client) error {\n\t// Let's keep this off for now so we can exercise this without a patched kernel...\n\tif !ctl.checkToptierLimitSupport() {\n\t\treturn memctlError(\"cgroup top tier memory limit control not available\")\n\t}\n\tctl.cache = cache\n\treturn nil\n}\n\n// Stop shuts down the controller.\nfunc (ctl *memctl) Stop() {\n}\n\n// PreCreateHook is the memory controller pre-create hook.\nfunc (ctl *memctl) PreCreateHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PreStartHook is the memory controller pre-start hook.\nfunc (ctl *memctl) PreStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostStartHook is the memory controller post-start hook.\nfunc (ctl *memctl) PostStartHook(c cache.Container) error {\n\tif !c.HasPending(MemoryController) {\n\t\treturn nil\n\t}\n\n\tif err := ctl.setToptierLimit(c); err != nil {\n\t\treturn err\n\t}\n\n\tc.ClearPending(MemoryController)\n\n\treturn nil\n}\n\n// PostUpdateHook is the memory controller post-update hook.\nfunc (ctl *memctl) PostUpdateHook(c cache.Container) error {\n\tif !c.HasPending(MemoryController) {\n\t\treturn nil\n\t}\n\n\tif err := ctl.setToptierLimit(c); err != nil {\n\t\treturn err\n\t}\n\n\tc.ClearPending(MemoryController)\n\n\treturn nil\n}\n\n// PostStop is the memory controller post-stop hook.\nfunc (ctl *memctl) PostStopHook(_ cache.Container) error {\n\treturn nil\n}\n\n// Check if memory cgroup controller supports top tier soft limits.\nfunc (ctl *memctl) checkToptierLimitSupport() bool {\n\t_, err := os.Stat(memoryCgroupPath + \"/\" + toptierSoftLimitControl)\n\tif err != nil && os.IsNotExist(err) {\n\t\tlog.Warn(\"cgroup top tier memory limit control not available\")\n\t\tctl.disabled = true\n\t}\n\treturn !ctl.disabled\n}\n\n// setToptierLimit sets the top tier memory (soft) limit for the container.\nfunc (ctl *memctl) setToptierLimit(c cache.Container) error {\n\tdir := c.GetCgroupDir()\n\tif dir == \"\" {\n\t\treturn memctlError(\"%q: failed to determine cgroup directory\",\n\t\t\tc.PrettyName())\n\t}\n\n\tlimit := strconv.FormatInt(c.GetToptierLimit(), 10)\n\tgroup := cgroups.Memory.Group(dir)\n\tentry := toptierSoftLimitControl\n\n\tif err := group.Write(entry, \"%s\\n\", limit); err != nil {\n\t\treturn err\n\t}\n\n\tlog.Info(\"%q: memory toptier soft limit set to %v\", c.PrettyName(), limit)\n\n\treturn nil\n}\n\n// memctlError creates a memory I/O-controller-specific formatted error message.\nfunc memctlError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"memory: \"+format, args...)\n}\n\n// init registers this controller.\nfunc init() {\n\tcontrol.Register(MemoryController, \"memory toptier controller\", getMemoryController())\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/page-migrate/demoter.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pagemigrate\n\nimport (\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// Support dynamic pushing of unused pages from DRAM to PMEM.\n//\n// The algorithm is be (roughly) this:\n//\n// Find out which processes belong to the container. For every process in the\n// container, find out which pages the process uses. Using move_pages(), push a\n// number of pages not in the working set, which are present in DRAM, from DRAM\n// to PMEM. This may need to be done for many times with a delay in between,\n// because the process will be \"stuck\" when the pages are moved. Repeat this\n// process.\n//\n// How to figure out which pages are not part of the working set:\n//\n// 1. Clear soft-dirty bits on the PTEs:\n//    https://www.kernel.org/doc/html/latest/admin-guide/mm/soft-dirty.html\n// 2. Wait for a while.\n// 3. Read out the process page maps:\n//    https://www.kernel.org/doc/html/latest/admin-guide/mm/pagemap.html The pages\n//    which don't have the soft-dirty bit are considered to be outside of the\n//    working set.\n\ntype page struct {\n\tpid  int\n\taddr uint64\n}\n\ntype addrRange struct {\n\taddr   uint64\n\tlength uint64\n}\n\ntype demoter struct {\n\tmigration *migration // controller backpointer\n\n\t// Finding pages\n\tdirtyBitReset time.Ticker      // Ticker for resetting the dirty bits.\n\tdirtyBitStop  chan interface{} // Channel for stopping the ticker.\n\n\t// Moving pages\n\tpageMover         PageMover\n\tcontainerDemoters map[string]chan interface{} // Channel for sending pagemap updates to demoters.\n\tpageScanInterval  config.Duration             // How often should we scan pages.\n\tpageMoveInterval  config.Duration             // How often should we move pages for a container.\n\tmaxPageMoveCount  uint                        // How many pages to move at once.\n}\n\ntype pagePool struct {\n\tpages        map[int][]page\n\tlongestRange uint\n}\n\ntype demotion struct {\n\tpagePool    pagePool\n\ttargetNodes idset.IDSet\n}\n\nfunc copyPagePool(p pagePool) pagePool {\n\tc := pagePool{\n\t\tlongestRange: p.longestRange,\n\t\tpages:        make(map[int][]page, 0),\n\t}\n\tfor pid, pages := range p.pages {\n\t\tc.pages[pid] = make([]page, len(pages))\n\t\tcopy(c.pages[pid], pages)\n\t}\n\treturn c\n}\n\nfunc newDemoter(m *migration) *demoter {\n\treturn &demoter{\n\t\tmigration:         m,\n\t\tcontainerDemoters: make(map[string]chan interface{}, 0),\n\t\tpageMover:         &linuxPageMover{},\n\t}\n}\n\nfunc (d *demoter) start() {\n\tif d.pageScanInterval > 0 && d.pageMoveInterval > 0 && d.maxPageMoveCount > 0 {\n\t\tlog.Info(\"scanning pages every %s, moving max. %d pages every %s\",\n\t\t\td.pageScanInterval.String(), d.maxPageMoveCount, d.pageMoveInterval.String())\n\t\td.startDirtyBitResetTimer()\n\t} else {\n\t\tlog.Info(\"scanning pages is disabled\")\n\t}\n}\n\n// Stop stops page scanning and demotion.\nfunc (d *demoter) Stop() {\n\td.stopDirtyBitResetTimer()\n\td.migration.Lock()\n\tdefer d.migration.Unlock()\n\td.stopDemoters()\n}\n\n// Reconfigure restarts, if necessary, page scanning and demotion with new options.\nfunc (d *demoter) Reconfigure() {\n\tif d.pageScanInterval != opt.PageScanInterval ||\n\t\td.pageMoveInterval != opt.PageMoveInterval ||\n\t\td.maxPageMoveCount != opt.MaxPageMoveCount {\n\t\td.Stop()\n\t\td.pageScanInterval = opt.PageScanInterval\n\t\td.pageMoveInterval = opt.PageMoveInterval\n\t\td.maxPageMoveCount = opt.MaxPageMoveCount\n\t}\n\td.start()\n}\n\nfunc (d *demoter) updateDemoter(cid string, p pagePool, targetNodes idset.IDSet) {\n\tchannel, found := d.containerDemoters[cid]\n\tif !found {\n\t\tchannel := make(chan interface{})\n\t\tgo func() {\n\t\t\tmoveTimer := time.NewTicker(time.Duration(d.pageMoveInterval))\n\t\t\tmoveTimerChan := moveTimer.C\n\t\t\tpagePool := p\n\t\t\tnodes := targetNodes\n\t\t\tcount := d.maxPageMoveCount\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase msg := <-channel:\n\t\t\t\t\tdemotion, ok := msg.(demotion)\n\t\t\t\t\tif ok {\n\t\t\t\t\t\tpagePool = demotion.pagePool\n\t\t\t\t\t\ttargetNodes = demotion.targetNodes\n\t\t\t\t\t\tif p.longestRange > d.maxPageMoveCount {\n\t\t\t\t\t\t\t// The number of pages moved needs to be at least as large as a range in numa_maps\n\t\t\t\t\t\t\t// file so that we know that all pages will be moved (even if some of them were\n\t\t\t\t\t\t\t// already on the PMEM node).\n\n\t\t\t\t\t\t\t// TODO: adjust the timer if we have a larger-than-usual range of pages to move.\n\t\t\t\t\t\t\tcount = p.longestRange\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tcount = d.maxPageMoveCount\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// A stop request.\n\t\t\t\t\t\tif moveTimer != nil {\n\t\t\t\t\t\t\tmoveTimer.Stop()\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\tcase _ = <-moveTimerChan:\n\t\t\t\t\terr := d.movePages(pagePool, count, nodes)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Error(\"Error demoting pages: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t\td.containerDemoters[cid] = channel\n\t\t// TODO: trigger instant update when run the first time?\n\t} else {\n\t\tchannel <- demotion{pagePool: p, targetNodes: targetNodes}\n\t}\n}\n\nfunc (d *demoter) stopDemoter(cid string) {\n\tchannel, found := d.containerDemoters[cid]\n\tif found {\n\t\tchannel <- \"stop\"\n\t\tdelete(d.containerDemoters, cid)\n\t}\n}\n\nfunc (d *demoter) stopUnusedDemoters(cs map[string]*container) {\n\tfor id := range d.containerDemoters {\n\t\tif _, found := cs[id]; !found {\n\t\t\td.stopDemoter(id)\n\t\t}\n\t}\n}\n\nfunc (d *demoter) stopDemoters() {\n\tfor cid, channel := range d.containerDemoters {\n\t\tchannel <- \"stop\"\n\t\tdelete(d.containerDemoters, cid)\n\t}\n}\n\nfunc (d *demoter) stopDirtyBitResetTimer() {\n\tif d.dirtyBitStop != nil {\n\t\tclose(d.dirtyBitStop)\n\t\td.dirtyBitStop = nil\n\t}\n}\n\nfunc (d *demoter) startDirtyBitResetTimer() {\n\tif d.dirtyBitStop != nil {\n\t\treturn\n\t}\n\n\tstop := make(chan interface{})\n\tgo func() {\n\t\tdirtyBitResetTimer := time.NewTicker(time.Duration(d.pageScanInterval))\n\t\tdirtyBitResetChan := dirtyBitResetTimer.C\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase _ = <-stop:\n\t\t\t\tif dirtyBitResetTimer != nil {\n\t\t\t\t\tdirtyBitResetTimer.Stop()\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\tcase _ = <-dirtyBitResetChan:\n\t\t\t\td.scanPages()\n\t\t\t}\n\t\t}\n\t}()\n\td.dirtyBitStop = stop\n}\n\nfunc resetDirtyBit(pid string) error {\n\t// Write magic value \"4\" to the clear_refs file. This resets the dirty bit.\n\tpath := \"/proc/\" + pid + \"/clear_refs\"\n\terr := os.WriteFile(path, []byte(\"4\"), 0600)\n\treturn err\n}\n\n// resetDirtyBit unsets soft-dirty bits for all processes in a container.\nfunc (d *demoter) resetDirtyBit(c *container) error {\n\tgroup := cgroups.Memory.Group(c.cgroupDir)\n\n\tpids, err := group.GetProcesses()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, pid := range pids {\n\t\terr = resetDirtyBit(pid)\n\t\tif err != nil {\n\t\t\tlog.Error(\"%s: failed to reset dirty but for process %s: %v\",\n\t\t\t\tc.prettyName, pid, err)\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// scanPages scans pages of tracked containers to detect idle ones.\nfunc (d *demoter) scanPages() {\n\td.migration.Lock()\n\tdefer d.migration.Unlock()\n\n\tfor _, container := range d.migration.containers {\n\t\tpm := container.GetPageMigration()\n\t\tif pm == nil {\n\t\t\tcontinue\n\t\t}\n\t\tdramNodes := pm.SourceNodes\n\t\tpmemNodes := pm.TargetNodes\n\t\tif dramNodes.Size() == 0 || pmemNodes.Size() == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Gather the known pages which need to be moved.\n\t\tpagePool, err := d.getPagesForContainer(container, dramNodes)\n\t\tif err != nil {\n\t\t\tlog.Error(\"failed to get pages for container %v\", container.prettyName)\n\t\t\tcontinue\n\t\t}\n\n\t\tcount := 0\n\t\tfor _, pages := range pagePool.pages {\n\t\t\tcount += len(pages)\n\t\t}\n\t\tlog.Debug(\"%d pages for (maybe) demoting for %v\", count, container.prettyName)\n\n\t\t// Reset the dirty bit from all pages.\n\t\td.resetDirtyBit(container)\n\n\t\t// Give the pages to the page moving goroutine. Copy the page pool so that there's no race.\n\t\td.updateDemoter(container.GetCacheID(), copyPagePool(pagePool), pmemNodes.Clone())\n\t}\n\n\td.stopUnusedDemoters(d.migration.containers)\n}\n\nfunc (d *demoter) getPagesForContainer(c *container, sourceNodes idset.IDSet) (pagePool, error) {\n\tpool := pagePool{\n\t\tpages:        make(map[int][]page, 0),\n\t\tlongestRange: 0,\n\t}\n\n\tgroup := cgroups.Memory.Group(c.cgroupDir)\n\tpids, err := group.GetProcesses()\n\tif err != nil {\n\t\treturn pagePool{}, err\n\t}\n\n\tfor _, pid := range pids {\n\t\taddressRanges := make([]addrRange, 0)\n\t\tpidNumber64, err := strconv.ParseInt(pid, 10, 32)\n\t\tif err != nil {\n\t\t\tlog.Error(\"Failed to parse addr to int: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tpidNumber := int(pidNumber64)\n\t\t// Read /proc/pid/numa_maps and /proc/pid/maps\n\t\tnumaMapsPath := \"/proc/\" + pid + \"/numa_maps\"\n\t\tnumaMapsBytes, err := os.ReadFile(numaMapsPath)\n\t\tif err != nil {\n\t\t\tlog.Error(\"Could not read numa_maps: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tmapsPath := \"/proc/\" + pid + \"/maps\"\n\t\tmapsBytes, err := os.ReadFile(mapsPath)\n\t\tif err != nil {\n\t\t\tlog.Error(\"Could not read maps: %v\\n\", err)\n\t\t\tcontinue\n\t\t}\n\t\tmapsLines := strings.Split(string(mapsBytes), \"\\n\")\n\n\t\tfor _, line := range strings.Split(string(numaMapsBytes), \"\\n\") {\n\t\t\ttokens := strings.Split(line, \" \")\n\t\t\tif len(tokens) < 3 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tattrs := strings.Join(tokens[2:], \" \")\n\t\t\t// Filter out lines which don't have \"anonymous\", since we are not\n\t\t\t// interested in file-mapped or shared pages. Save the interesting ranges.\n\t\t\t// TODO: consider dropping the \"heap\" requirement. There are often ranges\n\t\t\t// in the file which don't have any attributes indicating the memory\n\t\t\t// location.\n\t\t\tif !strings.Contains(attrs, \"heap\") || !strings.Contains(attrs, \"anon=\") {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// We only find out if *any* pages in the range are in a DRAM node. The\n\t\t\t// more fine-grained analysis is done later by running the move_pages()\n\t\t\t// system call twice.\n\t\t\tlocatedOnDRAMNode := false\n\t\t\tfor node := range sourceNodes {\n\t\t\t\tnumber := strconv.FormatInt(int64(node), 10)\n\t\t\t\tstr := \"N\" + number + \"=\"\n\t\t\t\tif strings.Contains(attrs, str) {\n\t\t\t\t\tlocatedOnDRAMNode = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !locatedOnDRAMNode {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tfor _, mapLine := range mapsLines {\n\t\t\t\tif strings.HasPrefix(mapLine, tokens[0]+\"-\") {\n\t\t\t\t\tspaceIndex := strings.Index(mapLine, \" \")\n\t\t\t\t\tif spaceIndex > len(tokens[0]+\"-\") {\n\t\t\t\t\t\tendAddrStr := mapLine[len(tokens[0]+\"-\"):spaceIndex]\n\t\t\t\t\t\tstartAddr, err := strconv.ParseInt(tokens[0], 16, 64)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tlog.Error(\"Failed to parse addr to int: %v\\n\", err)\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\tendAddr, err := strconv.ParseInt(endAddrStr, 16, 64)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tlog.Error(\"Failed to parse addr to int: %v\\n\", err)\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\trangeLength := endAddr - startAddr\n\t\t\t\t\t\taddressRanges = append(addressRanges, addrRange{uint64(startAddr), uint64(rangeLength / int64(os.Getpagesize()))})\n\t\t\t\t\t\t// log.Debug(\"found interesting page range for pid %s: %v\", pid, addressRanges[len(addressRanges)-1])\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Read /proc/pid/pagemap and process only interesting page ranges. For\n\t\t// every read-only page and for every page with the soft-dirty bit on, mark\n\t\t// them as candidates to be moved by adding them to pagePool.\n\n\t\tif len(addressRanges) > 0 {\n\t\t\t// log.Debug(\"Getting pages for PID %s for ranges %v\", pid, addressRanges)\n\t\t\tpages := make([]page, 0)\n\t\t\tpath := \"/proc/\" + pid + \"/pagemap\"\n\t\t\tpageMap, err := os.OpenFile(path, os.O_RDONLY, 0)\n\t\t\tif err != nil {\n\t\t\t\t// Probably the process just died?\n\t\t\t\tfmt.Printf(\"Could not read pagemaps: %v\\n\", err)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tfor _, addressRange := range addressRanges {\n\t\t\t\tidx := int64(addressRange.addr / uint64(os.Getpagesize()) * 8)\n\t\t\t\toffset, err := pageMap.Seek(idx, io.SeekStart)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Maybe there was a race condition and the maps changed?\n\t\t\t\t\tlog.Error(\"Failed to seek: %v\\n\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor i := uint64(0); i < addressRange.length; i++ {\n\t\t\t\t\tbytes := make([]byte, 8)\n\t\t\t\t\t// Read exactly 8 bytes (because the file interface breaks otherwise).\n\t\t\t\t\t_, err = io.ReadAtLeast(pageMap, bytes, 8)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t// Possibly the maps changed.\n\t\t\t\t\t\tlog.Error(\"Could not read data from pagemaps(%v)(page size: %d, seek offset: %d): %v\\n\", idx, os.Getpagesize(), offset, err)\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tdata := binary.LittleEndian.Uint64(bytes)\n\n\t\t\t\t\t// Check that the page is present (not swapped), exclusively\n\t\t\t\t\t// mapped (not used by any other process), and it has the\n\t\t\t\t\t// soft-dirty bit off.\n\n\t\t\t\t\t// Note: there appears to be no way to see from the pagemap entry what the NUMA node is.\n\t\t\t\t\t// We could map this back to the physical address ranges if needed. Currently this is handled\n\t\t\t\t\t// in movePages() by calling move_pages() first with an empty node array.\n\n\t\t\t\t\tsoftDirtyBit := uint64(0x1) << 55\n\t\t\t\t\texclusiveBit := uint64(0x1) << 56\n\t\t\t\t\tpresentBit := uint64(0x1) << 63\n\t\t\t\t\tpresent := (data&presentBit == presentBit)\n\t\t\t\t\texclusive := (data&exclusiveBit == exclusiveBit)\n\t\t\t\t\tsoftDirty := (data&softDirtyBit == softDirtyBit)\n\n\t\t\t\t\tif present && exclusive && !softDirty {\n\t\t\t\t\t\t// log.Debug(\"page a candidate for moving: 0x%08x\", addressRange.addr+i*uint64(os.Getpagesize()))\n\t\t\t\t\t\tpages = append(pages, page{addr: addressRange.addr + i*uint64(os.Getpagesize()), pid: pidNumber})\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, found := pool.pages[pidNumber]; found {\n\t\t\t\tpool.pages[pidNumber] = append(pool.pages[pidNumber], pages...)\n\t\t\t} else {\n\t\t\t\tpool.pages[pidNumber] = pages\n\t\t\t}\n\t\t\tif uint(len(addressRanges)) > pool.longestRange {\n\t\t\t\tpool.longestRange = uint(len(addressRanges))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn pool, nil\n}\n\nfunc pickClosestPMEMNode(targetNodes idset.IDSet) idset.ID {\n\t// TODO: analyze the topology information (and possibly the amount of free memory) and choose the \"best\"\n\t// PMEM node to demote the page to. The array targetNodes already contains only the subset of PMEM nodes\n\t// available in this topology subtree. Right now just pick a random controller.\n\tnodes := targetNodes.Members()\n\treturn nodes[rand.Intn(len(nodes))]\n}\n\nfunc (d *demoter) movePagesForPid(p []page, count uint, pid int, targetNodes idset.IDSet) (uint, error) {\n\t// We move at max count pages, but there might not be that much.\n\tnPages := count\n\tif uint(len(p)) < count {\n\t\tnPages = uint(len(p))\n\t}\n\n\t// Gather memory page pointers.\n\tpages := make([]uintptr, nPages)\n\tvar i uint\n\tfor i = 0; i < nPages; i++ {\n\t\tpages[i] = uintptr(p[i].addr)\n\t}\n\n\t// MPOL_MF_MOVE - only move pages exclusive to this process. There will be\n\t// permission denied errors for pages which couldn't be moved. FIXME: find\n\t// out if the whole move_pages() syscall failed or if just the non-exclusive\n\t// pages were not moved.\n\tflags := 1 << 1\n\n\t// Call move_pages() first with nil nodes array to find out the current controllers.\n\t_, currentStatus, err := d.pageMover.MovePagesSyscall(pid, nPages, pages, nil, flags)\n\tif err != nil {\n\t\tlog.Error(\"Failed to find out the current status of the pages: %v.\", err)\n\t\treturn 0, err\n\t}\n\n\tdramPages := make([]uintptr, 0)\n\tnodes := make([]int, 0)\n\t// Choose a target node for every page. Drop the pages which already are on the right controller from the list.\n\tfor i, pageStatus := range currentStatus {\n\t\tif pageStatus < 0 {\n\t\t\t// There was an error regarding this page.\n\t\t\tcontinue\n\t\t}\n\t\t// log.Debug(\"page 0x%08X: old status %d\", pages[i], pageStatus)\n\t\tif !targetNodes.Has(idset.ID(pageStatus)) {\n\t\t\t// In case of many PMEM controllers choose the one that is the closest.\n\t\t\tdramPages = append(dramPages, pages[i])\n\t\t\tnodes = append(nodes, int(pickClosestPMEMNode(targetNodes)))\n\t\t} // else no need to move.\n\t}\n\n\t// Call move_pages() to actually move the pages.\n\t_, _, err = d.pageMover.MovePagesSyscall(pid, uint(len(dramPages)), dramPages, nodes, flags)\n\n\t// We processed (moved or ignored) at least nPages.\n\treturn nPages, err\n}\n\nfunc (d *demoter) movePages(p pagePool, count uint, targetNodes idset.IDSet) error {\n\t// Select pid for moving the pages so that the process with the largest number\n\t// of non-dirty pages gets the pages moved first.\n\tprocessedPids := make(map[int]bool, 0)\n\n\tfor count > 0 {\n\t\tmostPagesPid := 0\n\t\tnPagesForPid := uint(0)\n\t\tfor pid, pages := range p.pages {\n\t\t\t_, alreadyProcessed := processedPids[pid]\n\t\t\tif alreadyProcessed {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif uint(len(pages)) > nPagesForPid {\n\t\t\t\tmostPagesPid = pid\n\t\t\t\tnPagesForPid = uint(len(pages))\n\t\t\t}\n\t\t}\n\n\t\tif nPagesForPid == 0 {\n\t\t\treturn nil\n\t\t}\n\n\t\tprocessedPids[mostPagesPid] = true\n\n\t\tnMovePages := nPagesForPid\n\t\tif count < nPagesForPid {\n\t\t\tnMovePages = count\n\t\t\tcount = 0\n\t\t} else {\n\t\t\tcount -= nPagesForPid\n\t\t}\n\n\t\tlog.Debug(\"moving %d pages for pid %d\", nMovePages, mostPagesPid)\n\t\tnPages, err := d.movePagesForPid(p.pages[mostPagesPid], nMovePages, mostPagesPid, targetNodes)\n\t\tif err != nil {\n\t\t\tlog.Error(\"Failed to move pages: %v\", err)\n\t\t\treturn err\n\t\t}\n\t\t// Remove processed pages from the pagemap.\n\t\tp.pages[mostPagesPid] = p.pages[mostPagesPid][nPages:]\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/page-migrate/demoter_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pagemigrate\n\nimport (\n\t\"fmt\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n\t\"testing\"\n)\n\ntype mockPageMover struct {\n\tfirstSuccess               bool\n\tsecondSuccess              bool\n\texpectedPagesForSecondCall uint\n\tfirstStatus                []int\n}\n\nfunc (m *mockPageMover) MovePagesSyscall(pid int, count uint, pages []uintptr, nodes []int, flags int) (uint, []int, error) {\n\n\tstatus := make([]int, len(pages))\n\n\tfmt.Printf(\"move_pages(): pid %d, count %d, pages %v, nodes %v, flags %d\\n\",\n\t\tpid, count, pages, nodes, flags)\n\n\tif nodes == nil {\n\t\t// First call is made without nodes\n\t\tif m.firstSuccess == false {\n\t\t\treturn 0, m.firstStatus, fmt.Errorf(\"Fake error\")\n\t\t}\n\t\treturn 0, m.firstStatus, nil\n\t}\n\n\t// Second call\n\tif m.secondSuccess == false {\n\t\treturn 0, status, fmt.Errorf(\"Fake error\")\n\t}\n\tif uint(len(pages)) != m.expectedPagesForSecondCall {\n\t\treturn 0, status, fmt.Errorf(\"Real error\")\n\t}\n\n\treturn 0, status, nil\n}\n\nfunc TestMovePages(t *testing.T) {\n\ttcases := []struct {\n\t\tname                       string\n\t\tpool                       pagePool\n\t\ttargetNodes                idset.IDSet\n\t\tpageCount                  uint\n\t\texpectedRemainingPageCount uint\n\t\texpectedError              bool\n\t\tpageMover                  PageMover\n\t\tpid                        int\n\t}{\n\t\t{\n\t\t\tname: \"move pages (both)\",\n\t\t\tpool: pagePool{\n\t\t\t\tpages: map[int][]page{\n\t\t\t\t\t500: {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xdeadbeef,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xc0ffee,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpid:       500,\n\t\t\tpageCount: 2,\n\t\t\tpageMover: &mockPageMover{\n\t\t\t\tfirstSuccess:               true,\n\t\t\t\tsecondSuccess:              true,\n\t\t\t\tfirstStatus:                []int{0, 0},\n\t\t\t\texpectedPagesForSecondCall: 2,\n\t\t\t},\n\t\t\ttargetNodes:                idset.NewIDSet(1, 2),\n\t\t\texpectedError:              false,\n\t\t\texpectedRemainingPageCount: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"move pages (only one)\",\n\t\t\tpool: pagePool{\n\t\t\t\tpages: map[int][]page{\n\t\t\t\t\t500: {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xdeadbeef,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xc0ffee,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpid:       500,\n\t\t\tpageCount: 2,\n\t\t\tpageMover: &mockPageMover{\n\t\t\t\tfirstSuccess:               true,\n\t\t\t\tsecondSuccess:              true,\n\t\t\t\tfirstStatus:                []int{0, 2},\n\t\t\t\texpectedPagesForSecondCall: 1,\n\t\t\t},\n\t\t\ttargetNodes:                idset.NewIDSet(1, 2),\n\t\t\texpectedError:              false,\n\t\t\texpectedRemainingPageCount: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"move pages (none)\",\n\t\t\tpool: pagePool{\n\t\t\t\tpages: map[int][]page{\n\t\t\t\t\t500: {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xdeadbeef,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xc0ffee,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpid:       500,\n\t\t\tpageCount: 2,\n\t\t\tpageMover: &mockPageMover{\n\t\t\t\tfirstSuccess:               true,\n\t\t\t\tsecondSuccess:              true,\n\t\t\t\tfirstStatus:                []int{2, 1},\n\t\t\t\texpectedPagesForSecondCall: 0,\n\t\t\t},\n\t\t\ttargetNodes:                idset.NewIDSet(1, 2),\n\t\t\texpectedError:              false,\n\t\t\texpectedRemainingPageCount: 0,\n\t\t},\n\t\t{\n\t\t\tname: \"move pages (count 1)\",\n\t\t\tpool: pagePool{\n\t\t\t\tpages: map[int][]page{\n\t\t\t\t\t500: {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xdeadbeef,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xc0ffee,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpid:       500,\n\t\t\tpageCount: 1,\n\t\t\tpageMover: &mockPageMover{\n\t\t\t\tfirstSuccess:               true,\n\t\t\t\tsecondSuccess:              true,\n\t\t\t\tfirstStatus:                []int{0},\n\t\t\t\texpectedPagesForSecondCall: 1,\n\t\t\t},\n\t\t\ttargetNodes:                idset.NewIDSet(1, 2),\n\t\t\texpectedError:              false,\n\t\t\texpectedRemainingPageCount: 1,\n\t\t},\n\t\t{\n\t\t\tname: \"move pages (first call error)\",\n\t\t\tpool: pagePool{\n\t\t\t\tpages: map[int][]page{\n\t\t\t\t\t500: {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xdeadbeef,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xc0ffee,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpid:       500,\n\t\t\tpageCount: 2,\n\t\t\tpageMover: &mockPageMover{\n\t\t\t\tfirstSuccess:               false,\n\t\t\t\tsecondSuccess:              true,\n\t\t\t\tfirstStatus:                []int{0, 0},\n\t\t\t\texpectedPagesForSecondCall: 0,\n\t\t\t},\n\t\t\ttargetNodes:                idset.NewIDSet(1, 2),\n\t\t\texpectedError:              true,\n\t\t\texpectedRemainingPageCount: 2,\n\t\t},\n\t\t{\n\t\t\tname: \"move pages (second call error)\",\n\t\t\tpool: pagePool{\n\t\t\t\tpages: map[int][]page{\n\t\t\t\t\t500: {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xdeadbeef,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tpid:  500,\n\t\t\t\t\t\t\taddr: 0xc0ffee,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpid:       500,\n\t\t\tpageCount: 2,\n\t\t\tpageMover: &mockPageMover{\n\t\t\t\tfirstSuccess:               true,\n\t\t\t\tsecondSuccess:              false,\n\t\t\t\tfirstStatus:                []int{0, 0},\n\t\t\t\texpectedPagesForSecondCall: 0,\n\t\t\t},\n\t\t\ttargetNodes:                idset.NewIDSet(1, 2),\n\t\t\texpectedError:              true,\n\t\t\texpectedRemainingPageCount: 2,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tdynamicDemoter := &demoter{\n\t\t\t\tmaxPageMoveCount: tc.pageCount,\n\t\t\t\tpageMover:        tc.pageMover,\n\t\t\t}\n\n\t\t\terr := dynamicDemoter.movePages(tc.pool, tc.pageCount, tc.targetNodes)\n\t\t\tif err != nil {\n\t\t\t\tif err.Error() != \"Fake error\" {\n\t\t\t\t\tt.Errorf(\"Non-fake error: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (err != nil) != tc.expectedError {\n\t\t\t\tt.Errorf(\"Unexpected error value\")\n\t\t\t}\n\n\t\t\tif uint(len(tc.pool.pages[tc.pid])) != tc.expectedRemainingPageCount {\n\t\t\t\tt.Errorf(\"Wrong number of remaining pages: %d\", len(tc.pool.pages[tc.pid]))\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/page-migrate/flags.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pagemigrate\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// options captures our configurable controller parameters.\ntype options struct {\n\t// PageScanInterval controls how much time we give containers to touch non-idle pages.\n\tPageScanInterval config.Duration\n\t// PageMoveInterval controls how often we trigger moving pages.\n\tPageMoveInterval config.Duration\n\t// MaxPageMoveCount controls how many pages we can move in a single go.\n\tMaxPageMoveCount uint\n}\n\n// Our runtime configuration.\nvar opt = defaultOptions().(*options)\n\n// defaultOptions returns a new options instance, all initialized to defaults.\nfunc defaultOptions() interface{} {\n\treturn &options{}\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tconfig.Register(PageMigrationConfigPath, PageMigrationDescription, opt, defaultOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/page-migrate/page-migrate.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pagemigrate\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// PageMigrationController is the name/domain of the page migration controller.\n\tPageMigrationController = cache.PageMigration\n\t// PageMigrationConfigPath is the configuration path for the page migration controller.\n\tPageMigrationConfigPath = \"resource-manager.control.\" + PageMigrationController\n\t// PageMigrationDescription is the description for the page migration controller.\n\tPageMigrationDescription = \"page migration controller\"\n)\n\n// migration implements the controller for memory page migration.\ntype migration struct {\n\tcache      cache.Cache           // resource manager cache\n\tsync.Mutex                       // protect access from multiple goroutines\n\tcontainers map[string]*container // containers we migrate\n\tdemoter    *demoter              // demoter adopted from topology-aware policy\n}\n\n//\n// The resource manager serializes access to the cache during request\n// processing, event processing, and configuration updates by locking\n// the resource-manager for each of these. Since controller hooks are\n// invoked either as part of processing a request or an event, access\n// to the cache from hooks is properly serialized.\n//\n// Page scanning or migration on the other hand happen asynchronously\n// from dedicated goroutines. In order to avoid having to serialize\n// access to the cache for these, we track and cache locally just enough\n// data about containers that we can perform these actions completely on\n// our own, without the need to access the resource manager cache at all.\n//\n// An alternative would have been to duplicate what we had originally in\n// the policy:\n//  - introduce controller events akin to policy events\n//  - have the resource-manager call controller event handlers with the\n//    lock held\n//  - periodically inject a controller event when we want to scan pages\n//  - perform page scanning or demotion from the event handler with the\n//    resource-manager lock held\n//\n// However that would have destroyed one of the goals of splitting page\n// scanning and migration out to a controller of its own, which was to\n// perform these potentially time consuming actions without blocking\n// concurrent processing of requests or events.\n//\n\n// container is the per container data we track locally.\ntype container struct {\n\tcacheID    string\n\tid         string\n\tprettyName string\n\tcgroupDir  string\n\tpm         *cache.PageMigrate\n}\n\n// Our logger instance.\nvar log = logger.NewLogger(PageMigrationController)\n\n// Our singleton page migration controller.\nvar singleton *migration\n\n// getMigrationController returns our singleton controller instance.\nfunc getMigrationController() *migration {\n\tif singleton == nil {\n\t\tsingleton = &migration{\n\t\t\tcontainers: make(map[string]*container),\n\t\t}\n\t\tsingleton.demoter = newDemoter(singleton)\n\t}\n\treturn singleton\n}\n\n// Start prepares the controller for resource control/decision enforcement.\nfunc (m *migration) Start(cache cache.Cache, _ client.Client) error {\n\tm.cache = cache\n\tm.syncWithCache()\n\tm.demoter.Reconfigure()\n\treturn nil\n}\n\n// Stop shuts down the controller.\nfunc (m *migration) Stop() {\n\tm.demoter.Stop()\n}\n\n// PreCreateHook is the controller's pre-create hook.\nfunc (m *migration) PreCreateHook(cache.Container) error {\n\treturn nil\n}\n\n// PreStartHook is the controller's pre-start hook.\nfunc (m *migration) PreStartHook(cache.Container) error {\n\treturn nil\n}\n\n// PostStartHook is the controller's post-start hook.\nfunc (m *migration) PostStartHook(cc cache.Container) error {\n\tm.Lock()\n\tdefer m.Unlock()\n\terr := m.insertContainer(cc)\n\tcc.ClearPending(PageMigrationController)\n\treturn err\n}\n\n// PostUpdateHook is the controller's post-update hook.\nfunc (m *migration) PostUpdateHook(cc cache.Container) error {\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.updateContainer(cc)\n\tcc.ClearPending(PageMigrationController)\n\treturn nil\n}\n\n// PostStopHook is the controller's post-stop hook.\nfunc (m *migration) PostStopHook(cc cache.Container) error {\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.deleteContainer(cc)\n\treturn nil\n}\n\n// syncWithCache synchronizes tracked containers with the cache.\nfunc (m *migration) syncWithCache() {\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.containers = make(map[string]*container)\n\tfor _, cc := range m.cache.GetContainers() {\n\t\tm.insertContainer(cc)\n\t}\n}\n\n// insertContainer creates a local copy of the container.\nfunc (m *migration) insertContainer(cc cache.Container) error {\n\tpm := cc.GetPageMigration()\n\tif pm == nil {\n\t\treturn nil\n\t}\n\n\tc := &container{\n\t\tcacheID:    cc.GetCacheID(),\n\t\tid:         cc.GetID(),\n\t\tprettyName: cc.PrettyName(),\n\t\tcgroupDir:  cc.GetCgroupDir(),\n\t\tpm:         pm.Clone(),\n\t}\n\tif c.cgroupDir == \"\" {\n\t\treturn migrationError(\"can't find cgroup dir for container %s\",\n\t\t\tc.prettyName)\n\t}\n\n\tm.containers[c.cacheID] = c\n\n\treturn nil\n}\n\n// updateContainer updates the local copy of the container.\nfunc (m *migration) updateContainer(cc cache.Container) error {\n\tpm := cc.GetPageMigration()\n\tif pm == nil {\n\t\tdelete(m.containers, cc.GetCacheID())\n\t\treturn nil\n\t}\n\n\tc, ok := m.containers[cc.GetCacheID()]\n\tif !ok {\n\t\treturn m.insertContainer(cc)\n\t}\n\n\tc.pm = pm.Clone()\n\treturn nil\n}\n\n// deleteContainer creates a local copy of the container.\nfunc (m *migration) deleteContainer(cc cache.Container) error {\n\tdelete(m.containers, cc.GetCacheID())\n\treturn nil\n}\n\n// GetCacheID replicates the respective cache.Container function.\nfunc (c *container) GetCacheID() string {\n\treturn c.cacheID\n}\n\n// GetID replicates the respective cache.Container function.\nfunc (c *container) GetID() string {\n\treturn c.id\n}\n\n// GetCgroupDir replicates the respective cache.Container function.\nfunc (c *container) GetCgroupDir() string {\n\treturn c.GetCgroupDir()\n}\n\n// GetPageMigration replicates the respective cache.Container function.\nfunc (c *container) GetPageMigration() *cache.PageMigrate {\n\treturn c.pm\n}\n\n// PrettyName replicates the respective cache.Container function.\nfunc (c *container) PrettyName() string {\n\treturn c.prettyName\n}\n\n// init registers this controller.\nfunc init() {\n\tcontrol.Register(PageMigrationController, \"page migration controller\", getMigrationController())\n}\n\n// migrationError creates a controller-specific formatted error message.\nfunc migrationError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"page-migrate: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/page-migrate/page-mover.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pagemigrate\n\nimport \"C\"\n\nimport (\n\t\"fmt\"\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\ntype linuxPageMover struct{}\n\n// PageMover implements the way to move pages in a given HW/SW platform.\ntype PageMover interface {\n\tMovePagesSyscall(pid int, count uint, pages []uintptr, nodes []int, flags int) (uint, []int, error)\n}\n\nfunc (m *linuxPageMover) MovePagesSyscall(pid int, count uint, pages []uintptr, nodes []int, flags int) (uint, []int, error) {\n\n\t// syscall:\n\t// long move_pages(int pid, unsigned long count, void **pages,\n\t//                 const int *nodes, int *status, int flags);\n\n\tvar err error\n\n\tif count == 0 {\n\t\treturn 0, []int{}, nil\n\t}\n\n\t// Go int is 64 bits on a 64-bit system, but C int is only guaranteed to be at least 16 bits, typically 32.\n\tcNodes := make([]C.int, len(nodes))\n\tfor i := 0; i < len(nodes); i++ {\n\t\tif nodes[i] < 0 || nodes[i] > 32767 {\n\t\t\treturn 0, []int{}, fmt.Errorf(\"int value error: %d\", nodes[i])\n\t\t}\n\t\tcNodes[i] = C.int(nodes[i]) // safe downcast\n\t}\n\n\tcStatus := make([]C.int, len(pages))\n\n\tnodesPtr := unsafe.Pointer(nil)\n\tif nodes != nil {\n\t\tnodesPtr = unsafe.Pointer(&cNodes[0])\n\t}\n\n\tret, _, en := unix.Syscall6(unix.SYS_MOVE_PAGES, uintptr(pid), uintptr(count), uintptr(unsafe.Pointer(&pages[0])), uintptr(nodesPtr), uintptr(unsafe.Pointer(&cStatus[0])), uintptr(flags))\n\tif en != 0 {\n\t\terr = unix.Errno(en)\n\t}\n\n\t// log.Debug(\"move_pages(): pid %d, count %d, pages %v, nodes %v, flags %d: return value %d, status %d, errno %v\",\n\t// \tpid, count, pages, nodes, flags, uint(ret), cStatus, err)\n\n\tstatus := make([]int, count)\n\tfor i := uint(0); i < count; i++ {\n\t\tstatus[i] = int(cStatus[i])\n\t}\n\n\treturn uint(ret), status, err\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/control/rdt/rdt.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage rdt\n\nimport (\n\t\"fmt\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/client\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/metrics\"\n\t\"github.com/intel/goresctrl/pkg/rdt\"\n)\n\nconst (\n\t// ConfigModuleName is the configuration section for RDT\n\tConfigModuleName = \"rdt\"\n\n\t// RDTController is the name of the RDT controller.\n\tRDTController = cache.RDT\n\n\tresctrlGroupPrefix = \"cri-resmgr.\"\n)\n\n// rdtctl encapsulates the runtime state of our RTD enforcement/controller.\ntype rdtctl struct {\n\tcache        cache.Cache   // resource manager cache\n\tnoQoSClasses bool          // true if mapping pod qos class to rdt class is disabled\n\tmode         OperatingMode // track the mode here to capture mode changes\n\topt          *config\n}\n\ntype config struct {\n\trdt.Config\n\n\tOptions struct {\n\t\trdt.Options\n\n\t\tMode               OperatingMode `json:\"mode\"`\n\t\tMonitoringDisabled bool          `json:\"monitoringDisabled\"`\n\t} `json:\"options\"`\n}\n\ntype OperatingMode string\n\nconst (\n\tOperatingModeDisabled  OperatingMode = \"Disabled\"\n\tOperatingModeDiscovery OperatingMode = \"Discovery\"\n\tOperatingModeFull      OperatingMode = \"Full\"\n)\n\n// Our logger instance.\nvar log logger.Logger = logger.NewLogger(RDTController)\n\n// our RDT controller singleton instance.\nvar singleton *rdtctl\n\n// getRDTController returns our singleton RDT controller instance.\nfunc getRDTController() *rdtctl {\n\tif singleton == nil {\n\t\tsingleton = &rdtctl{}\n\t\tsingleton.opt = singleton.defaultOptions().(*config)\n\t}\n\treturn singleton\n}\n\n// Start initializes the controller for enforcing decisions.\nfunc (ctl *rdtctl) Start(cache cache.Cache, _ client.Client) error {\n\tif err := rdt.Initialize(resctrlGroupPrefix); err != nil {\n\t\treturn rdtError(\"failed to initialize RDT controls: %v\", err)\n\t}\n\n\tctl.cache = cache\n\n\tif err := ctl.configure(); err != nil {\n\t\t// Just print an error. A config update later on may be valid.\n\t\tlog.Error(\"failed apply initial configuration: %v\", err)\n\t}\n\n\trdt.RegisterCustomPrometheusLabels(\"pod_name\", \"container_name\")\n\terr := metrics.RegisterCollector(\"rdt\", rdt.NewCollector)\n\tif err != nil {\n\t\tlog.Error(\"failed register rdt collector: %v\", err)\n\t}\n\n\tpkgcfg.GetModule(ConfigModuleName).AddNotify(getRDTController().configNotify)\n\n\treturn nil\n}\n\n// Stop shuts down the controller.\nfunc (ctl *rdtctl) Stop() {\n}\n\n// PreCreateHook is the RDT controller pre-create hook.\nfunc (ctl *rdtctl) PreCreateHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PreStartHook is the RDT controller pre-start hook.\nfunc (ctl *rdtctl) PreStartHook(_ cache.Container) error {\n\treturn nil\n}\n\n// PostStartHook is the RDT controller post-start hook.\nfunc (ctl *rdtctl) PostStartHook(c cache.Container) error {\n\tif !c.HasPending(RDTController) {\n\t\treturn nil\n\t}\n\n\tif err := ctl.assign(c); err != nil {\n\t\treturn err\n\t}\n\n\tc.ClearPending(RDTController)\n\n\treturn nil\n}\n\n// PostUpdateHook is the RDT controller post-update hook.\nfunc (ctl *rdtctl) PostUpdateHook(c cache.Container) error {\n\tif !c.HasPending(RDTController) {\n\t\treturn nil\n\t}\n\n\tif err := ctl.assign(c); err != nil {\n\t\treturn err\n\t}\n\n\tc.ClearPending(RDTController)\n\n\treturn nil\n}\n\n// PostStop is the RDT controller post-stop hook.\nfunc (ctl *rdtctl) PostStopHook(c cache.Container) error {\n\tif err := ctl.stopMonitor(c); err != nil {\n\t\treturn rdtError(\"%q: failed to remove monitoring group: %v\", c.PrettyName(), err)\n\t}\n\treturn nil\n}\n\n// assign assigns all processes/threads in a container to the correct class\nfunc (ctl *rdtctl) assign(c cache.Container) error {\n\tif ctl.opt.Options.Mode == OperatingModeDisabled {\n\t\treturn nil\n\t}\n\n\tclass := c.GetRDTClass()\n\tswitch class {\n\tcase \"\":\n\t\tclass = rdt.RootClassName\n\tcase cache.RDTClassPodQoS:\n\t\tif ctl.noQoSClasses {\n\t\t\tclass = rdt.RootClassName\n\t\t} else {\n\t\t\tclass = string(c.GetQOSClass())\n\t\t}\n\t}\n\n\terr := ctl.assignClass(c, class)\n\tif err != nil && class != rdt.RootClassName {\n\t\tlog.Warn(\"%v; falling back to system root class\", err)\n\t\treturn ctl.assignClass(c, rdt.RootClassName)\n\t}\n\treturn err\n}\n\n// assignClass assigns all processes/threads in a container to the specified class\nfunc (ctl *rdtctl) assignClass(c cache.Container, class string) error {\n\tcls, ok := rdt.GetClass(class)\n\tif !ok {\n\t\treturn rdtError(\"%q: unknown RDT class %q\", c.PrettyName(), class)\n\t}\n\n\tpod, ok := c.GetPod()\n\tif !ok {\n\t\treturn rdtError(\"%q: failed to get pod\", c.PrettyName())\n\t}\n\n\tpids, err := c.GetProcesses()\n\tif err != nil {\n\t\treturn rdtError(\"%q: failed to get process list: %v\", c.PrettyName(), err)\n\t}\n\n\tif err := cls.AddPids(pids...); err != nil {\n\t\treturn rdtError(\"%q: failed to assign to class %q: %v\", c.PrettyName(), class, err)\n\t}\n\n\tpretty := c.PrettyName()\n\tif _, ok := cls.GetMonGroup(pretty); !ok || ctl.monitoringDisabled() {\n\t\tctl.stopMonitor(c)\n\t}\n\n\tif !ctl.monitoringDisabled() {\n\t\tpname, name, id := pod.GetName(), c.GetName(), c.GetID()\n\t\tif err := ctl.monitor(cls, pname, name, id, pretty, pids); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tlog.Info(\"%q: assigned to class %q\", pretty, class)\n\n\treturn nil\n}\n\n// monitor starts monitoring a container.\nfunc (ctl *rdtctl) monitor(cls rdt.CtrlGroup, pod, name, id, pretty string, pids []string) error {\n\tif !rdt.MonSupported() {\n\t\treturn nil\n\t}\n\n\tannotations := map[string]string{\"pod_name\": pod, \"container_name\": name}\n\tif mg, err := cls.CreateMonGroup(id, annotations); err != nil {\n\t\tlog.Warn(\"%q: failed to create monitoring group: %v\", pretty, err)\n\t} else {\n\t\tif err := mg.AddPids(pids...); err != nil {\n\t\t\treturn rdtError(\"%q: failed to assign to monitoring group %q: %v\",\n\t\t\t\tpretty, cls.Name()+\"/\"+mg.Name(), err)\n\t\t}\n\t\tlog.Info(\"%q: assigned to monitoring group %q\", pretty, cls.Name()+\"/\"+mg.Name())\n\t}\n\treturn nil\n}\n\n// stopMonitor stops monitoring a container.\nfunc (ctl *rdtctl) stopMonitor(c cache.Container) error {\n\tname := c.PrettyName()\n\tfor _, cls := range rdt.GetClasses() {\n\t\tif mg, ok := cls.GetMonGroup(name); ok {\n\t\t\tif err := cls.DeleteMonGroup(name); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlog.Info(\"%q: removed monitoring group %q\",\n\t\t\t\tc.PrettyName(), cls.Name()+\"/\"+mg.Name())\n\t\t}\n\t}\n\treturn nil\n}\n\n// stopMonitorAll removes all monitoring groups\nfunc (ctl *rdtctl) stopMonitorAll() error {\n\tfor _, cls := range rdt.GetClasses() {\n\t\tif err := cls.DeleteMonGroups(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (ctl *rdtctl) assignAll(forceClass string) {\n\t// Assign all containers\n\tfor _, c := range ctl.cache.GetContainers() {\n\t\tvar err error\n\t\tif forceClass != \"\" {\n\t\t\terr = ctl.assignClass(c, forceClass)\n\t\t} else {\n\t\t\terr = ctl.assign(c)\n\t\t}\n\t\tif err != nil {\n\t\t\tlog.Warn(\"failed to assign rdt class of %q: %v\", c.PrettyName(), err)\n\t\t}\n\t}\n\n}\n\nfunc (ctl *rdtctl) monitoringDisabled() bool {\n\treturn ctl.mode == OperatingModeDisabled || ctl.opt.Options.MonitoringDisabled\n}\n\nfunc (ctl *rdtctl) configure() error {\n\t// Apply RDT configuration, depending on the operating mode\n\tswitch ctl.opt.Options.Mode {\n\tcase OperatingModeDisabled:\n\t\tif ctl.mode != ctl.opt.Options.Mode {\n\t\t\tctl.stopMonitorAll()\n\t\t\t// Drop all cri-resctrl specific groups by applying an empty config\n\t\t\tif err := rdt.SetConfig(&rdt.Config{}, true); err != nil {\n\t\t\t\treturn rdtError(\"failed apply empty rdt config: %v\", err)\n\t\t\t}\n\t\t\tctl.noQoSClasses = true\n\t\t\tctl.mode = ctl.opt.Options.Mode\n\t\t\tctl.assignAll(rdt.RootClassName)\n\t\t}\n\tcase OperatingModeDiscovery:\n\t\tif ctl.mode != ctl.opt.Options.Mode {\n\t\t\tctl.stopMonitorAll()\n\t\t\t// Drop all cri-resctrl specific groups by applying an empty config\n\t\t\tif err := rdt.SetConfig(&rdt.Config{}, true); err != nil {\n\t\t\t\treturn rdtError(\"failed apply empty rdt config: %v\", err)\n\t\t\t}\n\t\t}\n\t\t// Run Initialize with empty prefix to discover existing resctrl groups\n\t\tif err := rdt.DiscoverClasses(\"\"); err != nil {\n\t\t\treturn rdtError(\"failed to discover classes from fs: %v\", err)\n\t\t}\n\n\t\t// Disable mapping from Pod QoS to RDT class if no Pod QoS class equivalents exist\n\t\tctl.noQoSClasses = true\n\t\tcs := []corev1.PodQOSClass{corev1.PodQOSBestEffort, corev1.PodQOSBurstable, corev1.PodQOSGuaranteed}\n\t\tfor _, c := range cs {\n\t\t\tif _, ok := rdt.GetClass(string(c)); ok {\n\t\t\t\tctl.noQoSClasses = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tctl.mode = ctl.opt.Options.Mode\n\t\tctl.assignAll(\"\")\n\tcase OperatingModeFull:\n\t\tif ctl.mode != ctl.opt.Options.Mode {\n\t\t\tctl.stopMonitorAll()\n\t\t}\n\n\t\t// Copy goresctrl specific part from our extended options\n\t\tctl.opt.Config.Options = ctl.opt.Options.Options\n\t\tif err := rdt.SetConfig(&ctl.opt.Config, true); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Disable mapping from Pod QoS to RDT class if no classes have been defined\n\t\tctl.noQoSClasses = len(rdt.GetClasses()) <= 1\n\t\tctl.mode = ctl.opt.Options.Mode\n\t\tctl.assignAll(\"\")\n\tdefault:\n\t\treturn rdtError(\"invalid mode %q\", ctl.opt.Options.Mode)\n\t}\n\n\tlog.Debug(\"rdt controller operating mode set to %q\", ctl.mode)\n\n\tif ctl.opt.Options.Mode != OperatingModeDisabled {\n\t\tlog.Debug(\"rdt monitoring %s\", map[bool]string{true: \"disabled\", false: \"enabled\"}[ctl.monitoringDisabled()])\n\t}\n\n\treturn nil\n}\n\n// configNotify is our runtime configuration notification callback.\nfunc (ctl *rdtctl) configNotify(_ pkgcfg.Event, _ pkgcfg.Source) error {\n\tlog.Info(\"configuration update, applying new config\")\n\treturn ctl.configure()\n}\n\nfunc (ctl *rdtctl) defaultOptions() interface{} {\n\tc := &config{}\n\tc.Options.Mode = OperatingModeFull\n\treturn c\n}\n\n// GetClasses returns all available RDT classes\nfunc GetClasses() []rdt.CtrlGroup {\n\treturn rdt.GetClasses()\n}\n\n// rdtError creates an RDT-controller-specific formatted error message.\nfunc rdtError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"rdt: \"+format, args...)\n}\n\n// Register us as a controller.\nfunc init() {\n\tcontrol.Register(RDTController, \"RDT controller\", getRDTController())\n\tpkgcfg.Register(ConfigModuleName, \"RDT control\", getRDTController().opt, getRDTController().defaultOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/controllers.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t// List of controllers to pull in.\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/blockio\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/cpu\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/cri\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/memory\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/page-migrate\"\n\t_ \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/rdt\"\n)\n"
  },
  {
    "path": "pkg/cri/resource-manager/error.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"fmt\"\n)\n\n// resmgrError creates a resource manager-specific formatted error.\nfunc resmgrError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"resource-manager: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/events/events.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage events\n\n// Metrics is a set of metrics-related events we might need to act upon.\ntype Metrics struct {\n\t// Avx describes changes in container AVX512 instruction usage.\n\tAvx *Avx\n}\n\n// AVX contains data related to container AVX512 instruction usage.\ntype Avx struct {\n\t// Updates contains containers with a change in their AVX512 instruction usage.\n\tUpdates map[string]bool\n}\n\n// Policy is a policy-specific event to be handled by the active policy.\ntype Policy struct {\n\t// Event is the policy-specific type of this event.\n\tType string\n\t// Source describes where this event is originated from.\n\tSource string\n\t// Data is any optional arbitrary data associated with this event.\n\tData interface{}\n}\n\nconst (\n\t// ContainerStarted is delivered to policies when a StartContainer request succeeds.\n\tContainerStarted = \"container-started\"\n)\n"
  },
  {
    "path": "pkg/cri/resource-manager/events.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/metrics\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\n// Our logger instance for events.\nvar evtlog = logger.NewLogger(\"events\")\n\n// setupEventProcessing sets up event and metrics processing.\nfunc (m *resmgr) setupEventProcessing() error {\n\tvar err error\n\n\tm.events = make(chan interface{}, 8)\n\tm.stop = make(chan interface{})\n\toptions := metrics.Options{\n\t\tPollInterval: opt.MetricsTimer,\n\t\tEvents:       m.events,\n\t}\n\tif m.metrics, err = metrics.NewMetrics(options); err != nil {\n\t\treturn resmgrError(\"failed to create metrics (pre)processor: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// startEventProcessing starts event and metrics processing.\nfunc (m *resmgr) startEventProcessing() error {\n\tif err := m.metrics.Start(); err != nil {\n\t\treturn resmgrError(\"failed to start metrics (pre)processor: %v\", err)\n\t}\n\n\tstop := m.stop\n\tgo func() {\n\t\tvar rebalanceTimer *time.Ticker\n\t\tvar rebalanceChan <-chan time.Time\n\n\t\tif opt.RebalanceTimer > 0 {\n\t\t\trebalanceTimer = time.NewTicker(opt.RebalanceTimer)\n\t\t\trebalanceChan = rebalanceTimer.C\n\t\t} else {\n\t\t\tm.Info(\"periodic rebalancing is disabled\")\n\t\t}\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase _ = <-stop:\n\t\t\t\tif rebalanceTimer != nil {\n\t\t\t\t\trebalanceTimer.Stop()\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\tcase event := <-m.events:\n\t\t\t\tm.processEvent(event)\n\t\t\tcase _ = <-rebalanceChan:\n\t\t\t\tif err := m.RebalanceContainers(); err != nil {\n\t\t\t\t\tevtlog.Error(\"rebalancing failed: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tlogger.Flush()\n\t\t}\n\t}()\n\n\treturn nil\n}\n\n// stopEventProcessing stops event and metrics processing.\nfunc (m *resmgr) stopEventProcessing() {\n\tif m.stop != nil {\n\t\tclose(m.stop)\n\t\tm.metrics.Stop()\n\t\tm.stop = nil\n\t}\n}\n\n// SendEvent injects the given event to the resource manager's event processing loop.\nfunc (m *resmgr) SendEvent(event interface{}) error {\n\tif m.events == nil {\n\t\treturn resmgrError(\"can't send event, no event channel\")\n\t}\n\tselect {\n\tcase m.events <- event:\n\t\treturn nil\n\tdefault:\n\t\treturn resmgrError(\"can't send event of type %T, event channel full\", event)\n\t}\n}\n\n// processEvent processes the given event.\nfunc (m *resmgr) processEvent(e interface{}) {\n\tevtlog.Debug(\"received event of type %T...\", e)\n\n\tswitch event := e.(type) {\n\tcase string:\n\t\tevtlog.Debug(\"'%s'...\", event)\n\tcase *events.Metrics:\n\t\tm.processAvx(event.Avx)\n\tcase *events.Policy:\n\t\tm.DeliverPolicyEvent(event)\n\tdefault:\n\t\tevtlog.Warn(\"event of unexpected type %T...\", e)\n\t}\n}\n\n// processAvx processes AVX512 events.\nfunc (m *resmgr) processAvx(e *events.Avx) bool {\n\tif e == nil {\n\t\treturn false\n\t}\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tchanges := false\n\tfor cgroup, active := range e.Updates {\n\t\tc, ok := m.resolveCgroupPath(cgroup)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\t// XXX This is just for testing, we should effectively drive state transitions\n\t\t//     through a low-pass filter.\n\t\tif active {\n\t\t\tif _, wasTagged := c.SetTag(cache.TagAVX512, \"true\"); !wasTagged {\n\t\t\t\tevtlog.Info(\"container %s STARTED using AVX512 instructions\", c.PrettyName())\n\t\t\t}\n\t\t} else {\n\t\t\tif _, wasTagged := c.DeleteTag(cache.TagAVX512); wasTagged {\n\t\t\t\tevtlog.Info(\"container %s STOPPED using AVX512 instructions\", c.PrettyName())\n\t\t\t}\n\t\t}\n\t}\n\treturn changes\n}\n\n// resolveCgroupPath resolves a cgroup path to a container.\nfunc (m *resmgr) resolveCgroupPath(path string) (cache.Container, bool) {\n\treturn m.cache.LookupContainerByCgroup(path)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/flags.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"flag\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/relay\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/sockets\"\n\t\"github.com/intel/cri-resource-manager/pkg/pidfile\"\n)\n\n// Options captures our command line parameters.\ntype options struct {\n\tHostRoot              string\n\tImageSocket           string\n\tRuntimeSocket         string\n\tRelaySocket           string\n\tRelayDir              string\n\tAllowUntestedRuntimes bool\n\tAgentSocket           string\n\tConfigSocket          string\n\tPidFile               string\n\tResctrlPath           string\n\tFallbackConfig        string\n\tForceConfig           string\n\tForceConfigSignal     string\n\tDisablePolicySwitch   bool\n\tResetPolicy           bool\n\tResetConfig           bool\n\tMetricsTimer          time.Duration\n\tRebalanceTimer        time.Duration\n\tDisableUI             bool\n}\n\n// Relay command line options.\nvar opt = options{}\n\nconst (\n\tallowUntestedRuntimesFlag = \"allow-untested-runtimes\"\n)\n\n// Register us for command line option processing.\nfunc init() {\n\tflag.StringVar(&opt.HostRoot, \"host-root\", \"\",\n\t\t\"Directory prefix under which the host's sysfs, etc. are mounted.\")\n\n\tflag.StringVar(&opt.RuntimeSocket, \"runtime-socket\", sockets.Containerd,\n\t\t\"Unix domain socket path where CRI runtime service requests should be relayed to.\")\n\tflag.StringVar(&opt.ImageSocket, \"image-socket\", relay.DefaultImageSocket,\n\t\t\"CRI image service socket, defaults to the value used for --runtime-socket.\")\n\tflag.StringVar(&opt.RelaySocket, \"relay-socket\", sockets.ResourceManagerRelay,\n\t\t\"Unix domain socket path where the resource manager should serve requests on.\")\n\tflag.StringVar(&opt.RelayDir, \"relay-dir\", \"/var/lib/cri-resmgr\",\n\t\t\"Permanent storage directory path for the resource manager to store its state in.\")\n\tflag.BoolVar(&opt.AllowUntestedRuntimes, allowUntestedRuntimesFlag, false,\n\t\t\"Allow proxying for untested CRI runtimes. Usually this is not a good idea.\")\n\n\tflag.StringVar(&opt.AgentSocket, \"agent-socket\", sockets.ResourceManagerAgent,\n\t\t\"local socket of the cri-resmgr agent to connect\")\n\tflag.StringVar(&opt.ConfigSocket, \"config-socket\", sockets.ResourceManagerConfig,\n\t\t\"Unix domain socket path where the resource manager listens for cri-resmgr-agent\")\n\tflag.StringVar(&opt.PidFile, \"pid-file\", pidfile.GetPath(),\n\t\t\"PID file to write daemon PID to\")\n\tflag.StringVar(&opt.FallbackConfig, \"fallback-config\", \"\",\n\t\t\"Fallback configuration to use unless/until one is available from the cache or agent.\")\n\tflag.StringVar(&opt.ForceConfig, \"force-config\", \"\",\n\t\t\"Configuration used to override the one stored in the cache. Disables the agent.\")\n\tflag.StringVar(&opt.ForceConfigSignal, \"force-config-signal\", \"SIGHUP\",\n\t\t\"Signal used to reload forced configuration.\")\n\tflag.BoolVar(&opt.ResetConfig, \"reset-config\", false,\n\t\t\"Remove configuration (from the agent) stored in the cache, then exit.\")\n\n\tflag.BoolVar(&opt.ResetPolicy, \"reset-policy\", false,\n\t\t\"Reset policy data stored in the cache, then exit.\")\n\tflag.BoolVar(&opt.DisablePolicySwitch, \"disable-policy-switch\", false,\n\t\t\"Disable switching policies during startup.\")\n\n\tflag.DurationVar(&opt.MetricsTimer, \"metrics-interval\", 0,\n\t\t\"Interval for polling/gathering runtime metrics data. Use 'disable' for disabling.\")\n\tflag.DurationVar(&opt.RebalanceTimer, \"rebalance-interval\", 0,\n\t\t\"Minimum interval between two container rebalancing attempts. Use 'disable' for disabling.\")\n\n\tflag.BoolVar(&opt.DisableUI, \"disable-ui\", false,\n\t\t\"Disable serving container placement visualization UIs.\")\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/introspect/introspect.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage introspect\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sync\"\n\n\txhttp \"github.com/intel/cri-resource-manager/pkg/instrumentation/http\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n)\n\n// Pod describes a single pod and its containers.\ntype Pod struct {\n\tID         string                // pod CRI ID\n\tUID        string                // pod kubernetes ID\n\tName       string                // pod name\n\tContainers map[string]*Container // containers of this pod\n}\n\n// Container describes a single container.\ntype Container struct {\n\tID            string        // container CRI ID\n\tName          string        // container name\n\tCommand       []string      // command\n\tArgs          []string      // and its arguments\n\tCPURequest    int64         // CPU requested in milli-CPU (guaranteed amount)\n\tCPULimit      int64         // CPU limit in milli-CPU (maximum allowed CPU)\n\tMemoryRequest int64         // memory requested in bytes\n\tMemoryLimit   int64         // memory limit in bytes (maximum allowed memory)\n\tHints         TopologyHints // topology/allocation hints\n}\n\n// TopologyHints contain a set of allocation hints for a container.\ntype TopologyHints topology.Hints\n\n// Assignment describes resource assignments for a single container.\ntype Assignment struct {\n\tContainerID   string // ID of container for this assignment\n\tSharedCPUs    string // shared CPUs\n\tCPUShare      int    // CPU share/weight for SharedCPUs\n\tExclusiveCPUs string // exclusive CPUs\n\tMemory        string // memory controllers\n\tPool          string // pool container is assigned to\n}\n\n// Pool describes a single (resource) pool.\ntype Pool struct {\n\tName     string   // pool name\n\tCPUs     string   // CPUs in this pool\n\tMemory   string   // memory controllers (NUMA nodes) for this pool\n\tParent   string   // parent pool\n\tChildren []string // child pools\n}\n\n// Socket describes a single physical CPU socket in the system.\ntype Socket struct {\n\tID   int    // CPU ID\n\tCPUs string // CPUs in this socket\n}\n\n// Node describes a single NUMA node in the system.\ntype Node struct {\n\tID   int    // node ID\n\tCPUs string // CPUs with locality for this NUMA node.\n}\n\n// System describes the underlying HW/system.\ntype System struct {\n\tSockets        map[int]*Socket // physical sockets in the system\n\tNodes          map[int]*Node   // NUMA nodes in the system\n\tIsolated       string          // kernel-isolated CPUs\n\tOfflined       string          // CPUs offline\n\tRDTClasses     []string        // list of RDT classes\n\tBlockIOClasses []string        // list of block I/O classes\n\tPolicy         string          // active policy\n}\n\n// State is the current introspected state of the resource manager.\ntype State struct {\n\tPools       map[string]*Pool       // pools\n\tPods        map[string]*Pod        // pods and containers\n\tAssignments map[string]*Assignment // resource assignments\n\tSystem      *System                // info about hardware/system\n\tError       string\n}\n\n// our logger instance\nvar log = logger.NewLogger(\"instrospect\")\n\n// Server is our server for external introspection.\ntype Server struct {\n\tsync.RWMutex                 // need to protect against concurrent introspection/update\n\tmux          *xhttp.ServeMux // our HTTP request multiplexer\n\tstate        *State          // introspection data\n\tdata         string          // state as a JSON string\n\tready        bool\n}\n\n// Setup prepares the given HTTP request multiplexer for serving introspection.\nfunc Setup(mux *xhttp.ServeMux, state *State) (*Server, error) {\n\ts := &Server{mux: mux}\n\tif err := s.set(state); err != nil {\n\t\treturn nil, err\n\t}\n\tmux.HandleFunc(\"/introspect\", s.serve)\n\treturn s, nil\n}\n\n// Set sets the current state for introspection.\nfunc (s *Server) Set(state *State) error {\n\ts.Lock()\n\tdefer s.Unlock()\n\treturn s.set(state)\n}\n\n// Start enables serving HTTP requests.\nfunc (s *Server) Start() {\n\tlog.Info(\"starting introspection server...\")\n\ts.ready = true\n}\n\n// Stop stops serving further HTTP requests.\nfunc (s *Server) Stop() {\n\tlog.Info(\"stopping introspection server...\")\n\ts.ready = false\n}\n\n// set sets the given state and encodes it as a JSON string.\nfunc (s *Server) set(state *State) error {\n\tlog.Debug(\"updating introspection data...\")\n\ts.state = state\n\tdata, err := json.Marshal(s.state)\n\tif err != nil {\n\t\terr = introspectError(\"failed to marshal state for introspection: %v\", err)\n\t\ts.state = &State{Error: fmt.Sprintf(\"%v\", err)}\n\t\tdata, _ = json.Marshal(s.state)\n\t}\n\n\ts.data = string(data)\n\treturn err\n}\n\n// serve serves a single HTTP request.\nfunc (s *Server) serve(w http.ResponseWriter, _ *http.Request) {\n\tif !s.ready {\n\t\treturn\n\t}\n\tlog.Debug(\"serving introspection data...\")\n\ts.RLock()\n\tfmt.Fprintf(w, \"%s\\r\\n\", s.data)\n\ts.RUnlock()\n}\n\n// introspectError creates an introspection-specific error.\nfunc introspectError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"introspection: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/kubernetes/kubernetes.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage kubernetes\n\nconst (\n\t// ResmgrKeyNamespace is a CRI Resource Manager namespace\n\tResmgrKeyNamespace = \"cri-resource-manager.intel.com\"\n\n\t// NamespaceSystem is the kubernetes system namespace.\n\tNamespaceSystem = \"kube-system\"\n\t// PodNameLabel is the key for the kubernetes pod name label.\n\tPodNameLabel = \"io.kubernetes.pod.name\"\n\t// PodNameLabel is the key for the kubernetes pod UID label.\n\tPodUIDLabel = \"io.kubernetes.pod.uid\"\n\t// ContainerNameLabel is the key for the kubernetes container name label.\n\tContainerNameLabel = \"io.kubernetes.container.name\"\n)\n\n// ResmgrKey returns a full namespaced name of a resource manager specific key\nfunc ResmgrKey(name string) string {\n\treturn ResmgrKeyNamespace + \"/\" + name\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/kubernetes/resources.go",
    "content": "// Copyright The NRI Plugins Authors. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage kubernetes\n\nconst (\n\t// Constants for converting back and forth between CPU requirements in\n\t// terms of milli-CPUs and kernel cgroup/scheduling parameters.\n\n\t// MinShares is the minimum cpu.shares accepted by cgroups.\n\tMinShares = 2\n\t// MaxShares is the minimum cpu.shares accepted by cgroups.\n\tMaxShares = 262144\n\t// SharesPerCPU is cpu.shares worth one full CPU.\n\tSharesPerCPU = 1024\n\t// MilliCPUToCPU is milli-CPUs worth a full CPU.\n\tMilliCPUToCPU = 1000\n\t// QuotaPeriod is 100000 microseconds, or 100ms\n\tQuotaPeriod = 100000\n\t// MinQuotaPeriod is 1000 microseconds, or 1ms\n\tMinQuotaPeriod = 1000\n)\n\n// MilliCPUToQuota converts milliCPU to CFS quota and period values.\n// (Almost) identical to the same function in kubelet.\nfunc MilliCPUToQuota(milliCPU int64) (quota, period int64) {\n\tif milliCPU == 0 {\n\t\treturn 0, 0\n\t}\n\n\t// TODO(klihub): this is behind the CPUSFSQuotaPerdiod feature gate in kubelet\n\tperiod = int64(QuotaPeriod)\n\n\tquota = (milliCPU * period) / MilliCPUToCPU\n\n\tif quota < MinQuotaPeriod {\n\t\tquota = MinQuotaPeriod\n\t}\n\n\treturn quota, period\n}\n\n// MilliCPUToShares converts the milliCPU to CFS shares.\n// Identical to the same function in kubelet.\nfunc MilliCPUToShares(milliCPU int64) uint64 {\n\tif milliCPU == 0 {\n\t\treturn MinShares\n\t}\n\tshares := (milliCPU * SharesPerCPU) / MilliCPUToCPU\n\tif shares < MinShares {\n\t\treturn MinShares\n\t}\n\tif shares > MaxShares {\n\t\treturn MaxShares\n\t}\n\treturn uint64(shares)\n}\n\n// SharesToMilliCPU converts CFS CPU shares to milli-CPUs.\nfunc SharesToMilliCPU(shares int64) int64 {\n\tif shares == MinShares {\n\t\treturn 0\n\t}\n\treturn int64(float64(shares*MilliCPUToCPU)/float64(SharesPerCPU) + 0.5)\n}\n\n// QuotaToMilliCPU converts CFS quota and period to milli-CPUs.\nfunc QuotaToMilliCPU(quota, period int64) int64 {\n\tif quota == 0 || period == 0 {\n\t\treturn 0\n\t}\n\treturn int64(float64(quota*MilliCPUToCPU)/float64(period) + 0.5)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/metrics/avx.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\tmodel \"github.com/prometheus/client_model/go\"\n\t\"path/filepath\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n)\n\nfunc (m *Metrics) collectAvxEvents(raw map[string]*model.MetricFamily) *events.Avx {\n\tall, ok := raw[\"all_switch_count_per_cgroup\"]\n\tif !ok {\n\t\treturn nil\n\t}\n\tdump(\"all context switches\", all)\n\n\tavx, ok := raw[\"avx_switch_count_per_cgroup\"]\n\tif !ok {\n\t\treturn nil\n\t}\n\tdump(\"AVX context switches\", avx)\n\n\tratio := map[string]float64{}\n\tfor _, v := range avx.Metric {\n\t\tcgroup, err := filepath.Rel(cgroups.GetV2Dir(), v.Label[0].GetValue())\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tratio[cgroup] = v.Gauge.GetValue()\n\t}\n\tfor _, v := range all.Metric {\n\t\tcgroup, err := filepath.Rel(cgroups.GetV2Dir(), v.Label[0].GetValue())\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tratio[cgroup] /= v.Gauge.GetValue()\n\t}\n\n\tusage := map[string]bool{}\n\tfor cgroup, use := range ratio {\n\t\tactive := use >= m.opts.AvxThreshold\n\t\tlog.Debug(\" %s AVX ratio = %f, active?: %v\", cgroup, use, active)\n\t\tusage[\"/\"+cgroup] = active\n\t}\n\n\treturn &events.Avx{Updates: usage}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/metrics/metrics.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tmodel \"github.com/prometheus/client_model/go\"\n\t\"github.com/prometheus/common/expfmt\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation\"\n\t\"github.com/intel/cri-resource-manager/pkg/metrics\"\n\t// pull in all metrics collectors\n\t_ \"github.com/intel/cri-resource-manager/pkg/metrics/register\"\n)\n\nconst (\n\t// DefaultAvxThreshold is the cutoff below which a cgroup/container is not an AVX user.\n\tDefaultAvxThreshold = float64(0.1)\n)\n\n// Options describes options for metrics collection and processing.\ntype Options struct {\n\t// PollInterval is the interval for polling raw metrics.\n\tPollInterval time.Duration\n\t// Events is the channel for delivering metrics events.\n\tEvents chan interface{}\n\t// AvxThreshold is the threshold (0 - 1) for a cgroup to be considered AVX512-active\n\tAvxThreshold float64\n}\n\n// Metrics implements collecting, caching and processing of raw metrics.\ntype Metrics struct {\n\tsync.RWMutex\n\topts Options               // metrics collecting options\n\tg    prometheus.Gatherer   // prometheus/raw metrics gatherer\n\tstop chan interface{}      // channel to stop polling goroutine\n\traw  []*model.MetricFamily // latest set of raw metrics\n\tpend []*model.MetricFamily // pending metrics for forwarding\n}\n\n// Our logger instance.\nvar log = logger.NewLogger(\"metrics\")\n\n// NewMetrics creates a new instance for metrics collecting and processing.\nfunc NewMetrics(opts Options) (*Metrics, error) {\n\tif opts.Events == nil {\n\t\treturn nil, metricsError(\"invalid options, nil Event channel\")\n\t}\n\tif opts.AvxThreshold == 0.0 {\n\t\topts.AvxThreshold = DefaultAvxThreshold\n\t}\n\n\tg, err := metrics.NewMetricGatherer()\n\tif err != nil {\n\t\treturn nil, metricsError(\"failed to create raw metrics gatherer: %v\", err)\n\t}\n\n\tm := &Metrics{\n\t\topts: opts,\n\t\traw:  make([]*model.MetricFamily, 0),\n\t\tg:    g,\n\t}\n\n\tm.poll()\n\tinstrumentation.RegisterGatherer(m)\n\n\treturn m, nil\n}\n\n// Start starts metrics collection and processing.\nfunc (m *Metrics) Start() error {\n\tif m.stop != nil {\n\t\treturn nil\n\t}\n\n\tstop := make(chan interface{})\n\tgo func() {\n\t\tvar pollTimer *time.Ticker\n\t\tvar pollChan <-chan time.Time\n\n\t\tif m.opts.PollInterval > 0 {\n\t\t\tpollTimer = time.NewTicker(m.opts.PollInterval)\n\t\t\tpollChan = pollTimer.C\n\t\t} else {\n\t\t\tlog.Info(\"periodic collection of metrics is disabled\")\n\t\t}\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase _ = <-stop:\n\t\t\t\tif pollTimer != nil {\n\t\t\t\t\tpollTimer.Stop()\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\tcase _ = <-pollChan:\n\t\t\t\tif err := m.poll(); err != nil {\n\t\t\t\t\tlog.Error(\"failed to poll raw metrics: %v\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tif err := m.process(); err != nil {\n\t\t\t\t\tlog.Error(\"failed to deliver metrics event: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\tm.stop = stop\n\n\treturn nil\n}\n\n// Stop stops metrics collection and processing.\nfunc (m *Metrics) Stop() {\n\tif m.stop != nil {\n\t\tclose(m.stop)\n\t\tm.stop = nil\n\t}\n}\n\n// poll does a single round of raw metrics collection.\nfunc (m *Metrics) poll() error {\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tf, err := m.g.Gather()\n\tif err != nil {\n\t\treturn metricsError(\"failed to poll raw metrics: %v\", err)\n\t}\n\tm.raw = f\n\tm.pend = f\n\treturn nil\n}\n\n// process processes the collected raw metrics.\nfunc (m *Metrics) process() error {\n\traw := map[string]*model.MetricFamily{}\n\tfor _, f := range m.raw {\n\t\tdump(\" <metric \"+*f.Name+\"> \", f)\n\t\traw[*f.Name] = f\n\t}\n\n\tevent := &events.Metrics{\n\t\tAvx: m.collectAvxEvents(raw),\n\t}\n\n\treturn m.sendEvent(event)\n}\n\n// sendEvent sends a metrics-based event for processing.\nfunc (m *Metrics) sendEvent(e *events.Metrics) error {\n\tselect {\n\tcase m.opts.Events <- e:\n\t\treturn nil\n\tdefault:\n\t\treturn metricsError(\"failed to deliver event %v (channel full?)\", *e)\n\t}\n}\n\n// dump debug-dumps the given MetricFamily data\nfunc dump(prefix string, f *model.MetricFamily) {\n\tif !log.DebugEnabled() {\n\t\treturn\n\t}\n\tbuf := &bytes.Buffer{}\n\tif _, err := expfmt.MetricFamilyToText(buf, f); err != nil {\n\t\treturn\n\t}\n\tlog.DebugBlock(\"  <\"+prefix+\"> \", \"%s\", strings.TrimSpace(buf.String()))\n}\n\n// metricsError returns a new formatted error specific to metrics-processing.\nfunc metricsError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"metrics: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/metrics/prometheus.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\tmodel \"github.com/prometheus/client_model/go\"\n)\n\n// Gather is our prometheus.Gatherer interface for proxying metrics.\nfunc (m *Metrics) Gather() ([]*model.MetricFamily, error) {\n\tm.Lock()\n\tpend := m.pend\n\tm.Unlock()\n\n\tif pend == nil {\n\t\tlog.Debug(\"no data to proxy to prometheus...\")\n\t} else {\n\t\tlog.Debug(\"proxying data to prometheus...\")\n\t}\n\n\treturn pend, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/no-test-api.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !test\n// +build !test\n\npackage resmgr\n\n// ResourceManagerTestAPI is dummy if we're compiling without test build flag.\ntype ResourceManagerTestAPI interface {\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/balloons-policy.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tcpucontrol \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/cpu\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tpolicyapi \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy.\n\tPolicyName = \"balloons\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"Flexible pools with per-pool CPU parameters\"\n\t// PolicyPath is the path of this policy in the configuration hierarchy.\n\tPolicyPath = \"policy.\" + PolicyName\n\t// balloonKey is a pod annotation key, the value is a pod balloon name.\n\tballoonKey = \"balloon.\" + PolicyName + \".\" + kubernetes.ResmgrKeyNamespace\n\t// reservedBalloonDefName is the name in the reserved balloon definition.\n\treservedBalloonDefName = \"reserved\"\n\t// defaultBalloonDefName is the name in the default balloon definition.\n\tdefaultBalloonDefName = \"default\"\n\t// NoLimit value denotes no limit being set.\n\tNoLimit = 0\n)\n\n// balloons contains configuration and runtime attributes of the balloons policy\ntype balloons struct {\n\toptions          *policyapi.BackendOptions // configuration common to all policies\n\tbpoptions        BalloonsOptions           // balloons-specific configuration\n\tcch              cache.Cache               // cri-resmgr cache\n\tallowed          cpuset.CPUSet             // bounding set of CPUs we're allowed to use\n\treserved         cpuset.CPUSet             // system-/kube-reserved CPUs\n\tfreeCpus         cpuset.CPUSet             // CPUs to be included in growing or new ballons\n\tcpuTree          *cpuTreeNode              // system CPU topology\n\tcpuTreeAllocator *cpuTreeAllocator         // CPU allocator from system CPU topology\n\n\treservedBalloonDef *BalloonDef // built-in definition of the reserved balloon\n\tdefaultBalloonDef  *BalloonDef // built-in definition of the default balloon\n\tballoons           []*Balloon  // balloon instances: reserved, default and user-defined\n\n\tcpuAllocator cpuallocator.CPUAllocator // CPU allocator used by the policy\n}\n\n// Balloon contains attributes of a balloon instance\ntype Balloon struct {\n\t// Def is the definition from which this balloon instance is created.\n\tDef *BalloonDef\n\t// Instance is the index of this balloon instance, starting from\n\t// zero for every balloon definition.\n\tInstance int\n\t// Cpus is the set of CPUs exclusive to this balloon instance only.\n\tCpus cpuset.CPUSet\n\t// Mems is the set of memory nodes with minimal access delay\n\t// from CPUs.\n\tMems idset.IDSet\n\t// SharedIdleCpus is the set of idle CPUs that workloads in a\n\t// balloon are allowed to use with workloads in other balloons\n\t// that shareIdleCpus.\n\tSharedIdleCpus cpuset.CPUSet\n\t// PodIDs maps pod ID to list of container IDs.\n\t// - len(PodIDs) is the number of pods in the balloon.\n\t// - len(PodIDs[podID]) is the number of containers of podID\n\t//   currently assigned to the balloon.\n\tPodIDs           map[string][]string\n\tcpuTreeAllocator *cpuTreeAllocator\n}\n\nvar log logger.Logger = logger.NewLogger(\"policy\")\n\n// String is a stringer for a balloon.\nfunc (bln Balloon) String() string {\n\treturn fmt.Sprintf(\"%s{Cpus:%s, Mems:%s}\", bln.PrettyName(), bln.Cpus, bln.Mems)\n}\n\n// PrettyName returns a unique name for a balloon.\nfunc (bln Balloon) PrettyName() string {\n\treturn fmt.Sprintf(\"%s[%d]\", bln.Def.Name, bln.Instance)\n}\n\n// ContainerIDs returns IDs of containers assigned in a balloon.\n// (Using cache.Container.GetCacheID()'s)\nfunc (bln Balloon) ContainerIDs() []string {\n\tcIDs := []string{}\n\tfor _, ctrIDs := range bln.PodIDs {\n\t\tcIDs = append(cIDs, ctrIDs...)\n\t}\n\treturn cIDs\n}\n\n// ContainerCount returns the number of containers in a balloon.\nfunc (bln Balloon) ContainerCount() int {\n\tcount := 0\n\tfor _, ctrIDs := range bln.PodIDs {\n\t\tcount += len(ctrIDs)\n\t}\n\treturn count\n}\n\nfunc (bln Balloon) AvailMilliCpus() int {\n\treturn bln.Cpus.Size() * 1000\n}\n\nfunc (bln Balloon) MaxAvailMilliCpus(freeCpus cpuset.CPUSet) int {\n\tif bln.Def.MaxCpus == NoLimit {\n\t\treturn (bln.Cpus.Size() + freeCpus.Size()) * 1000\n\t}\n\treturn bln.Def.MaxCpus * 1000\n}\n\n// CreateBalloonsPolicy creates a new policy instance.\nfunc CreateBalloonsPolicy(policyOptions *policy.BackendOptions) policy.Backend {\n\tvar err error\n\tp := &balloons{\n\t\toptions:      policyOptions,\n\t\tcch:          policyOptions.Cache,\n\t\tcpuAllocator: cpuallocator.NewCPUAllocator(policyOptions.System),\n\t}\n\tlog.Info(\"creating %s policy...\", PolicyName)\n\tif p.cpuTree, err = NewCpuTreeFromSystem(); err != nil {\n\t\tlog.Errorf(\"creating CPU topology tree failed: %s\", err)\n\t}\n\tlog.Debug(\"CPU topology: %s\", p.cpuTree)\n\t// Handle common policy options: AvailableResources and ReservedResources.\n\t// p.allowed: CPUs available for the policy\n\tif allowed, ok := policyOptions.Available[policyapi.DomainCPU]; ok {\n\t\tp.allowed = allowed.(cpuset.CPUSet)\n\t} else {\n\t\t// Available CPUs not specified, default to all on-line CPUs.\n\t\tp.allowed = policyOptions.System.CPUSet().Difference(policyOptions.System.Offlined())\n\t}\n\t// p.reserved: CPUs reserved for kube-system pods, subset of p.allowed.\n\tp.reserved = cpuset.New()\n\tif reserved, ok := p.options.Reserved[policyapi.DomainCPU]; ok {\n\t\tswitch v := reserved.(type) {\n\t\tcase cpuset.CPUSet:\n\t\t\tp.reserved = p.allowed.Intersection(v)\n\t\tcase resapi.Quantity:\n\t\t\treserveCnt := (int(v.MilliValue()) + 999) / 1000\n\t\t\tcpus, err := p.cpuAllocator.AllocateCpus(&p.allowed, reserveCnt, cpuallocator.PriorityNone)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(\"failed to allocate reserved CPUs: %s\", err)\n\t\t\t}\n\t\t\tp.reserved = cpus\n\t\t\tp.allowed = p.allowed.Union(cpus)\n\t\t}\n\t}\n\tif p.reserved.IsEmpty() {\n\t\tlog.Fatal(\"%s cannot run without reserved CPUs that are also AvailableResources\", PolicyName)\n\t}\n\t// Handle policy-specific options\n\tlog.Debug(\"creating %s configuration\", PolicyName)\n\tif err := p.setConfig(balloonsOptions); err != nil {\n\t\tlog.Fatal(\"failed to create %s policy: %v\", PolicyName, err)\n\t}\n\tlog.Debug(\"first effective configuration:\\n%s\\n\", utils.DumpJSON(p.bpoptions))\n\tpkgcfg.GetModule(PolicyPath).AddNotify(p.configNotify)\n\n\treturn p\n}\n\n// Name returns the name of this policy.\nfunc (p *balloons) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (p *balloons) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (p *balloons) Start(add []cache.Container, del []cache.Container) error {\n\tlog.Info(\"%s policy started\", PolicyName)\n\t// reassign all containers\n\treturn p.Sync(p.cch.GetContainers(), del)\n}\n\n// Sync synchronizes the active policy state.\nfunc (p *balloons) Sync(add []cache.Container, del []cache.Container) error {\n\tlog.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\tp.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\tp.AllocateResources(c)\n\t}\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (p *balloons) AllocateResources(c cache.Container) error {\n\tlog.Debug(\"allocating resources for container %s (request %d mCPU, limit %d mCPU)...\",\n\t\tc.PrettyName(),\n\t\tp.containerRequestedMilliCpus(c.GetCacheID()),\n\t\tp.containerLimitedMilliCpus(c.GetCacheID()))\n\tbln, err := p.allocateBalloon(c)\n\tif err != nil {\n\t\treturn balloonsError(\"balloon allocation for container %s failed: %w\", c.PrettyName(), err)\n\t}\n\tif bln == nil {\n\t\treturn balloonsError(\"no suitable balloons found for container %s\", c.PrettyName())\n\t}\n\t// Resize selected balloon to fit the new container, unless it\n\t// uses the ReservedResources CPUs, which is a fixed set.\n\treqMilliCpus := p.containerRequestedMilliCpus(c.GetCacheID()) + p.requestedMilliCpus(bln)\n\t// Even if all containers in a balloon request is 0 mCPU in\n\t// total (all are BestEffort, for example), force the size of\n\t// the balloon to be enough for at least 1 mCPU\n\t// request. Otherwise balloon's cpuset becomes empty, which in\n\t// would mean no CPU pinning and balloon's containers would\n\t// run on any CPUs.\n\tif bln.AvailMilliCpus() < max(1, reqMilliCpus) {\n\t\tp.resizeBalloon(bln, max(1, reqMilliCpus))\n\t}\n\tp.assignContainer(c, bln)\n\tif log.DebugEnabled() {\n\t\tlog.Debug(p.dumpBalloon(bln))\n\t}\n\treturn nil\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (p *balloons) ReleaseResources(c cache.Container) error {\n\tlog.Debug(\"releasing container %s...\", c.PrettyName())\n\tif bln := p.balloonByContainer(c); bln != nil {\n\t\tp.dismissContainer(c, bln)\n\t\tif log.DebugEnabled() {\n\t\t\tlog.Debug(p.dumpBalloon(bln))\n\t\t}\n\t\tif bln.ContainerCount() == 0 {\n\t\t\t// Deflate the balloon completely before\n\t\t\t// freeing it.\n\t\t\tp.resizeBalloon(bln, 0)\n\t\t\tlog.Debug(\"all containers removed, free balloon allocation %s\", bln.PrettyName())\n\t\t\tp.freeBalloon(bln)\n\t\t} else {\n\t\t\t// Make sure that the balloon will have at\n\t\t\t// least 1 CPU to run remaining containers.\n\t\t\tp.resizeBalloon(bln, max(1, p.requestedMilliCpus(bln)))\n\t\t}\n\t} else {\n\t\tlog.Debug(\"ReleaseResources: balloon-less container %s, nothing to release\", c.PrettyName())\n\t}\n\treturn nil\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (p *balloons) UpdateResources(c cache.Container) error {\n\tlog.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (p *balloons) Rebalance() (bool, error) {\n\tlog.Debug(\"(not) rebalancing containers...\")\n\treturn false, nil\n}\n\n// HandleEvent handles policy-specific events.\nfunc (p *balloons) HandleEvent(*events.Policy) (bool, error) {\n\tlog.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (p *balloons) ExportResourceData(c cache.Container) map[string]string {\n\treturn nil\n}\n\n// Introspect provides data for external introspection.\nfunc (p *balloons) Introspect(*introspect.State) {\n\treturn\n}\n\n// balloonByContainer returns a balloon that contains a container.\nfunc (p *balloons) balloonByContainer(c cache.Container) *Balloon {\n\tpodID := c.GetPodID()\n\tcID := c.GetCacheID()\n\tfor _, bln := range p.balloons {\n\t\tfor _, ctrID := range bln.PodIDs[podID] {\n\t\t\tif ctrID == cID {\n\t\t\t\treturn bln\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// balloonsByNamespace returns balloons that contain containers in a\n// namespace.\nfunc (p *balloons) balloonsByNamespace(namespace string) []*Balloon {\n\tblns := []*Balloon{}\n\tfor _, bln := range p.balloons {\n\t\tfor podID, ctrIDs := range bln.PodIDs {\n\t\t\tif len(ctrIDs) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tpod, ok := p.cch.LookupPod(podID)\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif pod.GetNamespace() == namespace {\n\t\t\t\tblns = append(blns, bln)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn blns\n}\n\n// balloonsByPod returns balloons that contain any container of a pod.\nfunc (p *balloons) balloonsByPod(pod cache.Pod) []*Balloon {\n\tpodID := pod.GetID()\n\tblns := []*Balloon{}\n\tfor _, bln := range p.balloons {\n\t\tif _, ok := bln.PodIDs[podID]; ok {\n\t\t\tblns = append(blns, bln)\n\t\t}\n\t}\n\treturn blns\n}\n\n// balloonsByDef returns list of balloons instantiated from a balloon\n// definition.\nfunc (p *balloons) balloonsByDef(blnDef *BalloonDef) []*Balloon {\n\tballoons := []*Balloon{}\n\tfor _, balloon := range p.balloons {\n\t\tif balloon.Def == blnDef {\n\t\t\tballoons = append(balloons, balloon)\n\t\t}\n\t}\n\treturn balloons\n}\n\n// balloonDefByName returns a balloon definition with a name.\nfunc (p *balloons) balloonDefByName(defName string) *BalloonDef {\n\tif defName == \"reserved\" {\n\t\treturn p.reservedBalloonDef\n\t}\n\tif defName == \"default\" {\n\t\treturn p.defaultBalloonDef\n\t}\n\tfor _, blnDef := range p.bpoptions.BalloonDefs {\n\t\tif blnDef.Name == defName {\n\t\t\treturn blnDef\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (p *balloons) chooseBalloonDef(c cache.Container) (*BalloonDef, error) {\n\tvar blnDef *BalloonDef\n\t// BalloonDef is defined by annotation?\n\tif blnDefName, ok := c.GetEffectiveAnnotation(balloonKey); ok {\n\t\tblnDef = p.balloonDefByName(blnDefName)\n\t\tif blnDef == nil {\n\t\t\treturn nil, balloonsError(\"no balloon for annotation %q\", blnDefName)\n\t\t}\n\t\treturn blnDef, nil\n\t}\n\n\t// BalloonDef is defined by a special namespace (kube-system +\n\t// ReservedPoolNamespaces)?\n\tif namespaceMatches(c.GetNamespace(), append(p.bpoptions.ReservedPoolNamespaces, metav1.NamespaceSystem)) {\n\t\treturn p.balloons[0].Def, nil\n\t}\n\n\t// BalloonDef is defined by the namespace.\n\tfor _, blnDef := range append([]*BalloonDef{p.reservedBalloonDef, p.defaultBalloonDef}, p.bpoptions.BalloonDefs...) {\n\t\tif namespaceMatches(c.GetNamespace(), blnDef.Namespaces) {\n\t\t\treturn blnDef, nil\n\t\t}\n\t}\n\n\t// Fallback to the default balloon.\n\treturn p.defaultBalloonDef, nil\n}\n\nfunc (p *balloons) containerRequestedMilliCpus(contID string) int {\n\tcont, ok := p.cch.LookupContainer(contID)\n\tif !ok {\n\t\treturn 0\n\t}\n\treqCpu, ok := cont.GetResourceRequirements().Requests[corev1.ResourceCPU]\n\tif !ok {\n\t\treturn 0\n\t}\n\treturn int(reqCpu.MilliValue())\n}\n\nfunc (p *balloons) containerLimitedMilliCpus(contID string) int {\n\tcont, ok := p.cch.LookupContainer(contID)\n\tif !ok {\n\t\treturn 0\n\t}\n\treqCpu, ok := cont.GetResourceRequirements().Limits[corev1.ResourceCPU]\n\tif !ok {\n\t\treturn 0\n\t}\n\treturn int(reqCpu.MilliValue())\n}\n\n// requestedMilliCpus sums up and returns CPU requests of all\n// containers assigned to a balloon.\nfunc (p *balloons) requestedMilliCpus(bln *Balloon) int {\n\tcpuRequested := 0\n\tfor _, cID := range bln.ContainerIDs() {\n\t\tcpuRequested += p.containerRequestedMilliCpus(cID)\n\t}\n\treturn cpuRequested\n}\n\n// freeMilliCpus returns free CPU resources in a balloon without\n// inflating the balloon.\nfunc (p *balloons) freeMilliCpus(bln *Balloon) int {\n\treturn bln.AvailMilliCpus() - p.requestedMilliCpus(bln)\n}\n\n// maxFreeMilliCpus returns free CPU resources in a balloon when it is\n// inflated as large as possible.\nfunc (p *balloons) maxFreeMilliCpus(bln *Balloon) int {\n\treturn bln.MaxAvailMilliCpus(p.freeCpus) - p.requestedMilliCpus(bln)\n}\n\n// largest helps finding the largest element and value in a slice.\n// Input the length of a slice and a function that returns the\n// magnitude of given element in the slice as int.\nfunc largest(sliceLen int, valueOf func(i int) int) (int, int) {\n\tlargestIndex := -1\n\tlargestValue := 0\n\tfor index := 0; index < sliceLen; index++ {\n\t\tvalue := valueOf(index)\n\t\tif largestIndex == -1 || value > largestValue {\n\t\t\tlargestIndex = index\n\t\t\tlargestValue = value\n\t\t}\n\t}\n\treturn largestIndex, largestValue\n}\n\n// resetCpuClass resets CPU configurations globally. All balloons can\n// be ignored, their CPU configurations will be applied later.\nfunc (p *balloons) resetCpuClass() error {\n\t// Usual inputs:\n\t// - p.allowed (cpuset.CPUset): all CPUs available for this\n\t//   policy.\n\t// - p.IdleCpuClass (string): CPU class for allowed CPUs.\n\t//\n\t// Other inputs, if needed:\n\t// - p.reserved (cpuset.CPUset): CPUs of ReservedResources\n\t//   (typically for kube-system containers).\n\t//\n\t// Note: p.useCpuClass(balloon) will be called before assigning\n\t// containers on the balloon, including the reserved balloon.\n\t//\n\t// TODO: don't depend on cpu controller directly\n\tcpucontrol.Assign(p.cch, p.bpoptions.IdleCpuClass, p.allowed.UnsortedList()...)\n\tlog.Debugf(\"resetCpuClass available: %s; reserved: %s\", p.allowed, p.reserved)\n\treturn nil\n}\n\n// useCpuClass configures CPUs of a balloon.\nfunc (p *balloons) useCpuClass(bln *Balloon) error {\n\t// Usual inputs:\n\t// - CPUs that cpuallocator has reserved for this balloon:\n\t//   bln.Cpus (cpuset.CPUSet).\n\t// - User-defined CPU configuration for CPUs of balloon of this type:\n\t//   bln.Def.CpuClass (string).\n\t// - Current configuration(?): feel free to add data\n\t//   structure for this. For instance policy-global p.cpuConfs,\n\t//   or balloon-local bln.cpuConfs.\n\t//\n\t// Other input examples, if needed:\n\t// - Requested CPU resources by all containers in the balloon:\n\t//   p.requestedMilliCpus(bln).\n\t// - Free CPU resources in the balloon: p.freeMilliCpus(bln).\n\t// - Number of assigned containers: bln.ContainerCount().\n\t// - Container details: access p.cch with bln.ContainerIDs().\n\t// - User-defined CPU AllocatorPriority: bln.Def.AllocatorPriority.\n\t// - All existing balloon instances: p.balloons.\n\t// - CPU configurations by user: bln.Def.CpuClass (for bln in p.balloons)\n\tcpucontrol.Assign(p.cch, bln.Def.CpuClass, bln.Cpus.UnsortedList()...)\n\tlog.Debugf(\"useCpuClass Cpus: %s; CpuClass: %s\", bln.Cpus, bln.Def.CpuClass)\n\treturn nil\n}\n\n// forgetCpuClass is called when CPUs of a balloon are released from duty.\nfunc (p *balloons) forgetCpuClass(bln *Balloon) {\n\t// Use p.IdleCpuClass for bln.Cpus.\n\t// Usual inputs: see useCpuClass\n\tcpucontrol.Assign(p.cch, p.bpoptions.IdleCpuClass, bln.Cpus.UnsortedList()...)\n\tlog.Debugf(\"forgetCpuClass Cpus: %s; CpuClass: %s\", bln.Cpus, bln.Def.CpuClass)\n}\n\nfunc (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool) (*Balloon, error) {\n\tvar cpus cpuset.CPUSet\n\tvar err error\n\tblnsOfDef := p.balloonsByDef(blnDef)\n\t// Allowed to create new balloon instance from blnDef?\n\tif blnDef.MaxBalloons > NoLimit && blnDef.MaxBalloons <= len(blnsOfDef) {\n\t\treturn nil, balloonsError(\"cannot create new %q balloon, MaxBalloons limit (%d) reached\", blnDef.Name, blnDef.MaxBalloons)\n\t}\n\t// Find the first unused balloon instance index.\n\tfreeInstance := 0\n\tfor freeInstance = 0; freeInstance < len(blnsOfDef); freeInstance++ {\n\t\tisFree := true\n\t\tfor _, bln := range blnsOfDef {\n\t\t\tif bln.Instance == freeInstance {\n\t\t\t\tisFree = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif isFree {\n\t\t\tbreak\n\t\t}\n\t}\n\t// Configure new cpuTreeAllocator for this balloon if there\n\t// are type specific allocator options, otherwise use policy\n\t// default allocator.\n\tcpuTreeAllocator := p.cpuTreeAllocator\n\tif blnDef.AllocatorTopologyBalancing != nil || blnDef.PreferSpreadOnPhysicalCores != nil {\n\t\tallocatorOptions := cpuTreeAllocatorOptions{\n\t\t\ttopologyBalancing:           p.bpoptions.AllocatorTopologyBalancing,\n\t\t\tpreferSpreadOnPhysicalCores: p.bpoptions.PreferSpreadOnPhysicalCores,\n\t\t}\n\t\tif blnDef.AllocatorTopologyBalancing != nil {\n\t\t\tallocatorOptions.topologyBalancing = *blnDef.AllocatorTopologyBalancing\n\t\t}\n\t\tif blnDef.PreferSpreadOnPhysicalCores != nil {\n\t\t\tallocatorOptions.preferSpreadOnPhysicalCores = *blnDef.PreferSpreadOnPhysicalCores\n\t\t}\n\t\tcpuTreeAllocator = p.cpuTree.NewAllocator(allocatorOptions)\n\t}\n\n\t// Allocate CPUs\n\tif blnDef == p.reservedBalloonDef ||\n\t\t(blnDef == p.defaultBalloonDef && blnDef.MinCpus == 0 && blnDef.MaxCpus == 0) {\n\t\t// The reserved balloon uses ReservedResources CPUs.\n\t\t// So does the default balloon unless its CPU counts are tweaked.\n\t\tcpus = p.reserved\n\t} else {\n\t\taddFromCpus, _, err := cpuTreeAllocator.ResizeCpus(cpuset.New(), p.freeCpus, blnDef.MinCpus)\n\t\tif err != nil {\n\t\t\treturn nil, balloonsError(\"failed to choose a cpuset for allocating first %d CPUs from %#s\", blnDef.MinCpus, p.freeCpus)\n\t\t}\n\t\tcpus, err = p.cpuAllocator.AllocateCpus(&addFromCpus, blnDef.MinCpus, blnDef.AllocatorPriority)\n\t\tif err != nil {\n\t\t\treturn nil, balloonsError(\"could not allocate %d MinCpus for balloon %s[%d]: %w\", blnDef.MinCpus, blnDef.Name, freeInstance, err)\n\t\t}\n\t\tp.freeCpus = p.freeCpus.Difference(cpus)\n\t}\n\tbln := &Balloon{\n\t\tDef:              blnDef,\n\t\tInstance:         freeInstance,\n\t\tPodIDs:           make(map[string][]string),\n\t\tCpus:             cpus,\n\t\tSharedIdleCpus:   cpuset.New(),\n\t\tMems:             p.closestMems(cpus),\n\t\tcpuTreeAllocator: cpuTreeAllocator,\n\t}\n\tif confCpus {\n\t\tif err = p.useCpuClass(bln); err != nil {\n\t\t\tlog.Errorf(\"failed to apply CPU configuration to new balloon %s[%d] (cpus: %s): %w\", blnDef.Name, freeInstance, cpus, err)\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn bln, nil\n}\n\n// deleteBalloon removes an empty balloon.\nfunc (p *balloons) deleteBalloon(bln *Balloon) {\n\tlog.Debugf(\"deleting balloon %s\", bln)\n\tremainingBalloons := []*Balloon{}\n\tfor _, b := range p.balloons {\n\t\tif b != bln {\n\t\t\tremainingBalloons = append(remainingBalloons, b)\n\t\t}\n\t}\n\tp.balloons = remainingBalloons\n\tp.forgetCpuClass(bln)\n\tp.freeCpus = p.freeCpus.Union(bln.Cpus)\n\tp.cpuAllocator.ReleaseCpus(&bln.Cpus, bln.Cpus.Size(), bln.Def.AllocatorPriority)\n}\n\n// freeBalloon clears a balloon and deletes it if allowed.\nfunc (p *balloons) freeBalloon(bln *Balloon) {\n\tbln.PodIDs = make(map[string][]string)\n\tblnsSameDef := p.balloonsByDef(bln.Def)\n\tif len(blnsSameDef) > bln.Def.MinBalloons {\n\t\tp.deleteBalloon(bln)\n\t}\n}\n\nfunc (p *balloons) chooseBalloonInstance(blnDef *BalloonDef, fm FillMethod, c cache.Container) (*Balloon, error) {\n\t// If assigning to the reserved or the default balloon, fill\n\t// method is ignored: always fill the chosen balloon.\n\tif blnDef == p.balloons[0].Def {\n\t\treturn p.balloons[0], nil\n\t}\n\tif blnDef == p.balloons[1].Def {\n\t\treturn p.balloons[1], nil\n\t}\n\treqMilliCpus := p.containerRequestedMilliCpus(c.GetCacheID())\n\t// Handle fill methods that do not use existing instances of\n\t// balloonDef.\n\tswitch fm {\n\tcase FillReservedBalloon:\n\t\treturn p.balloons[0], nil\n\tcase FillDefaultBalloon:\n\t\treturn p.balloons[1], nil\n\tcase FillNewBalloon, FillNewBalloonMust:\n\t\t// Choosing an existing balloon without containers is\n\t\t// preferred over instantiating a new balloon.\n\t\tfor _, bln := range p.balloonsByDef(blnDef) {\n\t\t\tif len(bln.PodIDs) == 0 {\n\t\t\t\treturn bln, nil\n\t\t\t}\n\t\t}\n\t\tnewBln, err := p.newBalloon(blnDef, false)\n\t\tif err != nil {\n\t\t\tif fm == FillNewBalloonMust {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t}\n\t\t// newBln may already have CPUs allocated for it. If\n\t\t// we notice that the new balloon fill method cannot\n\t\t// be used after all, collect steps to undo() new\n\t\t// balloon creation.\n\t\tundoFuncs := []func(){}\n\t\tundo := func() {\n\t\t\tfor _, undoFunc := range undoFuncs {\n\t\t\t\tundoFunc()\n\t\t\t}\n\t\t}\n\t\tundoFuncs = append(undoFuncs, func() {\n\t\t\tp.freeCpus = p.freeCpus.Union(newBln.Cpus)\n\t\t})\n\t\tif newBln.MaxAvailMilliCpus(p.freeCpus) < reqMilliCpus {\n\t\t\t// New balloon cannot be inflated to fit new\n\t\t\t// container. Release its CPUs if already\n\t\t\t// allocated (MinCPUs > 0), and never add it\n\t\t\t// to the list of balloons.\n\t\t\tundo()\n\t\t\tif fm == FillNewBalloonMust {\n\t\t\t\treturn nil, balloonsError(\"not enough CPUs to run container %s requesting %s mCPU. %s.MaxCPUs: %d mCPU, free CPUs: %s\",\n\t\t\t\t\tc.PrettyName(), reqMilliCpus, blnDef.Name, blnDef.MaxCpus*1000, p.freeCpus.Size()*1000)\n\t\t\t} else {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t}\n\t\t// Make the existence of the new balloon official by\n\t\t// adding it to the balloons slice.\n\t\tp.balloons = append(p.balloons, newBln)\n\t\tundoFuncs = append(undoFuncs, func() {\n\t\t\tp.balloons = p.balloons[:len(p.balloons)-1]\n\t\t})\n\t\t// If the new balloon already has CPUs, there is some\n\t\t// housekeeping to do.\n\t\tif newBln.Cpus.Size() > 0 {\n\t\t\t// Make sure CPUs in the balloon use correct\n\t\t\t// CPU class.\n\t\t\tif err = p.useCpuClass(newBln); err != nil {\n\t\t\t\tlog.Errorf(\"failed to apply CPU configuration to new balloon %s (cpus: %s): %s\",\n\t\t\t\t\tnewBln.PrettyName(), newBln.Cpus, err)\n\t\t\t\tundo()\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\t// Reshare idle CPUs because freeCpus have\n\t\t\t// changed and CPUs of the new balloon are no\n\t\t\t// more idle.\n\t\t\tp.updatePinning(p.shareIdleCpus(p.freeCpus, newBln.Cpus)...)\n\t\t}\n\t\treturn newBln, nil\n\tcase FillSameNamespace:\n\t\tfor _, bln := range p.balloonsByNamespace(c.GetNamespace()) {\n\t\t\tif bln.Def == blnDef && p.maxFreeMilliCpus(bln) >= reqMilliCpus {\n\t\t\t\treturn bln, nil\n\t\t\t}\n\t\t}\n\t\treturn nil, nil\n\tcase FillSamePod:\n\t\tif pod, ok := c.GetPod(); ok {\n\t\t\tfor _, bln := range p.balloonsByPod(pod) {\n\t\t\t\tif p.maxFreeMilliCpus(bln) >= reqMilliCpus {\n\t\t\t\t\treturn bln, nil\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t} else {\n\t\t\treturn nil, balloonsError(\"fill method %s failed: cannot find pod for container %s\", fm, c.PrettyName())\n\t\t}\n\t}\n\t// Handle fill methods that need existing instances of\n\t// balloonDef, and fail if there are no instances.\n\tballoons := p.balloonsByDef(blnDef)\n\tif len(balloons) == 0 {\n\t\treturn nil, nil\n\t}\n\tswitch fm {\n\tcase FillBalanced:\n\t\t// Are there balloons where the container would fit\n\t\t// without inflating the balloon?\n\t\tblnIdx, freeMilliCpus := largest(len(balloons), func(i int) int {\n\t\t\treturn p.freeMilliCpus(balloons[i])\n\t\t})\n\t\tif freeMilliCpus >= reqMilliCpus {\n\t\t\treturn balloons[blnIdx], nil\n\t\t}\n\tcase FillBalancedInflate:\n\t\t// Are there balloons where the container would fit\n\t\t// after inflating the balloon?\n\t\tblnIdx, maxFreeMilliCpus := largest(len(balloons), func(i int) int {\n\t\t\treturn p.maxFreeMilliCpus(balloons[i])\n\t\t})\n\t\tif maxFreeMilliCpus >= reqMilliCpus {\n\t\t\treturn balloons[blnIdx], nil\n\t\t}\n\tdefault:\n\t\treturn nil, balloonsError(\"balloon type fill method not implemented: %s\", fm)\n\t}\n\t// No error, but balloon type remains undecided in this assign method.\n\treturn nil, nil\n}\n\nfunc namespaceMatches(namespace string, patterns []string) bool {\n\tfor _, pattern := range patterns {\n\t\tret, err := filepath.Match(pattern, namespace)\n\t\tif err == nil && ret {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// allocateBalloon returns a balloon allocated for a container.\nfunc (p *balloons) allocateBalloon(c cache.Container) (*Balloon, error) {\n\tblnDef, err := p.chooseBalloonDef(c)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif blnDef == nil {\n\t\treturn nil, balloonsError(\"no applicable balloon type found\")\n\t}\n\n\tbln, err := p.allocateBalloonOfDef(blnDef, c)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif bln == nil {\n\t\treturn nil, balloonsError(\"no suitable balloon instance available\")\n\t}\n\treturn bln, nil\n}\n\n// allocateBalloonOfDef returns a balloon instantiated from a\n// definition for a container.\nfunc (p *balloons) allocateBalloonOfDef(blnDef *BalloonDef, c cache.Container) (*Balloon, error) {\n\tif blnDef == p.reservedBalloonDef {\n\t\treturn p.balloons[0], nil\n\t}\n\tif blnDef == p.defaultBalloonDef {\n\t\treturn p.balloons[1], nil\n\t}\n\n\tfillChain := []FillMethod{}\n\tif !blnDef.PreferSpreadingPods {\n\t\tfillChain = append(fillChain, FillSamePod)\n\t}\n\tif blnDef.PreferPerNamespaceBalloon {\n\t\tfillChain = append(fillChain, FillSameNamespace, FillNewBalloon)\n\t}\n\tif blnDef.PreferNewBalloons {\n\t\tfillChain = append(fillChain, FillNewBalloon, FillBalanced, FillBalancedInflate)\n\t} else {\n\t\tfillChain = append(fillChain, FillBalanced, FillBalancedInflate, FillNewBalloon)\n\t}\n\tfor _, fillMethod := range fillChain {\n\t\tbln, err := p.chooseBalloonInstance(blnDef, fillMethod, c)\n\t\tif err != nil {\n\t\t\tlog.Debugf(\"fill method %q prevents allocation: %w\", fillMethod, err)\n\t\t\treturn nil, err\n\t\t}\n\t\tif bln == nil {\n\t\t\tlog.Debugf(\"fill method %q not applicable\", fillMethod)\n\t\t\tcontinue\n\t\t}\n\t\tlog.Debugf(\"fill method %q suggests balloon instance %v\", fillMethod, bln)\n\t\treturn bln, nil\n\t}\n\treturn nil, nil\n}\n\n// dumpBalloon dumps balloon contents in detail.\nfunc (p *balloons) dumpBalloon(bln *Balloon) string {\n\tconts := []string{}\n\tpods := []string{}\n\tfor podID, contIDs := range bln.PodIDs {\n\t\tpodName := podID\n\t\tif pod, ok := p.cch.LookupPod(podID); ok {\n\t\t\tpodName = pod.GetName()\n\t\t}\n\t\tpods = append(pods, podName)\n\t\tfor _, contID := range contIDs {\n\t\t\tif cont, ok := p.cch.LookupContainer(contID); ok {\n\t\t\t\tconts = append(conts, cont.PrettyName())\n\t\t\t} else {\n\t\t\t\tconts = append(conts, podName+\".\"+contID)\n\t\t\t}\n\t\t}\n\t}\n\ts := fmt.Sprintf(\"Balloon %s{Cpus: %s; Mems: %s; mCPU used: %d; capacity: %d; max. capacity: %d; pods: %s; conts: %s}\",\n\t\tbln.PrettyName(),\n\t\tbln.Cpus,\n\t\tbln.Mems,\n\t\tp.requestedMilliCpus(bln),\n\t\tbln.AvailMilliCpus(),\n\t\tbln.MaxAvailMilliCpus(p.freeCpus),\n\t\tpods,\n\t\tconts)\n\treturn s\n}\n\n// getPodMilliCPU returns mCPUs requested by podID.\nfunc (p *balloons) getPodMilliCPU(podID string) int64 {\n\tcpuRequested := int64(0)\n\tfor _, c := range p.cch.GetContainers() {\n\t\tif c.GetPodID() == podID {\n\t\t\tif reqCpu, ok := c.GetResourceRequirements().Requests[corev1.ResourceCPU]; ok {\n\t\t\t\tcpuRequested += reqCpu.MilliValue()\n\t\t\t}\n\t\t}\n\t}\n\treturn cpuRequested\n}\n\n// changesBalloons returns true if two balloons policy configurations\n// may lead into different balloon instances or workload assignment.\nfunc changesBalloons(opts0, opts1 *BalloonsOptions) bool {\n\tif opts0 == nil && opts1 == nil {\n\t\treturn false\n\t}\n\tif opts0 == nil || opts1 == nil {\n\t\treturn true\n\t}\n\tif len(opts0.BalloonDefs) != len(opts1.BalloonDefs) {\n\t\treturn true\n\t}\n\to0 := opts0.DeepCopy()\n\to1 := opts1.DeepCopy()\n\t// Ignore differences in CPU class names. Every other change\n\t// potentially changes balloons or workloads.\n\to0.IdleCpuClass = \"\"\n\to1.IdleCpuClass = \"\"\n\tfor i := range o0.BalloonDefs {\n\t\to0.BalloonDefs[i].CpuClass = \"\"\n\t\to1.BalloonDefs[i].CpuClass = \"\"\n\t}\n\treturn utils.DumpJSON(o0) != utils.DumpJSON(o1)\n}\n\n// changesCpuClasses returns true if two balloons policy\n// configurations can lead to using different CPU classes on\n// corresponding balloon instances. Calling changesCpuClasses(o0, o1)\n// makes sense only if changesBalloons(o0, o1) has returned false.\nfunc changesCpuClasses(opts0, opts1 *BalloonsOptions) bool {\n\tif opts0 == nil && opts1 == nil {\n\t\treturn false\n\t}\n\tif opts0 == nil || opts1 == nil {\n\t\treturn true\n\t}\n\tif opts0.IdleCpuClass != opts1.IdleCpuClass {\n\t\treturn true\n\t}\n\tif len(opts0.BalloonDefs) != len(opts1.BalloonDefs) {\n\t\treturn true\n\t}\n\tfor i := range opts0.BalloonDefs {\n\t\tif opts0.BalloonDefs[i].CpuClass != opts1.BalloonDefs[i].CpuClass {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// configNotify applies new configuration.\nfunc (p *balloons) configNotify(event pkgcfg.Event, source pkgcfg.Source) error {\n\tlog.Info(\"configuration %s\", event)\n\tdefer log.Debug(\"effective configuration:\\n%s\\n\", utils.DumpJSON(p.bpoptions))\n\tnewBalloonsOptions := balloonsOptions.DeepCopy()\n\tif !changesBalloons(&p.bpoptions, newBalloonsOptions) {\n\t\tif !changesCpuClasses(&p.bpoptions, newBalloonsOptions) {\n\t\t\tlog.Info(\"no configuration changes\")\n\t\t} else {\n\t\t\tlog.Info(\"configuration changes only on CPU classes\")\n\t\t\t// Update new CPU classes to existing balloon\n\t\t\t// definitions. The same BalloonDef instances\n\t\t\t// must be kept in use, because each Balloon\n\t\t\t// instance holds a direct reference to its\n\t\t\t// BalloonDef.\n\t\t\tfor i := range p.bpoptions.BalloonDefs {\n\t\t\t\tp.bpoptions.BalloonDefs[i].CpuClass = newBalloonsOptions.BalloonDefs[i].CpuClass\n\t\t\t}\n\t\t\t// (Re)configures all CPUs in balloons.\n\t\t\tp.resetCpuClass()\n\t\t\tfor _, bln := range p.balloons {\n\t\t\t\tp.useCpuClass(bln)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\tif err := p.setConfig(newBalloonsOptions); err != nil {\n\t\tlog.Error(\"config update failed: %v\", err)\n\t\treturn err\n\t}\n\tlog.Info(\"config updated successfully\")\n\tp.Sync(p.cch.GetContainers(), p.cch.GetContainers())\n\treturn nil\n}\n\n// applyBalloonDef creates user-defined balloons or reconfigures built-in\n// balloons according to the blnDef. Does not initialize balloon CPUs.\nfunc (p *balloons) applyBalloonDef(balloons *[]*Balloon, blnDef *BalloonDef, freeCpus *cpuset.CPUSet) error {\n\tif len(*balloons) < 2 {\n\t\treturn balloonsError(\"internal error: reserved and default balloons missing, cannot apply balloon definitions\")\n\t}\n\treservedBalloon := (*balloons)[0]\n\tdefaultBalloon := (*balloons)[1]\n\t// Every BalloonDef does one of the following:\n\t// 1. reconfigures the \"reserved\" balloon (most restricted)\n\t// 2. reconfigures the \"default\" balloon (somewhat restricted)\n\t// 3. defines new user-defined balloons.\n\tswitch blnDef.Name {\n\tcase \"\":\n\t\t// Case 0: bad name\n\t\treturn balloonsError(\"undefined or empty balloon name\")\n\tcase reservedBalloon.Def.Name:\n\t\t// Case 1: reconfigure the \"reserved\" balloon.\n\t\tif blnDef.MinCpus != 0 {\n\t\t\treturn balloonsError(\"cannot reconfigure the reserved balloon MinCpus, specified in ReservedResources CPUs\")\n\t\t}\n\t\tif blnDef.MaxCpus != 0 {\n\t\t\treturn balloonsError(\"cannot reconfigure the reserved balloon MaxCpus, specified in ReservedResources CPUs\")\n\t\t}\n\t\tif blnDef.MinBalloons != 0 {\n\t\t\treturn balloonsError(\"cannot reconfigure the reserved balloon MinBalloons\")\n\t\t}\n\t\tp.reservedBalloonDef.AllocatorPriority = blnDef.AllocatorPriority\n\t\tp.reservedBalloonDef.CpuClass = blnDef.CpuClass\n\t\tp.reservedBalloonDef.Namespaces = blnDef.Namespaces\n\tcase defaultBalloon.Def.Name:\n\t\t// Case 2: reconfigure the \"default\" balloon.\n\t\tdefaultUsesReservedCpus := true\n\t\tif blnDef.MinCpus != 0 || blnDef.MaxCpus != 0 {\n\t\t\tdefaultUsesReservedCpus = false\n\t\t}\n\t\tif blnDef.MinBalloons != 0 {\n\t\t\treturn balloonsError(\"cannot reconfigure the default balloon MinBalloons\")\n\t\t}\n\t\tp.defaultBalloonDef.MinCpus = blnDef.MinCpus\n\t\tp.defaultBalloonDef.MaxCpus = blnDef.MaxCpus\n\t\tp.defaultBalloonDef.AllocatorPriority = blnDef.AllocatorPriority\n\t\tp.defaultBalloonDef.CpuClass = blnDef.CpuClass\n\t\tp.defaultBalloonDef.Namespaces = blnDef.Namespaces\n\t\tif !defaultUsesReservedCpus {\n\t\t\t// Overwrite existing default balloon instance\n\t\t\t// that uses reserved CPUs with a balloon that\n\t\t\t// uses its own CPUs.\n\t\t\tnewDefaultBln, err := p.newBalloon(p.defaultBalloonDef, false)\n\t\t\tif err != nil {\n\t\t\t\treturn balloonsError(\"cannot create new default balloon: %w\", err)\n\t\t\t}\n\t\t\tnewDefaultBln.Instance = 0\n\t\t\t(*balloons)[1] = newDefaultBln\n\t\t}\n\tdefault:\n\t\t// Case 3: create minimum amount (MinBalloons) of each user-defined balloons.\n\t\tfor allocPrio := cpuallocator.CPUPriority(0); allocPrio < cpuallocator.NumCPUPriorities; allocPrio++ {\n\t\t\tif blnDef.AllocatorPriority != allocPrio {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor blnIdx := 0; blnIdx < blnDef.MinBalloons; blnIdx++ {\n\t\t\t\tnewBln, err := p.newBalloon(blnDef, false)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif newBln == nil {\n\t\t\t\t\treturn balloonsError(\"failed to create balloon '%s[%d]' as required by MinBalloons=%d\", blnDef.Name, blnIdx, blnDef.MinBalloons)\n\t\t\t\t}\n\t\t\t\t*balloons = append(*balloons, newBln)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (p *balloons) validateConfig(bpoptions *BalloonsOptions) error {\n\tfor _, blnDef := range bpoptions.BalloonDefs {\n\t\tif blnDef.MaxCpus != NoLimit && blnDef.MinCpus > blnDef.MaxCpus {\n\t\t\treturn balloonsError(\"MinCpus (%d) > MaxCpus (%d) in balloon type %q\",\n\t\t\t\tblnDef.MinCpus, blnDef.MaxCpus, blnDef.Name)\n\t\t}\n\t\tif blnDef.MaxBalloons != NoLimit && blnDef.MinBalloons > blnDef.MaxBalloons {\n\t\t\treturn balloonsError(\"MinBalloons (%d) > MaxBalloons (%d) in balloon type %q\",\n\t\t\t\tblnDef.MinCpus, blnDef.MaxCpus, blnDef.Name)\n\t\t}\n\t}\n\treturn nil\n}\n\n// setConfig takes new balloon configuration into use.\nfunc (p *balloons) setConfig(bpoptions *BalloonsOptions) error {\n\t// TODO: revert allocations (p.freeCpus) to old ones if the\n\t// configuration is invalid. Currently bad configuration\n\t// leaves a mess in bookkeeping.\n\tif err := p.validateConfig(bpoptions); err != nil {\n\t\treturn balloonsError(\"invalid configuration: %w\", err)\n\t}\n\n\t// Create the default reserved and default balloon\n\t// definitions. Some properties of these definitions may be\n\t// altered by user configuration.\n\tp.reservedBalloonDef = &BalloonDef{\n\t\tName:              reservedBalloonDefName,\n\t\tMinBalloons:       1,\n\t\tAllocatorPriority: 3,\n\t}\n\tp.defaultBalloonDef = &BalloonDef{\n\t\tName:              defaultBalloonDefName,\n\t\tMinBalloons:       1,\n\t\tAllocatorPriority: 3,\n\t}\n\tp.balloons = []*Balloon{}\n\tp.freeCpus = p.allowed.Clone()\n\tp.freeCpus = p.freeCpus.Difference(p.reserved)\n\tp.cpuTreeAllocator = p.cpuTree.NewAllocator(cpuTreeAllocatorOptions{\n\t\ttopologyBalancing:           bpoptions.AllocatorTopologyBalancing,\n\t\tpreferSpreadOnPhysicalCores: bpoptions.PreferSpreadOnPhysicalCores,\n\t})\n\t// We can't delay taking new configuration into use beyond this point,\n\t// because p.newBalloon() dereferences our options via p.bpoptions, so\n\t// it would end up using the old configuration.\n\tp.bpoptions = *bpoptions\n\t// Instantiate built-in reserved and default balloons.\n\treservedBalloon, err := p.newBalloon(p.reservedBalloonDef, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tp.balloons = append(p.balloons, reservedBalloon)\n\tdefaultBalloon, err := p.newBalloon(p.defaultBalloonDef, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tp.balloons = append(p.balloons, defaultBalloon)\n\t// First apply customizations to built-in balloons: \"reserved\"\n\t// and \"default\".\n\tfor _, blnDef := range bpoptions.BalloonDefs {\n\t\tif blnDef.Name != reservedBalloonDefName && blnDef.Name != defaultBalloonDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif err := p.applyBalloonDef(&p.balloons, blnDef, &p.freeCpus); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Apply all user balloon definitions, skip already customized\n\t// \"reserved\" and \"default\" balloons.\n\tfor _, blnDef := range bpoptions.BalloonDefs {\n\t\tif blnDef.Name == reservedBalloonDefName || blnDef.Name == defaultBalloonDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif err := p.applyBalloonDef(&p.balloons, blnDef, &p.freeCpus); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Finish balloon instance initialization.\n\tlog.Info(\"%s policy balloons:\", PolicyName)\n\tfor blnIdx, bln := range p.balloons {\n\t\tlog.Info(\"- balloon %d: %s\", blnIdx, bln)\n\t}\n\tp.updatePinning(p.shareIdleCpus(p.freeCpus, cpuset.New())...)\n\t// (Re)configures all CPUs in balloons.\n\tp.resetCpuClass()\n\tfor _, bln := range p.balloons {\n\t\tp.useCpuClass(bln)\n\t}\n\treturn nil\n}\n\n// closestMems returns memory node IDs good for pinning containers\n// that run on given CPUs\nfunc (p *balloons) closestMems(cpus cpuset.CPUSet) idset.IDSet {\n\tmems := idset.NewIDSet()\n\tsys := p.options.System\n\tfor _, nodeID := range sys.NodeIDs() {\n\t\tif !cpus.Intersection(sys.Node(nodeID).CPUSet()).IsEmpty() {\n\t\t\tmems.Add(nodeID)\n\t\t}\n\t}\n\treturn mems\n}\n\n// filterBalloons returns balloons for which the test function returns true\nfunc filterBalloons(balloons []*Balloon, test func(*Balloon) bool) (ret []*Balloon) {\n\tfor _, bln := range balloons {\n\t\tif test(bln) {\n\t\t\tret = append(ret, bln)\n\t\t}\n\t}\n\treturn\n}\n\n// availableMilliCPU returns mCPUs available in a balloon.\nfunc (p *balloons) availableMilliCpus(balloon *Balloon) int64 {\n\tcpuAvail := int64(balloon.Cpus.Size() * 1000)\n\tcpuRequested := int64(0)\n\tfor podID := range balloon.PodIDs {\n\t\tcpuRequested += p.getPodMilliCPU(podID)\n\t}\n\treturn cpuAvail - cpuRequested\n}\n\n// resizeBalloon changes the CPUs allocated for a balloon, if allowed.\nfunc (p *balloons) resizeBalloon(bln *Balloon, newMilliCpus int) error {\n\tif bln.Cpus.Equals(p.reserved) {\n\t\tlog.Debugf(\"not resizing %s to %d mCPU, using fixed CPUs\", bln, newMilliCpus)\n\t\treturn nil\n\t}\n\toldCpuCount := bln.Cpus.Size()\n\tnewCpuCount := (newMilliCpus + 999) / 1000\n\tif bln.Def.MaxCpus > NoLimit && newCpuCount > bln.Def.MaxCpus {\n\t\tnewCpuCount = bln.Def.MaxCpus\n\t}\n\tif bln.Def.MinCpus > 0 && newCpuCount < bln.Def.MinCpus {\n\t\tnewCpuCount = bln.Def.MinCpus\n\t}\n\tlog.Debugf(\"resize %s to fit %d mCPU\", bln, newMilliCpus)\n\tlog.Debugf(\"- change full CPUs from %d to %d\", oldCpuCount, newCpuCount)\n\tlog.Debugf(\"- freecpus: %#s\", p.freeCpus)\n\tif oldCpuCount == newCpuCount {\n\t\treturn nil\n\t}\n\tcpuCountDelta := newCpuCount - oldCpuCount\n\tp.forgetCpuClass(bln)\n\tdefer p.useCpuClass(bln)\n\tif cpuCountDelta > 0 {\n\t\t// Inflate the balloon.\n\t\taddFromCpus, _, err := bln.cpuTreeAllocator.ResizeCpus(bln.Cpus, p.freeCpus, cpuCountDelta)\n\t\tif err != nil {\n\t\t\treturn balloonsError(\"resize/inflate: failed to choose a cpuset for allocating additional %d CPUs: %w\", cpuCountDelta, err)\n\t\t}\n\t\tlog.Debugf(\"- allocate CPUs %d from %#s\", cpuCountDelta, addFromCpus)\n\t\tnewCpus, err := p.cpuAllocator.AllocateCpus(&addFromCpus, newCpuCount-oldCpuCount, bln.Def.AllocatorPriority)\n\t\tif err != nil {\n\t\t\treturn balloonsError(\"resize/inflate: allocating %d CPUs for %s failed: %w\", cpuCountDelta, bln, err)\n\t\t}\n\t\tp.freeCpus = p.freeCpus.Difference(newCpus)\n\t\tbln.Cpus = bln.Cpus.Union(newCpus)\n\t\tp.updatePinning(p.shareIdleCpus(p.freeCpus, newCpus)...)\n\t} else {\n\t\t// Deflate the balloon.\n\t\t_, removeFromCpus, err := bln.cpuTreeAllocator.ResizeCpus(bln.Cpus, p.freeCpus, cpuCountDelta)\n\t\tif err != nil {\n\t\t\treturn balloonsError(\"resize/deflate: failed to choose a cpuset for releasing %d CPUs: %w\", -cpuCountDelta, err)\n\t\t}\n\t\tlog.Debugf(\"- releasing %d CPUs from cpuset %#s\", -cpuCountDelta, removeFromCpus)\n\t\t_, err = p.cpuAllocator.ReleaseCpus(&removeFromCpus, -cpuCountDelta, bln.Def.AllocatorPriority)\n\t\tif err != nil {\n\t\t\treturn balloonsError(\"resize/deflate: releasing %d CPUs from %s failed: %w\", -cpuCountDelta, bln, err)\n\t\t}\n\t\tlog.Debugf(\"- old freeCpus: %#s, old bln.Cpus: %#s, releasing: %#s\", p.freeCpus, bln.Cpus, removeFromCpus)\n\t\tp.freeCpus = p.freeCpus.Union(removeFromCpus)\n\t\tbln.Cpus = bln.Cpus.Difference(removeFromCpus)\n\t\tp.updatePinning(p.shareIdleCpus(removeFromCpus, cpuset.New())...)\n\t}\n\tlog.Debugf(\"- resize successful: %s, freecpus: %#s\", bln, p.freeCpus)\n\tp.updatePinning(bln)\n\treturn nil\n}\n\nfunc (p *balloons) updatePinning(blns ...*Balloon) {\n\tfor _, bln := range blns {\n\t\tcpus := bln.Cpus.Union(bln.SharedIdleCpus)\n\t\tbln.Mems = p.closestMems(cpus)\n\t\tfor _, cID := range bln.ContainerIDs() {\n\t\t\tif c, ok := p.cch.LookupContainer(cID); ok {\n\t\t\t\tp.pinCpuMem(c, cpus, bln.Mems)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// shareIdleCpus adds addCpus and removes removeCpus to those balloons\n// that whose containers are allowed to use shared idle CPUs. Returns\n// balloons that will need re-pinning.\nfunc (p *balloons) shareIdleCpus(addCpus, removeCpus cpuset.CPUSet) []*Balloon {\n\tupdateBalloons := map[int]struct{}{}\n\tif removeCpus.Size() > 0 {\n\t\tfor blnIdx, bln := range p.balloons {\n\t\t\tif bln.SharedIdleCpus.Intersection(removeCpus).Size() > 0 {\n\t\t\t\tbln.SharedIdleCpus = bln.SharedIdleCpus.Difference(removeCpus)\n\t\t\t\tupdateBalloons[blnIdx] = struct{}{}\n\t\t\t}\n\t\t}\n\t}\n\tif addCpus.Size() > 0 {\n\t\tfor blnIdx, bln := range p.balloons {\n\t\t\ttopoLevel := bln.Def.ShareIdleCpusInSame\n\t\t\tif topoLevel == CPUTopologyLevelUndefined {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tidleCpusInTopoLevel := cpuset.New()\n\t\t\tp.cpuTree.DepthFirstWalk(func(t *cpuTreeNode) error {\n\t\t\t\t// Dive in correct topology level.\n\t\t\t\tif t.level != topoLevel {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\t// Does the balloon include CPUs in the correct topology level?\n\t\t\t\tif t.cpus.Intersection(bln.Cpus).Size() > 0 {\n\t\t\t\t\t// Share idle CPUs on this level to this balloon.\n\t\t\t\t\tidleCpusInTopoLevel = idleCpusInTopoLevel.Union(t.cpus.Intersection(addCpus))\n\t\t\t\t}\n\t\t\t\t// Do not walk deeper than the correct level.\n\t\t\t\treturn WalkSkipChildren\n\t\t\t})\n\t\t\tif idleCpusInTopoLevel.Size() == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsharedBefore := bln.SharedIdleCpus.Size()\n\t\t\tbln.SharedIdleCpus = bln.SharedIdleCpus.Union(idleCpusInTopoLevel)\n\t\t\tsharedNow := bln.SharedIdleCpus.Size()\n\t\t\tif sharedBefore != sharedNow {\n\t\t\t\tlog.Debugf(\"balloon %d shares %d new idle CPU(s) in %s(s), %d in total (%s)\",\n\t\t\t\t\tbln.PrettyName(), sharedNow-sharedBefore,\n\t\t\t\t\ttopoLevel, bln.SharedIdleCpus.Size(), bln.SharedIdleCpus)\n\t\t\t\tupdateBalloons[blnIdx] = struct{}{}\n\t\t\t}\n\t\t}\n\t}\n\tupdatedBalloons := make([]*Balloon, 0, len(updateBalloons))\n\tfor blnIdx := range updateBalloons {\n\t\tupdatedBalloons = append(updatedBalloons, p.balloons[blnIdx])\n\t}\n\treturn updatedBalloons\n}\n\n// assignContainer adds a container to a balloon\nfunc (p *balloons) assignContainer(c cache.Container, bln *Balloon) {\n\tlog.Info(\"assigning container %s to balloon %s\", c.PrettyName(), bln)\n\t// TODO: inflate the balloon (add CPUs / reconfigure balloons)\n\t// if necessary\n\tpodID := c.GetPodID()\n\tbln.PodIDs[podID] = append(bln.PodIDs[podID], c.GetCacheID())\n\tp.updatePinning(bln)\n}\n\n// dismissContainer removes a container from a balloon\nfunc (p *balloons) dismissContainer(c cache.Container, bln *Balloon) {\n\tpodID := c.GetPodID()\n\tbln.PodIDs[podID] = removeString(bln.PodIDs[podID], c.GetCacheID())\n\tif len(bln.PodIDs[podID]) == 0 {\n\t\tdelete(bln.PodIDs, podID)\n\t}\n}\n\n// pinCpuMem pins container to CPUs and memory nodes if flagged\nfunc (p *balloons) pinCpuMem(c cache.Container, cpus cpuset.CPUSet, mems idset.IDSet) {\n\tif p.bpoptions.PinCPU == nil || *p.bpoptions.PinCPU {\n\t\tlog.Debug(\"  - pinning %s to cpuset: %s\", c.PrettyName(), cpus)\n\t\tc.SetCpusetCpus(cpus.String())\n\t\tif reqCpu, ok := c.GetResourceRequirements().Requests[corev1.ResourceCPU]; ok {\n\t\t\tmCpu := int(reqCpu.MilliValue())\n\t\t\tc.SetCPUShares(int64(cache.MilliCPUToShares(int64(mCpu))))\n\t\t}\n\t}\n\tif p.bpoptions.PinMemory == nil || *p.bpoptions.PinMemory {\n\t\tlog.Debug(\"  - pinning %s to memory %s\", c.PrettyName(), mems)\n\t\tc.SetCpusetMems(mems.String())\n\t}\n}\n\n// balloonsError formats an error from this policy.\nfunc balloonsError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n\n// removeString returns the first occurrence of a string from string slice.\nfunc removeString(strings []string, element string) []string {\n\tfor index, s := range strings {\n\t\tif s == element {\n\t\t\tstrings[index] = strings[len(strings)-1]\n\t\t\treturn strings[:len(strings)-1]\n\t\t}\n\t}\n\treturn strings\n}\n\nfunc max(a, b int) int {\n\tif a > b {\n\t\treturn a\n\t}\n\treturn b\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, CreateBalloonsPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/balloons-policy_test.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"testing\"\n)\n\nfunc TestChangesBalloons(t *testing.T) {\n\ttcases := []struct {\n\t\tname          string\n\t\topts1         *BalloonsOptions\n\t\topts2         *BalloonsOptions\n\t\texpectedValue bool\n\t}{\n\t\t{\n\t\t\tname:          \"both options are nil\",\n\t\t\texpectedValue: false,\n\t\t},\n\t\t{\n\t\t\tname:          \"one option is nil\",\n\t\t\topts2:         &BalloonsOptions{},\n\t\t\texpectedValue: true,\n\t\t},\n\t\t{\n\t\t\tname: \"reserved pool namespaces differ by len\",\n\t\t\topts1: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc0\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t},\n\t\t\topts2: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc0\",\n\t\t\t\tReservedPoolNamespaces: []string{},\n\t\t\t},\n\t\t\texpectedValue: true,\n\t\t},\n\t\t{\n\t\t\tname: \"reserved pool namespaces differ by content\",\n\t\t\topts1: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc0\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t},\n\t\t\topts2: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc0\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns1\"},\n\t\t\t},\n\t\t\texpectedValue: true,\n\t\t},\n\t\t{\n\t\t\tname: \"idle cpu classes differ\",\n\t\t\topts1: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc0\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t},\n\t\t\topts2: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc1\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t},\n\t\t\texpectedValue: false,\n\t\t},\n\t\t{\n\t\t\tname: \"balloon defs differ\",\n\t\t\topts1: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc0\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t\tBalloonDefs:            []*BalloonDef{},\n\t\t\t},\n\t\t\topts2: &BalloonsOptions{\n\t\t\t\tIdleCpuClass:           \"icc1\",\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t\tBalloonDefs:            []*BalloonDef{},\n\t\t\t},\n\t\t\texpectedValue: false,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tvalue := changesBalloons(tc.opts1, tc.opts2)\n\t\t\tif value != tc.expectedValue {\n\t\t\t\tt.Errorf(\"Expected return value %v but got %v\", tc.expectedValue, value)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/cputree.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\ntype CPUTopologyLevel int\n\nconst (\n\tCPUTopologyLevelUndefined CPUTopologyLevel = iota\n\tCPUTopologyLevelSystem\n\tCPUTopologyLevelPackage\n\tCPUTopologyLevelDie\n\tCPUTopologyLevelNuma\n\tCPUTopologyLevelCore\n\tCPUTopologyLevelThread\n\tCPUTopologyLevelCount\n)\n\n// cpuTreeNode is a node in the CPU tree.\ntype cpuTreeNode struct {\n\tname     string\n\tlevel    CPUTopologyLevel\n\tparent   *cpuTreeNode\n\tchildren []*cpuTreeNode\n\tcpus     cpuset.CPUSet // union of CPUs of child nodes\n\n}\n\n// cpuTreeNodeAttributes contains various attributes of a CPU tree\n// node. When allocating or releasing CPUs, all CPU tree nodes in\n// which allocating/releasing could be possible are stored to the same\n// slice with these attributes. The attributes contain all necessary\n// information for comparing which nodes are the best choices for\n// allocating/releasing, thus traversing the tree is not needed in the\n// comparison phase.\ntype cpuTreeNodeAttributes struct {\n\tt                *cpuTreeNode\n\tdepth            int\n\tcurrentCpus      cpuset.CPUSet\n\tfreeCpus         cpuset.CPUSet\n\tcurrentCpuCount  int\n\tcurrentCpuCounts []int\n\tfreeCpuCount     int\n\tfreeCpuCounts    []int\n}\n\n// cpuTreeAllocator allocates CPUs from the branch of a CPU tree\n// where the \"root\" node is the topmost CPU of the branch.\ntype cpuTreeAllocator struct {\n\toptions cpuTreeAllocatorOptions\n\troot    *cpuTreeNode\n}\n\n// cpuTreeAllocatorOptions contains parameters for the CPU allocator\n// that that selects CPUs from a CPU tree.\ntype cpuTreeAllocatorOptions struct {\n\t// topologyBalancing true prefers allocating from branches\n\t// with most free CPUs (spread allocations), while false is\n\t// the opposite (packed allocations).\n\ttopologyBalancing           bool\n\tpreferSpreadOnPhysicalCores bool\n}\n\n// Strings returns topology level as a string\nfunc (ctl CPUTopologyLevel) String() string {\n\ts, ok := cpuTopologyLevelToString[ctl]\n\tif ok {\n\t\treturn s\n\t}\n\treturn fmt.Sprintf(\"CPUTopologyLevelUnknown(%d)\", ctl)\n}\n\n// cpuTopologyLevelToString defines names for all CPU topology levels.\nvar cpuTopologyLevelToString = map[CPUTopologyLevel]string{\n\tCPUTopologyLevelUndefined: \"\",\n\tCPUTopologyLevelSystem:    \"system\",\n\tCPUTopologyLevelPackage:   \"package\",\n\tCPUTopologyLevelDie:       \"die\",\n\tCPUTopologyLevelNuma:      \"numa\",\n\tCPUTopologyLevelCore:      \"core\",\n\tCPUTopologyLevelThread:    \"thread\",\n}\n\n// MarshalJSON()\nfunc (ctl CPUTopologyLevel) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(ctl.String())\n}\n\n// UnmarshalJSON unmarshals a JSON string to CPUTopologyLevel\nfunc (ctl *CPUTopologyLevel) UnmarshalJSON(data []byte) error {\n\tvar dataString string\n\tif err := json.Unmarshal(data, &dataString); err != nil {\n\t\treturn err\n\t}\n\tname := strings.ToLower(dataString)\n\tfor ctlConst, ctlString := range cpuTopologyLevelToString {\n\t\tif ctlString == name {\n\t\t\t*ctl = ctlConst\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn fmt.Errorf(\"invalid CPU topology level %q\", name)\n}\n\n// String returns string representation of a CPU tree node.\nfunc (t *cpuTreeNode) String() string {\n\tif len(t.children) == 0 {\n\t\treturn t.name\n\t}\n\treturn fmt.Sprintf(\"%s%v\", t.name, t.children)\n}\n\nfunc (t *cpuTreeNode) PrettyPrint() string {\n\torigDepth := t.Depth()\n\tlines := []string{}\n\tt.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\tlines = append(lines,\n\t\t\tfmt.Sprintf(\"%s%s: %q cpus: %s\",\n\t\t\t\tstrings.Repeat(\" \", (tn.Depth()-origDepth)*4),\n\t\t\t\ttn.level, tn.name, tn.cpus))\n\t\treturn nil\n\t})\n\treturn strings.Join(lines, \"\\n\")\n}\n\n// String returns cpuTreeNodeAttributes as a string.\nfunc (tna cpuTreeNodeAttributes) String() string {\n\treturn fmt.Sprintf(\"%s{%d,%v,%d,%d}\", tna.t.name, tna.depth,\n\t\ttna.currentCpuCounts,\n\t\ttna.freeCpuCount, tna.freeCpuCounts)\n}\n\n// NewCpuTree returns a named CPU tree node.\nfunc NewCpuTree(name string) *cpuTreeNode {\n\treturn &cpuTreeNode{\n\t\tname: name,\n\t\tcpus: cpuset.New(),\n\t}\n}\n\nfunc (t *cpuTreeNode) CopyTree() *cpuTreeNode {\n\tnewNode := t.CopyNode()\n\tnewNode.children = make([]*cpuTreeNode, 0, len(t.children))\n\tfor _, child := range t.children {\n\t\tnewNode.AddChild(child.CopyTree())\n\t}\n\treturn newNode\n}\n\nfunc (t *cpuTreeNode) CopyNode() *cpuTreeNode {\n\tnewNode := cpuTreeNode{\n\t\tname:     t.name,\n\t\tlevel:    t.level,\n\t\tparent:   t.parent,\n\t\tchildren: t.children,\n\t\tcpus:     t.cpus,\n\t}\n\treturn &newNode\n}\n\n// Depth returns the distance from the root node.\nfunc (t *cpuTreeNode) Depth() int {\n\tif t.parent == nil {\n\t\treturn 0\n\t}\n\treturn t.parent.Depth() + 1\n}\n\n// AddChild adds new child node to a CPU tree node.\nfunc (t *cpuTreeNode) AddChild(child *cpuTreeNode) {\n\tchild.parent = t\n\tt.children = append(t.children, child)\n}\n\n// AddCpus adds CPUs to a CPU tree node and all its parents.\nfunc (t *cpuTreeNode) AddCpus(cpus cpuset.CPUSet) {\n\tt.cpus = t.cpus.Union(cpus)\n\tif t.parent != nil {\n\t\tt.parent.AddCpus(cpus)\n\t}\n}\n\n// Cpus returns CPUs of a CPU tree node.\nfunc (t *cpuTreeNode) Cpus() cpuset.CPUSet {\n\treturn t.cpus\n}\n\n// SiblingIndex returns the index of this node among its parents\n// children. Returns -1 for the root node, -2 if this node is not\n// listed among the children of its parent.\nfunc (t *cpuTreeNode) SiblingIndex() int {\n\tif t.parent == nil {\n\t\treturn -1\n\t}\n\tfor idx, child := range t.parent.children {\n\t\tif child == t {\n\t\t\treturn idx\n\t\t}\n\t}\n\treturn -2\n}\n\nfunc (t *cpuTreeNode) FindLeafWithCpu(cpu int) *cpuTreeNode {\n\tvar found *cpuTreeNode\n\tt.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\tif len(tn.children) > 0 {\n\t\t\treturn nil\n\t\t}\n\t\tfor _, cpuHere := range tn.cpus.List() {\n\t\t\tif cpu == cpuHere {\n\t\t\t\tfound = tn\n\t\t\t\treturn WalkStop\n\t\t\t}\n\t\t}\n\t\treturn nil // not found here, no more children to search\n\t})\n\treturn found\n}\n\n// WalkSkipChildren error returned from a DepthFirstWalk handler\n// prevents walking deeper in the tree. The caller of the\n// DepthFirstWalk will get no error.\nvar WalkSkipChildren error = errors.New(\"skip children\")\n\n// WalkStop error returned from a DepthFirstWalk handler stops the\n// walk altogether. The caller of the DepthFirstWalk will get the\n// WalkStop error.\nvar WalkStop error = errors.New(\"stop\")\n\n// DepthFirstWalk walks through nodes in a CPU tree. Every node is\n// passed to the handler callback that controls next step by\n// returning:\n// - nil: continue walking to the next node\n// - WalkSkipChildren: continue to the next node but skip children of this node\n// - WalkStop: stop walking.\nfunc (t *cpuTreeNode) DepthFirstWalk(handler func(*cpuTreeNode) error) error {\n\tif err := handler(t); err != nil {\n\t\tif err == WalkSkipChildren {\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\tfor _, child := range t.children {\n\t\tif err := child.DepthFirstWalk(handler); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// CpuLocations returns a slice where each element contains names of\n// topology elements over which a set of CPUs spans. Example:\n// systemNode.CpuLocations(cpuset:0,99) = [[\"system\"],[\"p0\", \"p1\"], [\"p0d0\", \"p1d0\"], ...]\nfunc (t *cpuTreeNode) CpuLocations(cpus cpuset.CPUSet) [][]string {\n\tnames := make([][]string, int(CPUTopologyLevelCount)-int(t.level))\n\tt.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\tif tn.cpus.Intersection(cpus).Size() == 0 {\n\t\t\treturn WalkSkipChildren\n\t\t}\n\t\tlevelIndex := int(tn.level) - int(t.level)\n\t\tnames[levelIndex] = append(names[levelIndex], tn.name)\n\t\treturn nil\n\t})\n\treturn names\n}\n\n// NewCpuTreeFromSystem returns the root node of the topology tree\n// constructed from the underlying system.\nfunc NewCpuTreeFromSystem() (*cpuTreeNode, error) {\n\tsys, err := system.DiscoverSystem()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// TODO: split deep nested loops into functions\n\tsysTree := NewCpuTree(\"system\")\n\tsysTree.level = CPUTopologyLevelSystem\n\tfor _, packageID := range sys.PackageIDs() {\n\t\tpackageTree := NewCpuTree(fmt.Sprintf(\"p%d\", packageID))\n\t\tpackageTree.level = CPUTopologyLevelPackage\n\t\tcpuPackage := sys.Package(packageID)\n\t\tsysTree.AddChild(packageTree)\n\t\tfor _, dieID := range cpuPackage.DieIDs() {\n\t\t\tdieTree := NewCpuTree(fmt.Sprintf(\"p%dd%d\", packageID, dieID))\n\t\t\tdieTree.level = CPUTopologyLevelDie\n\t\t\tpackageTree.AddChild(dieTree)\n\t\t\tfor _, nodeID := range cpuPackage.DieNodeIDs(dieID) {\n\t\t\t\tnodeTree := NewCpuTree(fmt.Sprintf(\"p%dd%dn%d\", packageID, dieID, nodeID))\n\t\t\t\tnodeTree.level = CPUTopologyLevelNuma\n\t\t\t\tdieTree.AddChild(nodeTree)\n\t\t\t\tnode := sys.Node(nodeID)\n\t\t\t\tthreadsSeen := map[int]struct{}{}\n\t\t\t\tfor _, cpuID := range node.CPUSet().List() {\n\t\t\t\t\tif _, alreadySeen := threadsSeen[cpuID]; alreadySeen {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tcpuTree := NewCpuTree(fmt.Sprintf(\"p%dd%dn%dcpu%d\", packageID, dieID, nodeID, cpuID))\n\n\t\t\t\t\tcpuTree.level = CPUTopologyLevelCore\n\t\t\t\t\tnodeTree.AddChild(cpuTree)\n\t\t\t\t\tcpu := sys.CPU(cpuID)\n\t\t\t\t\tfor _, threadID := range cpu.ThreadCPUSet().List() {\n\t\t\t\t\t\tthreadsSeen[threadID] = struct{}{}\n\t\t\t\t\t\tthreadTree := NewCpuTree(fmt.Sprintf(\"p%dd%dn%dcpu%dt%d\", packageID, dieID, nodeID, cpuID, threadID))\n\t\t\t\t\t\tthreadTree.level = CPUTopologyLevelThread\n\t\t\t\t\t\tcpuTree.AddChild(threadTree)\n\t\t\t\t\t\tthreadTree.AddCpus(cpuset.New(threadID))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn sysTree, nil\n}\n\n// ToAttributedSlice returns a CPU tree node and recursively all its\n// child nodes in a slice that contains nodes with their attributes\n// for allocation/releasing comparison.\n// - currentCpus is the set of CPUs that can be freed in coming operation\n// - freeCpus is the set of CPUs that can be allocated in coming operation\n// - filter(tna) returns false if the node can be ignored\nfunc (t *cpuTreeNode) ToAttributedSlice(\n\tcurrentCpus, freeCpus cpuset.CPUSet,\n\tfilter func(*cpuTreeNodeAttributes) bool) []cpuTreeNodeAttributes {\n\ttnas := []cpuTreeNodeAttributes{}\n\tcurrentCpuCounts := []int{}\n\tfreeCpuCounts := []int{}\n\tt.toAttributedSlice(currentCpus, freeCpus, filter, &tnas, 0, currentCpuCounts, freeCpuCounts)\n\treturn tnas\n}\n\nfunc (t *cpuTreeNode) toAttributedSlice(\n\tcurrentCpus, freeCpus cpuset.CPUSet,\n\tfilter func(*cpuTreeNodeAttributes) bool,\n\ttnas *[]cpuTreeNodeAttributes,\n\tdepth int,\n\tcurrentCpuCounts []int,\n\tfreeCpuCounts []int) {\n\tcurrentCpusHere := t.cpus.Intersection(currentCpus)\n\tfreeCpusHere := t.cpus.Intersection(freeCpus)\n\tcurrentCpuCountHere := currentCpusHere.Size()\n\tcurrentCpuCountsHere := make([]int, len(currentCpuCounts)+1, len(currentCpuCounts)+1)\n\tcopy(currentCpuCountsHere, currentCpuCounts)\n\tcurrentCpuCountsHere[depth] = currentCpuCountHere\n\n\tfreeCpuCountHere := freeCpusHere.Size()\n\tfreeCpuCountsHere := make([]int, len(freeCpuCounts)+1, len(freeCpuCounts)+1)\n\tcopy(freeCpuCountsHere, freeCpuCounts)\n\tfreeCpuCountsHere[depth] = freeCpuCountHere\n\n\ttna := cpuTreeNodeAttributes{\n\t\tt:                t,\n\t\tdepth:            depth,\n\t\tcurrentCpus:      currentCpusHere,\n\t\tfreeCpus:         freeCpusHere,\n\t\tcurrentCpuCount:  currentCpuCountHere,\n\t\tcurrentCpuCounts: currentCpuCountsHere,\n\t\tfreeCpuCount:     freeCpuCountHere,\n\t\tfreeCpuCounts:    freeCpuCountsHere,\n\t}\n\n\tif filter != nil && !filter(&tna) {\n\t\treturn\n\t}\n\n\t*tnas = append(*tnas, tna)\n\tfor _, child := range t.children {\n\t\tchild.toAttributedSlice(currentCpus, freeCpus, filter,\n\t\t\ttnas, depth+1, currentCpuCountsHere, freeCpuCountsHere)\n\t}\n}\n\n// SplitLevel returns the root node of a new CPU tree where all\n// branches of a topology level have been split into new classes.\nfunc (t *cpuTreeNode) SplitLevel(splitLevel CPUTopologyLevel, cpuClassifier func(int) int) *cpuTreeNode {\n\tnewRoot := t.CopyTree()\n\tnewRoot.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\t// Dive into the level that will be split.\n\t\tif tn.level != splitLevel {\n\t\t\treturn nil\n\t\t}\n\t\t// Classify CPUs to the map: class -> list of cpus\n\t\tclassCpus := map[int][]int{}\n\t\tfor _, cpu := range t.cpus.List() {\n\t\t\tclass := cpuClassifier(cpu)\n\t\t\tclassCpus[class] = append(classCpus[class], cpu)\n\t\t}\n\t\t// Clear existing children of this node. New children\n\t\t// will be classes whose children are masked versions\n\t\t// of original children of this node.\n\t\torigChildren := tn.children\n\t\ttn.children = make([]*cpuTreeNode, 0, len(classCpus))\n\t\t// Add new child corresponding each class.\n\t\tfor class, cpus := range classCpus {\n\t\t\tcpuMask := cpuset.New(cpus...)\n\t\t\tnewNode := NewCpuTree(fmt.Sprintf(\"%sclass%d\", tn.name, class))\n\t\t\ttn.AddChild(newNode)\n\t\t\tnewNode.cpus = tn.cpus.Intersection(cpuMask)\n\t\t\tnewNode.level = tn.level\n\t\t\tnewNode.parent = tn\n\t\t\tfor _, child := range origChildren {\n\t\t\t\tnewChild := child.CopyTree()\n\t\t\t\tnewChild.DepthFirstWalk(func(cn *cpuTreeNode) error {\n\t\t\t\t\tcn.cpus = cn.cpus.Intersection(cpuMask)\n\t\t\t\t\tif cn.cpus.Size() == 0 && cn.parent != nil {\n\t\t\t\t\t\t// all cpus masked\n\t\t\t\t\t\t// out: cut out this\n\t\t\t\t\t\t// branch\n\t\t\t\t\t\tnewSiblings := []*cpuTreeNode{}\n\t\t\t\t\t\tfor _, child := range cn.parent.children {\n\t\t\t\t\t\t\tif child != cn {\n\t\t\t\t\t\t\t\tnewSiblings = append(newSiblings, child)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tcn.parent.children = newSiblings\n\t\t\t\t\t\treturn WalkSkipChildren\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t\tnewNode.AddChild(newChild)\n\t\t\t}\n\t\t}\n\t\treturn WalkSkipChildren\n\t})\n\treturn newRoot\n}\n\n// NewAllocator returns new CPU allocator for allocating CPUs from a\n// CPU tree branch.\nfunc (t *cpuTreeNode) NewAllocator(options cpuTreeAllocatorOptions) *cpuTreeAllocator {\n\tta := &cpuTreeAllocator{\n\t\troot:    t,\n\t\toptions: options,\n\t}\n\tif options.preferSpreadOnPhysicalCores {\n\t\tnewTree := t.SplitLevel(CPUTopologyLevelNuma,\n\t\t\t// CPU classifier: class of the CPU equals to\n\t\t\t// the index in the child list of its parent\n\t\t\t// node in the tree. Expect leaf node is a\n\t\t\t// hyperthread, parent a physical core.\n\t\t\tfunc(cpu int) int {\n\t\t\t\tleaf := t.FindLeafWithCpu(cpu)\n\t\t\t\tif leaf == nil {\n\t\t\t\t\tlog.Fatalf(\"SplitLevel CPU classifier: cpu %d not in tree:\\n%s\\n\\n\", cpu, t.PrettyPrint())\n\t\t\t\t}\n\t\t\t\treturn leaf.SiblingIndex()\n\t\t\t})\n\t\tta.root = newTree\n\t}\n\treturn ta\n}\n\n// sorterAllocate implements an \"is-less-than\" callback that helps\n// sorting a slice of cpuTreeNodeAttributes. The first item in the\n// sorted list contains an optimal CPU tree node for allocating new\n// CPUs.\nfunc (ta *cpuTreeAllocator) sorterAllocate(tnas []cpuTreeNodeAttributes) func(int, int) bool {\n\treturn func(i, j int) bool {\n\t\tif tnas[i].depth != tnas[j].depth {\n\t\t\treturn tnas[i].depth > tnas[j].depth\n\t\t}\n\t\tfor tdepth := 0; tdepth < len(tnas[i].currentCpuCounts); tdepth += 1 {\n\t\t\t// After this currentCpus will increase.\n\t\t\t// Maximize the maximal amount of currentCpus\n\t\t\t// as high level in the topology as possible.\n\t\t\tif tnas[i].currentCpuCounts[tdepth] != tnas[j].currentCpuCounts[tdepth] {\n\t\t\t\treturn tnas[i].currentCpuCounts[tdepth] > tnas[j].currentCpuCounts[tdepth]\n\t\t\t}\n\t\t}\n\t\tfor tdepth := 0; tdepth < len(tnas[i].freeCpuCounts); tdepth += 1 {\n\t\t\t// After this freeCpus will decrease.\n\t\t\tif tnas[i].freeCpuCounts[tdepth] != tnas[j].freeCpuCounts[tdepth] {\n\t\t\t\tif ta.options.topologyBalancing {\n\t\t\t\t\t// Goal: minimize maximal freeCpus in topology.\n\t\t\t\t\treturn tnas[i].freeCpuCounts[tdepth] > tnas[j].freeCpuCounts[tdepth]\n\t\t\t\t} else {\n\t\t\t\t\t// Goal: maximize maximal freeCpus in topology.\n\t\t\t\t\treturn tnas[i].freeCpuCounts[tdepth] < tnas[j].freeCpuCounts[tdepth]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn tnas[i].t.name < tnas[j].t.name\n\t}\n}\n\n// sorterRelease implements an \"is-less-than\" callback that helps\n// sorting a slice of cpuTreeNodeAttributes. The first item in the\n// list contains an optimal CPU tree node for releasing new CPUs.\nfunc (ta *cpuTreeAllocator) sorterRelease(tnas []cpuTreeNodeAttributes) func(int, int) bool {\n\treturn func(i, j int) bool {\n\t\tif tnas[i].depth != tnas[j].depth {\n\t\t\treturn tnas[i].depth > tnas[j].depth\n\t\t}\n\t\tfor tdepth := 0; tdepth < len(tnas[i].currentCpuCounts); tdepth += 1 {\n\t\t\t// After this currentCpus will decrease. Aim\n\t\t\t// to minimize the minimal amount of\n\t\t\t// currentCpus in order to decrease\n\t\t\t// fragmentation as high level in the topology\n\t\t\t// as possible.\n\t\t\tif tnas[i].currentCpuCounts[tdepth] != tnas[j].currentCpuCounts[tdepth] {\n\t\t\t\treturn tnas[i].currentCpuCounts[tdepth] < tnas[j].currentCpuCounts[tdepth]\n\t\t\t}\n\t\t}\n\t\tfor tdepth := 0; tdepth < len(tnas[i].freeCpuCounts); tdepth += 1 {\n\t\t\t// After this freeCpus will increase. Try to\n\t\t\t// maximize minimal free CPUs for better\n\t\t\t// isolation as high level in the topology as\n\t\t\t// possible.\n\t\t\tif tnas[i].freeCpuCounts[tdepth] != tnas[j].freeCpuCounts[tdepth] {\n\t\t\t\tif ta.options.topologyBalancing {\n\t\t\t\t\treturn tnas[i].freeCpuCounts[tdepth] < tnas[j].freeCpuCounts[tdepth]\n\t\t\t\t} else {\n\t\t\t\t\treturn tnas[i].freeCpuCounts[tdepth] < tnas[j].freeCpuCounts[tdepth]\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn tnas[i].t.name > tnas[j].t.name\n\t}\n}\n\n// ResizeCpus implements topology awareness to both adding CPUs to and\n// removing them from a set of CPUs. It returns CPUs from which actual\n// allocation or releasing of CPUs can be done. ResizeCpus does not\n// allocate or release CPUs.\n//\n// Parameters:\n//   - currentCpus: a set of CPUs to/from which CPUs would be added/removed.\n//   - freeCpus: a set of CPUs available CPUs.\n//   - delta: number of CPUs to add (if positive) or remove (if negative).\n//\n// Return values:\n//   - addFromCpus contains free CPUs from which delta CPUs can be\n//     allocated. Note that the size of the set may be larger than\n//     delta: there is room for other allocation logic to select from\n//     these CPUs.\n//   - removeFromCpus contains CPUs in currentCpus set from which\n//     abs(delta) CPUs can be freed.\nfunc (ta *cpuTreeAllocator) ResizeCpus(currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) {\n\tif delta > 0 {\n\t\taddFromSuperset, removeFromSuperset, err := ta.resizeCpus(currentCpus, freeCpus, delta)\n\t\tif !ta.options.preferSpreadOnPhysicalCores || addFromSuperset.Size() == delta {\n\t\t\treturn addFromSuperset, removeFromSuperset, err\n\t\t}\n\t\t// addFromSuperset contains more CPUs (equally good\n\t\t// choices) than actually needed. In case of\n\t\t// preferSpreadOnPhysicalCores, however, selecting any\n\t\t// of these does not result in equally good\n\t\t// result. Therefore, in this case, construct addFrom\n\t\t// set by adding one CPU at a time.\n\t\taddFrom := cpuset.New()\n\t\tfor n := 0; n < delta; n++ {\n\t\t\taddSingleFrom, _, err := ta.resizeCpus(currentCpus, freeCpus, 1)\n\t\t\tif err != nil {\n\t\t\t\treturn addFromSuperset, removeFromSuperset, err\n\t\t\t}\n\t\t\tif addSingleFrom.Size() != 1 {\n\t\t\t\treturn addFromSuperset, removeFromSuperset, fmt.Errorf(\"internal error: failed to find single CPU to allocate, \"+\n\t\t\t\t\t\"currentCpus=%s freeCpus=%s expectedSingle=%s\",\n\t\t\t\t\tcurrentCpus, freeCpus, addSingleFrom)\n\t\t\t}\n\t\t\taddFrom = addFrom.Union(addSingleFrom)\n\t\t\tif addFrom.Size() != n+1 {\n\t\t\t\treturn addFromSuperset, removeFromSuperset, fmt.Errorf(\"internal error: double add the same CPU (%s) to cpuset %s on round %d\",\n\t\t\t\t\taddSingleFrom, addFrom, n+1)\n\t\t\t}\n\t\t\tcurrentCpus = currentCpus.Union(addSingleFrom)\n\t\t\tfreeCpus = freeCpus.Difference(addSingleFrom)\n\t\t}\n\t\treturn addFrom, removeFromSuperset, nil\n\t}\n\t// In multi-CPU removal, remove CPUs one by one instead of\n\t// trying to find a single topology element from which all of\n\t// them could be removed.\n\tremoveFrom := cpuset.New()\n\taddFrom := cpuset.New()\n\tfor n := 0; n < -delta; n++ {\n\t\t_, removeSingleFrom, err := ta.resizeCpus(currentCpus, freeCpus, -1)\n\t\tif err != nil {\n\t\t\treturn addFrom, removeFrom, err\n\t\t}\n\t\t// Make cheap internal error checks in order to capture\n\t\t// issues in alternative algorithms.\n\t\tif removeSingleFrom.Size() != 1 {\n\t\t\treturn addFrom, removeFrom, fmt.Errorf(\"internal error: failed to find single cpu to free, \"+\n\t\t\t\t\"currentCpus=%s freeCpus=%s expectedSingle=%s\",\n\t\t\t\tcurrentCpus, freeCpus, removeSingleFrom)\n\t\t}\n\t\tif removeFrom.Union(removeSingleFrom).Size() != n+1 {\n\t\t\treturn addFrom, removeFrom, fmt.Errorf(\"internal error: double release of a cpu, \"+\n\t\t\t\t\"currentCpus=%s freeCpus=%s alreadyRemoved=%s removedNow=%s\",\n\t\t\t\tcurrentCpus, freeCpus, removeFrom, removeSingleFrom)\n\t\t}\n\t\tremoveFrom = removeFrom.Union(removeSingleFrom)\n\t\tcurrentCpus = currentCpus.Difference(removeSingleFrom)\n\t\tfreeCpus = freeCpus.Union(removeSingleFrom)\n\t}\n\treturn addFrom, removeFrom, nil\n}\n\nfunc (ta *cpuTreeAllocator) resizeCpus(currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) {\n\ttnas := ta.root.ToAttributedSlice(currentCpus, freeCpus,\n\t\tfunc(tna *cpuTreeNodeAttributes) bool {\n\t\t\t// filter out branches with insufficient cpus\n\t\t\tif delta > 0 && tna.freeCpuCount-delta < 0 {\n\t\t\t\t// cannot allocate delta cpus\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif delta < 0 && tna.currentCpuCount+delta < 0 {\n\t\t\t\t// cannot release delta cpus\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\n\t// Sort based on attributes\n\tif delta > 0 {\n\t\tsort.Slice(tnas, ta.sorterAllocate(tnas))\n\t} else {\n\t\tsort.Slice(tnas, ta.sorterRelease(tnas))\n\t}\n\tif len(tnas) == 0 {\n\t\treturn freeCpus, currentCpus, fmt.Errorf(\"not enough free CPUs\")\n\t}\n\treturn tnas[0].freeCpus, tnas[0].currentCpus, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/cputree_test.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\ntype cpuInTopology struct {\n\tpackageID, dieID, numaID, coreID, threadID, cpuID             int\n\tpackageName, dieName, numaName, coreName, threadName, cpuName string\n}\n\ntype cpusInTopology map[int]cpuInTopology\n\nfunc (cit cpuInTopology) TopoName(topoLevel string) string {\n\tswitch topoLevel {\n\tcase \"thread\":\n\t\treturn cit.threadName\n\tcase \"core\":\n\t\treturn cit.coreName\n\tcase \"numa\":\n\t\treturn cit.numaName\n\tcase \"die\":\n\t\treturn cit.dieName\n\tcase \"package\":\n\t\treturn cit.packageName\n\t}\n\tpanic(\"invalid topoLevel\")\n}\n\nfunc (csit cpusInTopology) dumps(nameCpus map[string]cpuset.CPUSet) string {\n\tlines := []string{}\n\tnames := make([]string, 0, len(nameCpus))\n\tfor name := range nameCpus {\n\t\tnames = append(names, name)\n\t}\n\tsort.Strings(names)\n\tfor cpuID := 0; cpuID < len(csit); cpuID++ {\n\t\tline := fmt.Sprintf(\"cpu%02d %s\", cpuID, csit[cpuID].threadName)\n\t\tfor _, name := range names {\n\t\t\tif nameCpus[name].Contains(cpuID) {\n\t\t\t\tline = fmt.Sprintf(\"%s %s\", line, name)\n\t\t\t}\n\t\t}\n\t\tlines = append(lines, line)\n\t}\n\treturn strings.Join(lines, \"\\n\")\n}\n\nfunc newCpuTreeFromInt5(pdnct [5]int) (*cpuTreeNode, cpusInTopology) {\n\tpkgs := pdnct[0]\n\tdies := pdnct[1]\n\tnumas := pdnct[2]\n\tcores := pdnct[3]\n\tthreads := pdnct[4]\n\tcpuID := 0\n\tsysTree := NewCpuTree(\"system\")\n\tsysTree.level = CPUTopologyLevelSystem\n\tcsit := cpusInTopology{}\n\tfor packageID := 0; packageID < pkgs; packageID++ {\n\t\tpackageTree := NewCpuTree(fmt.Sprintf(\"p%d\", packageID))\n\t\tpackageTree.level = CPUTopologyLevelPackage\n\t\tsysTree.AddChild(packageTree)\n\t\tfor dieID := 0; dieID < dies; dieID++ {\n\t\t\tdieTree := NewCpuTree(fmt.Sprintf(\"p%dd%d\", packageID, dieID))\n\t\t\tdieTree.level = CPUTopologyLevelDie\n\t\t\tpackageTree.AddChild(dieTree)\n\t\t\tfor numaID := 0; numaID < numas; numaID++ {\n\t\t\t\tnumaTree := NewCpuTree(fmt.Sprintf(\"p%dd%dn%d\", packageID, dieID, numaID))\n\t\t\t\tnumaTree.level = CPUTopologyLevelNuma\n\t\t\t\tdieTree.AddChild(numaTree)\n\t\t\t\tfor coreID := 0; coreID < cores; coreID++ {\n\t\t\t\t\tcoreTree := NewCpuTree(fmt.Sprintf(\"p%dd%dn%dc%02d\", packageID, dieID, numaID, coreID))\n\t\t\t\t\tcoreTree.level = CPUTopologyLevelCore\n\t\t\t\t\tnumaTree.AddChild(coreTree)\n\t\t\t\t\tfor threadID := 0; threadID < threads; threadID++ {\n\t\t\t\t\t\tthreadTree := NewCpuTree(fmt.Sprintf(\"p%dd%dn%dc%02dt%d\", packageID, dieID, numaID, coreID, threadID))\n\t\t\t\t\t\tthreadTree.level = CPUTopologyLevelThread\n\t\t\t\t\t\tcoreTree.AddChild(threadTree)\n\t\t\t\t\t\tthreadTree.AddCpus(cpuset.New(cpuID))\n\t\t\t\t\t\tcsit[cpuID] = cpuInTopology{\n\t\t\t\t\t\t\tpackageID, dieID, numaID, coreID, threadID, cpuID,\n\t\t\t\t\t\t\tpackageTree.name, dieTree.name, numaTree.name, coreTree.name, threadTree.name,\n\t\t\t\t\t\t\tfmt.Sprintf(\"cpu%d\", cpuID),\n\t\t\t\t\t\t}\n\t\t\t\t\t\tcpuID += 1\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn sysTree, csit\n}\n\nfunc verifyNotOn(t *testing.T, nameContents string, cpus cpuset.CPUSet, csit cpusInTopology) {\n\tfor _, cpuID := range cpus.List() {\n\t\tname := csit[cpuID].threadName\n\t\tif strings.Contains(name, nameContents) {\n\t\t\tt.Errorf(\"cpu%d (%s) in unexpected region %s\", cpuID, name, nameContents)\n\t\t}\n\t}\n}\n\nfunc doVerifySame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusInTopology, inversed bool) {\n\tseenName := \"\"\n\tseenCpuID := -1\n\tfor _, cpuID := range cpus.List() {\n\t\tcit := csit[cpuID]\n\t\tthisName := cit.TopoName(topoLevel)\n\t\tthisCpuID := cit.cpuID\n\t\tif thisName == \"\" {\n\t\t\tt.Errorf(\"unexpected (invalid) topology level %q\", topoLevel)\n\t\t}\n\t\tif seenName == \"\" {\n\t\t\tseenName = thisName\n\t\t\tseenCpuID = cit.cpuID\n\t\t\tcontinue\n\t\t}\n\t\tif (seenName != thisName && !inversed) ||\n\t\t\t(seenName == thisName && inversed) {\n\t\t\tmsg := \"the same\"\n\t\t\tif inversed {\n\t\t\t\tmsg = \"not the same\"\n\t\t\t}\n\t\t\tt.Errorf(\"expected %s %s, got: cpu%d in %s, cpu%d in %s\",\n\t\t\t\tmsg,\n\t\t\t\ttopoLevel,\n\t\t\t\tseenCpuID, seenName,\n\t\t\t\tthisCpuID, thisName)\n\t\t}\n\t}\n}\n\nfunc verifySame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusInTopology) {\n\tdoVerifySame(t, topoLevel, cpus, csit, false)\n}\n\nfunc verifyNotSame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusInTopology) {\n\tdoVerifySame(t, topoLevel, cpus, csit, true)\n}\n\nfunc (csit cpusInTopology) getElements(topoLevel string, cpus cpuset.CPUSet) []string {\n\telts := []string{}\n\tfor _, cpuID := range cpus.List() {\n\t\telts = append(elts, csit[cpuID].TopoName(topoLevel))\n\t}\n\treturn elts\n}\n\nfunc (csit cpusInTopology) verifyDisjoint(t *testing.T, topoLevel string, cpusA cpuset.CPUSet, cpusB cpuset.CPUSet) {\n\teltsA := csit.getElements(topoLevel, cpusA)\n\teltsB := csit.getElements(topoLevel, cpusB)\n\tfor _, eltA := range eltsA {\n\t\tfor _, eltB := range eltsB {\n\t\t\tif eltA == eltB {\n\t\t\t\tt.Errorf(\"expected disjoint %ss, got %s on both cpusets %s and %s\",\n\t\t\t\t\ttopoLevel, eltA, cpusA, cpusB)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n/*\n   CPU ids and locations in the 2-2-2-2-2-topology for verifying\n   current and developing future unit tests. The location in topology\n   is in format:\n\n   p<package-id>/d<die-id>/n<numa-id>/c<core-index>/t<thread-id>\n\ntopology: [5]int{2, 2, 2, 2, 2},\nallocations: []int{\n\t0,  // cpu on p0/d0/n0/c0/t0\n\t1,  // cpu on p0/d0/n0/c0/t1\n\t2,  // cpu on p0/d0/n0/c1/t0\n\t3,  // cpu on p0/d0/n0/c1/t1\n\t4,  // cpu on p0/d0/n1/c0/t0\n\t5,  // cpu on p0/d0/n1/c0/t1\n\t6,  // cpu on p0/d0/n1/c1/t0\n\t7,  // cpu on p0/d0/n1/c1/t1\n\t8,  // cpu on p0/d1/n0/c0/t0\n\t9,  // cpu on p0/d1/n0/c0/t1\n\t10, // cpu on p0/d1/n0/c1/t0\n\t11, // cpu on p0/d1/n0/c1/t1\n\t12, // cpu on p0/d1/n1/c0/t0\n\t13, // cpu on p0/d1/n1/c0/t1\n\t14, // cpu on p0/d1/n1/c1/t0\n\t15, // cpu on p0/d1/n1/c1/t1\n\t16, // cpu on p1/d0/n0/c0/t0\n\t17, // cpu on p1/d0/n0/c0/t1\n\t18, // cpu on p1/d0/n0/c1/t0\n\t19, // cpu on p1/d0/n0/c1/t1\n\t20, // cpu on p1/d0/n1/c0/t0\n\t21, // cpu on p1/d0/n1/c0/t1\n\t22, // cpu on p1/d0/n1/c1/t0\n\t23, // cpu on p1/d0/n1/c1/t1\n\t24, // cpu on p1/d1/n0/c0/t0\n\t25, // cpu on p1/d1/n0/c0/t1\n\t26, // cpu on p1/d1/n0/c1/t0\n\t27, // cpu on p1/d1/n0/c1/t1\n\t28, // cpu on p1/d1/n1/c0/t0\n\t29, // cpu on p1/d1/n1/c0/t1\n\t30, // cpu on p1/d1/n1/c1/t0\n\t31, // cpu on p1/d1/n1/c1/t1\n},\n*/\n\nfunc TestResizeCpus(t *testing.T) {\n\ttype TopoCcids struct {\n\t\ttopo  string\n\t\tccids []int\n\t}\n\ttcases := []struct {\n\t\tname                   string\n\t\ttopology               [5]int // package, die, numa, core, thread count\n\t\tallocatorTB            bool   // allocator topologyBalancing\n\t\tallocatorPSoPC         bool   // allocator preferSpreadOnPhysicalCores\n\t\tallocations            []int\n\t\tdeltas                 []int\n\t\tallocate               bool\n\t\toperateOnCcid          []int // which ccid (currentCpus id) will be used on call\n\t\texpectCurrentOnSame    []string\n\t\texpectCurrentNotOnSame []string\n\t\texpectAllOnSame        []string\n\t\texpectCurrentNotOn     []string\n\t\texpectAddSizes         []int\n\t\texpectDisjoint         []TopoCcids // which ccids should be disjoint\n\t\texpectErrors           []string\n\t}{\n\t\t{\n\t\t\tname:           \"first allocations\",\n\t\t\ttopology:       [5]int{2, 2, 2, 2, 2},\n\t\t\tdeltas:         []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32},\n\t\t\texpectAddSizes: []int{0, 1, 2, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32},\n\t\t},\n\t\t{\n\t\t\tname:         \"too large an allocation\",\n\t\t\ttopology:     [5]int{2, 2, 2, 2, 2},\n\t\t\tdeltas:       []int{33},\n\t\t\texpectErrors: []string{\"not enough free CPUs\"},\n\t\t},\n\t\t{\n\t\t\tname:          \"spread allocations\",\n\t\t\ttopology:      [5]int{2, 2, 2, 2, 2},\n\t\t\tallocatorTB:   true,\n\t\t\tdeltas:        []int{1, 1, 1, 1, 1, 1, 1, 1},\n\t\t\tallocate:      true,\n\t\t\toperateOnCcid: []int{1, 2, 3, 4, 5, 6, 7, 8},\n\t\t\texpectDisjoint: []TopoCcids{\n\t\t\t\t{},\n\t\t\t\t{\"package\", []int{1, 2}},\n\t\t\t\t{\"die\", []int{1, 2, 3}},\n\t\t\t\t{\"die\", []int{1, 2, 3, 4}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5, 6}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5, 6, 7}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5, 6, 7, 8}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:          \"spread allocations2\",\n\t\t\ttopology:      [5]int{4, 1, 4, 8, 2},\n\t\t\tallocatorTB:   true,\n\t\t\tdeltas:        []int{1, 3, 2, 4, 1, 4, 2, 4},\n\t\t\tallocate:      true,\n\t\t\toperateOnCcid: []int{1, 2, 3, 4, 5, 6, 7, 8},\n\t\t\texpectDisjoint: []TopoCcids{\n\t\t\t\t{},\n\t\t\t\t{\"package\", []int{1, 2}},\n\t\t\t\t{\"package\", []int{1, 2, 3}},\n\t\t\t\t{\"package\", []int{1, 2, 3, 4}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5, 6}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5, 6, 7}},\n\t\t\t\t{\"numa\", []int{1, 2, 3, 4, 5, 6, 7, 8}},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:          \"pack allocations\",\n\t\t\ttopology:      [5]int{2, 2, 2, 2, 2},\n\t\t\tallocatorTB:   false,\n\t\t\tdeltas:        []int{1, 1, 1, 1},\n\t\t\tallocate:      true,\n\t\t\toperateOnCcid: []int{1, 2, 3, 4, 5},\n\t\t\texpectAllOnSame: []string{\n\t\t\t\t\"\", \"core\", \"numa\", \"numa\", \"die\", \"die\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:     \"inflate\",\n\t\t\ttopology: [5]int{2, 2, 2, 2, 2},\n\t\t\tallocate: true,\n\t\t\tdeltas: []int{\n\t\t\t\t1, 1, 1, 1, // cpu0..cpu3 on numaN0, dieD0\n\t\t\t\t1, 3, // cpu4..cpu7 on numaN1, still dieD0\n\t\t\t\t6, 1, 1, // cpu8..15 on dieD1, still packageP0\n\t\t\t},\n\t\t\toperateOnCcid: []int{\n\t\t\t\t1, 1, 1, 1,\n\t\t\t\t1, 1,\n\t\t\t\t1, 1, 1},\n\t\t\texpectCurrentOnSame: []string{\n\t\t\t\t\"core\", \"core\", \"numa\", \"numa\",\n\t\t\t\t\"die\", \"die\",\n\t\t\t\t\"package\", \"package\", \"package\"},\n\t\t\texpectAddSizes: []int{\n\t\t\t\t1, 1, 1, 1,\n\t\t\t\t1, 3,\n\t\t\t\t8, 1, 1},\n\t\t},\n\t\t{\n\t\t\tname:     \"defragmenting single removals\",\n\t\t\ttopology: [5]int{2, 2, 2, 2, 2},\n\t\t\tallocations: []int{\n\t\t\t\t0,  // cpu on p0/d0/n0/c0/t0\n\t\t\t\t2,  // cpu on p0/d0/n0/c1/t0\n\t\t\t\t3,  // cpu on p0/d0/n0/c1/t1\n\t\t\t\t7,  // cpu on p0/d0/n1/c1/t1\n\t\t\t\t10, // cpu on p0/d1/n0/c1/t0\n\t\t\t\t17, // cpu on p1/d0/n0/c0/t1\n\t\t\t\t18, // cpu on p1/d0/n0/c1/t0\n\t\t\t},\n\t\t\tallocate: true,\n\t\t\tdeltas: []int{\n\t\t\t\t-1, // release cpu17 or cpu18\n\t\t\t\t-1, // release cpu17 or cpu18 => all on same package\n\t\t\t\t-1, // release cpu10 => all on same die\n\t\t\t\t-1, // release cpu7 => all on same numa\n\t\t\t\t-1, // release cpu0 => all on same core\n\t\t\t\t-1, // release cpu2 or cpu3\n\t\t\t\t-1, // release cpu2 or cpu3\n\t\t\t},\n\t\t\toperateOnCcid: []int{1, 1, 1, 1, 1, 1, 1},\n\t\t\texpectCurrentOnSame: []string{\n\t\t\t\t\"\",\n\t\t\t\t\"package\",\n\t\t\t\t\"die\",\n\t\t\t\t\"numa\",\n\t\t\t\t\"core\",\n\t\t\t\t\"core\",\n\t\t\t\t\"core\",\n\t\t\t},\n\t\t\texpectCurrentNotOn: []string{\n\t\t\t\t\"\",\n\t\t\t\t\"p1\",\n\t\t\t\t\"p0d1\",\n\t\t\t\t\"p0d0n1\",\n\t\t\t\t\"p0d0n0c00\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:     \"defragmenting multi-removals\",\n\t\t\ttopology: [5]int{2, 2, 2, 2, 2},\n\t\t\tallocations: []int{\n\t\t\t\t0,  // cpu on p0/d0/n0/c0/t0\n\t\t\t\t2,  // cpu on p0/d0/n0/c1/t0\n\t\t\t\t4,  // cpu on p0/d0/n1/c0/t0\n\t\t\t\t6,  // cpu on p0/d0/n1/c1/t0\n\t\t\t\t8,  // cpu on p0/d1/n0/c0/t0\n\t\t\t\t9,  // cpu on p0/d1/n0/c0/t1\n\t\t\t\t10, // cpu on p0/d1/n0/c1/t0\n\n\t\t\t\t24, // cpu on p1/d1/n0/c0/t0\n\t\t\t\t25, // cpu on p1/d1/n0/c0/t1\n\t\t\t\t26, // cpu on p1/d1/n0/c1/t0\n\t\t\t\t27, // cpu on p1/d1/n0/c1/t1\n\t\t\t\t28, // cpu on p1/d1/n1/c0/t0\n\t\t\t\t29, // cpu on p1/d1/n1/c0/t1\n\t\t\t\t30, // cpu on p1/d1/n1/c1/t0\n\t\t\t\t31, // cpu on p1/d1/n1/c1/t1\n\t\t\t},\n\t\t\tallocate: true,\n\t\t\tdeltas: []int{\n\t\t\t\t-2, // release from p0d1n0\n\t\t\t\t-1, // release completely p0d1\n\t\t\t\t-5, // release completely p0, one from p1d1nX\n\t\t\t\t-3, // release completely p1d1nX => all on same numa\n\t\t\t},\n\t\t\toperateOnCcid: []int{1, 1, 1, 1},\n\t\t\texpectCurrentOnSame: []string{\n\t\t\t\t\"\",\n\t\t\t\t\"\",\n\t\t\t\t\"die\",\n\t\t\t\t\"numa\",\n\t\t\t},\n\t\t\texpectCurrentNotOn: []string{\n\t\t\t\t\"\",\n\t\t\t\t\"p0d1\",\n\t\t\t\t\"p0\",\n\t\t\t\t\"\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:     \"gentle rebalancing\",\n\t\t\ttopology: [5]int{2, 1, 1, 16, 2}, // 2 packages, 16 hyperthreaded cores per package => 64 cpus in total\n\t\t\tdeltas: []int{\n\t\t\t\t4, 4, 14, 7, 7, 4, 4, 14, // allocate 8 sets of cpus, the last 14cpus fills package0, spills over to package1\n\t\t\t\t-2, -2, -2, -2, // free a little room to package0\n\t\t\t\t-1, 1, -1, 1, -1, 1, -1, 1}, // deflate/inflate the last 14cpus, see that it gradually travels to package0\n\t\t\toperateOnCcid: []int{\n\t\t\t\t1, 2, 3, 4, 5, 6, 7, 8,\n\t\t\t\t1, 2, 3, 4,\n\t\t\t\t8, 8, 8, 8, 8, 8, 8, 8,\n\t\t\t},\n\t\t\tallocate: true,\n\t\t\texpectCurrentOnSame: []string{\n\t\t\t\t\"package\", \"package\", \"package\", \"package\",\n\t\t\t\t\"package\", \"package\", \"package\", \"\",\n\t\t\t\t\"\", \"\", \"\", \"\",\n\t\t\t\t\"\", \"\", \"\", \"\", \"\", \"\", \"package\", \"package\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:           \"prefer spread on physical cores\",\n\t\t\ttopology:       [5]int{4, 1, 4, 8, 2},\n\t\t\tallocatorTB:    true,\n\t\t\tallocatorPSoPC: true,\n\t\t\tdeltas: []int{\n\t\t\t\t2, 1, 4, 1, // allocate one thread from each core from the same NUMA\n\t\t\t\t3, 9, 16, // allocate three other cpusets, each should be from separate package (due to topology balancing)\n\t\t\t\t3, 4, 3, // increase the size of the\n\t\t\t\t// original, 3+4 fits to the same\n\t\t\t\t// NUMA, in the last 3: first cpu\n\t\t\t\t// should fill the NUMA and the rest 2\n\t\t\t\t// go to another NUMA on the same package.\n\t\t\t\t-2, 2, // release two CPUs that went to another NUMA on the same package, and put them back\n\t\t\t\t-10, // release 2+8 CPUs, the rest should be single threads each on their own core\n\t\t\t},\n\t\t\tallocate: true,\n\t\t\toperateOnCcid: []int{\n\t\t\t\t1, 1, 1, 1, // allocate one thread from each core from the same NUMA by inflating all the time the same cpuset\n\t\t\t\t2, 3, 4, // three new cpusets\n\t\t\t\t1, 1, 1, // increase size over one NUMA\n\t\t\t\t1, 1,\n\t\t\t\t1,\n\t\t\t},\n\t\t\texpectCurrentOnSame: []string{\n\t\t\t\t\"numa\", \"numa\", \"numa\", \"numa\",\n\t\t\t\t\"numa\", \"numa\", \"numa\",\n\t\t\t\t\"numa\", \"numa\", \"package\",\n\t\t\t\t\"numa\", \"package\",\n\t\t\t\t\"numa\",\n\t\t\t},\n\t\t\texpectCurrentNotOnSame: []string{\n\t\t\t\t\"core\", \"core\", \"core\", \"core\",\n\t\t\t\t\"core\", \"\", \"\",\n\t\t\t\t\"\", \"\", \"\",\n\t\t\t\t\"\", \"\",\n\t\t\t\t\"core\",\n\t\t\t},\n\t\t\texpectDisjoint: []TopoCcids{\n\t\t\t\t{}, {}, {}, {},\n\t\t\t\t{\"package\", []int{1, 2}}, {\"package\", []int{1, 2, 3}}, {\"package\", []int{1, 2, 3, 4}},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\ttree, csit := newCpuTreeFromInt5(tc.topology)\n\t\t\ttreeA := tree.NewAllocator(cpuTreeAllocatorOptions{\n\t\t\t\ttopologyBalancing:           tc.allocatorTB,\n\t\t\t\tpreferSpreadOnPhysicalCores: tc.allocatorPSoPC,\n\t\t\t})\n\t\t\tcurrentCpus := cpuset.New()\n\t\t\tfreeCpus := tree.Cpus()\n\t\t\tif len(tc.allocations) > 0 {\n\t\t\t\tcurrentCpus = currentCpus.Union(cpuset.New(tc.allocations...))\n\t\t\t\tfreeCpus = freeCpus.Difference(cpuset.New(tc.allocations...))\n\t\t\t}\n\t\t\tccidCurrentCpus := map[int]cpuset.CPUSet{0: currentCpus}\n\t\t\tallocs := map[string]cpuset.CPUSet{\"--:allo\": currentCpus}\n\t\t\tfor i, delta := range tc.deltas {\n\t\t\t\tif i < len(tc.operateOnCcid) && tc.operateOnCcid[i] > 0 {\n\t\t\t\t\tcurrentCpus = ccidCurrentCpus[tc.operateOnCcid[i]]\n\t\t\t\t}\n\t\t\t\tt.Logf(\"ResizeCpus(current=%s; free=%s; delta=%d)\", currentCpus, freeCpus, delta)\n\t\t\t\taddFrom, removeFrom, err := treeA.ResizeCpus(currentCpus, freeCpus, delta)\n\t\t\t\tt.Logf(\"== addFrom=%s; removeFrom=%s, err=%v\", addFrom, removeFrom, err)\n\t\t\t\tif i < len(tc.expectAddSizes) {\n\t\t\t\t\tif tc.expectAddSizes[i] != addFrom.Size() {\n\t\t\t\t\t\tt.Errorf(\"expected add size: %d, got %d\", tc.expectAddSizes[i], addFrom.Size())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif i < len(tc.expectErrors) {\n\t\t\t\t\tif tc.expectErrors[i] == \"\" && err != nil {\n\t\t\t\t\t\tt.Errorf(\"expected nil error, but got %v\", err)\n\t\t\t\t\t}\n\t\t\t\t\tif tc.expectErrors[i] != \"\" {\n\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\tt.Errorf(\"expected error containing %q, got nil\", tc.expectErrors[i])\n\t\t\t\t\t\t} else if !strings.Contains(fmt.Sprintf(\"%s\", err), tc.expectErrors[i]) {\n\t\t\t\t\t\t\tt.Errorf(\"expected error containing %q, got %q\", tc.expectErrors[i], err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif tc.allocate {\n\t\t\t\t\tallocName := fmt.Sprintf(\"%02d:allo\", i+1)\n\t\t\t\t\tallocs[allocName] = cpuset.New()\n\n\t\t\t\t\tfor n, cpuID := range addFrom.List() {\n\t\t\t\t\t\tif n >= delta {\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfreeCpus = freeCpus.Difference(cpuset.New(cpuID))\n\t\t\t\t\t\tcurrentCpus = currentCpus.Union(cpuset.New(cpuID))\n\t\t\t\t\t\tallocs[allocName] = allocs[allocName].Union(cpuset.New(cpuID))\n\t\t\t\t\t}\n\t\t\t\t\tallocName = fmt.Sprintf(\"%02d:free\", i+1)\n\t\t\t\t\tfor n, cpuID := range removeFrom.List() {\n\t\t\t\t\t\tif n >= -delta {\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t\tfreeCpus = freeCpus.Union(cpuset.New(cpuID))\n\t\t\t\t\t\tif i < len(tc.operateOnCcid) && tc.operateOnCcid[i] > 0 {\n\t\t\t\t\t\t\tcurrentCpus = currentCpus.Difference(cpuset.New(cpuID))\n\t\t\t\t\t\t}\n\t\t\t\t\t\tallocs[allocName] = allocs[allocName].Union(cpuset.New(cpuID))\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(tc.operateOnCcid) && tc.operateOnCcid[i] > 0 {\n\t\t\t\t\t\tccidCurrentCpus[tc.operateOnCcid[i]] = currentCpus\n\t\t\t\t\t}\n\n\t\t\t\t\tallocs[\"free\"] = freeCpus\n\t\t\t\t\tt.Logf(\"=> current=%s; free=%s\", currentCpus, freeCpus)\n\t\t\t\t\tif i < len(tc.expectCurrentOnSame) && tc.expectCurrentOnSame[i] != \"\" {\n\t\t\t\t\t\tverifySame(t, tc.expectCurrentOnSame[i], currentCpus, csit)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(tc.expectCurrentNotOnSame) && tc.expectCurrentNotOnSame[i] != \"\" {\n\t\t\t\t\t\tverifyNotSame(t, tc.expectCurrentNotOnSame[i], currentCpus, csit)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(tc.expectCurrentNotOn) && tc.expectCurrentNotOn[i] != \"\" {\n\t\t\t\t\t\tverifyNotOn(t, tc.expectCurrentNotOn[i], currentCpus, csit)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(tc.expectAllOnSame) && tc.expectAllOnSame[i] != \"\" {\n\t\t\t\t\t\tallCpus := cpuset.New()\n\t\t\t\t\t\tfor _, cpus := range ccidCurrentCpus {\n\t\t\t\t\t\t\tallCpus = allCpus.Union(cpus)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tverifySame(t, tc.expectAllOnSame[i], allCpus, csit)\n\t\t\t\t\t}\n\n\t\t\t\t\tif i < len(tc.expectDisjoint) && len(tc.expectDisjoint) > 1 {\n\t\t\t\t\t\tfor first := 0; first < len(tc.expectDisjoint[i].ccids); first++ {\n\t\t\t\t\t\t\tfor second := first + 1; second < len(tc.expectDisjoint[i].ccids); second++ {\n\t\t\t\t\t\t\t\tcsit.verifyDisjoint(t, tc.expectDisjoint[i].topo,\n\t\t\t\t\t\t\t\t\tccidCurrentCpus[tc.expectDisjoint[i].ccids[first]],\n\t\t\t\t\t\t\t\t\tccidCurrentCpus[tc.expectDisjoint[i].ccids[second]])\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif t.Failed() {\n\t\t\t\t\tt.Logf(\"current and free cpus:\\n%s\\n\", csit.dumps(allocs))\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWalk(t *testing.T) {\n\tt.Run(\"single-node tree\", func(t *testing.T) {\n\t\ttree := NewCpuTree(\"system\")\n\t\ttree.level = CPUTopologyLevelSystem\n\t\tfoundName := \"unfound\"\n\t\tfoundLevel := CPUTopologyLevelUndefined\n\t\trv := tree.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\t\tfoundName = tn.name\n\t\t\tfoundLevel = tn.level\n\t\t\treturn nil\n\t\t})\n\t\tif rv != nil {\n\t\t\tt.Errorf(\"expected no error, got %s\", rv)\n\t\t}\n\t\tif foundLevel != CPUTopologyLevelSystem {\n\t\t\tt.Errorf(\"expected to find level %q, got %q\",\n\t\t\t\tCPUTopologyLevelSystem, foundLevel)\n\t\t}\n\t\tif foundName != \"system\" {\n\t\t\tt.Errorf(\"expected to find name %q, got %q\",\n\t\t\t\t\"system\", foundName)\n\t\t}\n\t})\n\n\tt.Run(\"fetch first core\", func(t *testing.T) {\n\t\ttree, _ := newCpuTreeFromInt5([5]int{2, 2, 2, 2, 2})\n\t\tfoundCount := 0\n\t\tfoundName := \"\"\n\t\trv := tree.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\t\tfoundCount += 1\n\t\t\tif tn.level == CPUTopologyLevelCore {\n\t\t\t\tfoundName = tn.name\n\t\t\t\treturn WalkStop\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif rv != WalkStop {\n\t\t\tt.Errorf(\"expected WalkStop error, got %s\", rv)\n\t\t}\n\t\tif foundCount != 5 {\n\t\t\tt.Errorf(\"expected to find 5 nodes, got %d\", foundCount)\n\t\t}\n\t\tif foundName != \"p0d0n0c00\" {\n\t\t\tt.Errorf(\"expected to find p0d0n0c00, got %q\", foundName)\n\t\t}\n\t})\n\n\tt.Run(\"skip children\", func(t *testing.T) {\n\t\ttree, _ := newCpuTreeFromInt5([5]int{2, 2, 2, 2, 2})\n\t\tfoundCount := 0\n\t\trv := tree.DepthFirstWalk(func(tn *cpuTreeNode) error {\n\t\t\tfoundCount += 1\n\t\t\tif tn.level == CPUTopologyLevelDie {\n\t\t\t\treturn WalkSkipChildren\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t\tif rv != nil {\n\t\t\tt.Errorf(\"expected no error, got %s\", rv)\n\t\t}\n\t\tif foundCount != 7 {\n\t\t\tt.Errorf(\"expected to find 7 nodes, got %d\", foundCount)\n\t\t}\n\t})\n}\n\nfunc TestCpuLocations(t *testing.T) {\n\ttree, _ := newCpuTreeFromInt5([5]int{2, 2, 2, 4, 2})\n\tcpus := cpuset.New(0, 1, 3, 4, 16)\n\tsystemlocations := tree.CpuLocations(cpus)\n\tpackage1locations := tree.children[1].CpuLocations(cpus)\n\tif len(package1locations) != 5 {\n\t\tt.Errorf(\"expected package1locations length 5, got %d\", len(package1locations))\n\t\treturn\n\t}\n\tif len(systemlocations) != 6 {\n\t\tt.Errorf(\"expected systemlocations length 6, got %d\", len(systemlocations))\n\t\treturn\n\t}\n\tif systemlocations[0][0] != \"system\" {\n\t\tt.Errorf(\"expected 'system' location, got %q\", systemlocations[0][0])\n\t\treturn\n\t}\n\tif systemlocations[1][0] != \"p0\" {\n\t\tt.Errorf(\"expected 'system' location, got %q\", systemlocations[1][0])\n\t\treturn\n\t}\n\tif len(systemlocations[4]) != 4 {\n\t\tt.Errorf(\"expected len(systemlocations[4]) 4, got %d\", len(systemlocations[4]))\n\t\treturn\n\t}\n}\n\nfunc TestCPUTopologyLevel(t *testing.T) {\n\tvar lvl CPUTopologyLevel\n\tif lvl != CPUTopologyLevelUndefined {\n\t\tt.Errorf(\"unexpected default inital value for lvl: %s, expected undefined\", lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"\\\"\\\"\")); err != nil || lvl != CPUTopologyLevelUndefined {\n\t\tt.Errorf(\"unexpected outcome unmarshalling topology level: \\\"\\\", error: %s, result: %s\", err, lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"\\\"system\\\"\")); err != nil || lvl != CPUTopologyLevelSystem {\n\t\tt.Errorf(\"unexpected outcome unmarshalling topology level: system, error: %s, result: %s\", err, lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"\\\"NUMA\\\"\")); err != nil || lvl != CPUTopologyLevelNuma {\n\t\tt.Errorf(\"unexpected outcome unmarshalling topology level: \\\"NUMA\\\", error: %s, result: %s\", err, lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"\\\"undefined\\\"\")); err == nil {\n\t\tt.Errorf(\"unexpected outcome unmarshalling topology level: \\\"undefined\\\", error: %s, result: %s\", err, lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"system\")); err == nil {\n\t\tt.Errorf(\"unexpected non-error outcome unmarshalling topology level: system, error: %s, result: %s\", err, lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"0\")); err == nil {\n\t\tt.Errorf(\"unexpected non-error outcome unmarshalling topology level: 0, error: %s, result: %s\", err, lvl)\n\t}\n\tif err := lvl.UnmarshalJSON([]byte(\"\\\"4\\\"\")); err == nil {\n\t\tt.Errorf(\"unexpected non-error outcome unmarshalling topology level: \\\"0\\\", error: %s, result: %s\", err, lvl)\n\t}\n\tif undefBytes, err := CPUTopologyLevelUndefined.MarshalJSON(); err != nil {\n\t\tt.Errorf(\"unexpected error marshaling undefined: %s\", err)\n\t} else {\n\t\tif err = lvl.UnmarshalJSON(undefBytes); err != nil || lvl != CPUTopologyLevelUndefined {\n\t\t\tt.Errorf(\"unexpected outcome unmarshaling marshaled undefined: error: %s, result: %s\", err, lvl)\n\t\t}\n\t}\n\tif threadBytes, err := CPUTopologyLevelThread.MarshalJSON(); err != nil {\n\t\tt.Errorf(\"unexpected error marshaling thread: %s\", err)\n\t} else {\n\t\tif err = lvl.UnmarshalJSON(threadBytes); err != nil || lvl != CPUTopologyLevelThread {\n\t\t\tt.Errorf(\"unexpected outcome unmarshaling marshaled thread: error: %s, result: %s\", err, lvl)\n\t\t}\n\t}\n\n}\n\nfunc TestSplitLevel(t *testing.T) {\n\troot, _ := newCpuTreeFromInt5([5]int{2, 2, 2, 4, 2})\n\tnewRoot := root.SplitLevel(CPUTopologyLevelNuma,\n\t\tfunc(cpu int) int {\n\t\t\tleaf := root.FindLeafWithCpu(cpu)\n\t\t\tif leaf == nil {\n\t\t\t\tt.Fatalf(\"cpu %d not in tree:\\n%s\\n\\n\", cpu, root.PrettyPrint())\n\t\t\t}\n\t\t\treturn leaf.SiblingIndex()\n\t\t})\n\n\toldc62 := root.FindLeafWithCpu(62)\n\toldc63 := root.FindLeafWithCpu(63)\n\tif oldc62.parent != oldc63.parent {\n\t\tt.Errorf(\"expected: 62 and 63 are hyperthreads of the same physical core in the original tree, observed parents %s and %s\", oldc62.parent, oldc63.parent)\n\t}\n\tnewc62 := newRoot.FindLeafWithCpu(62)\n\tnewc63 := newRoot.FindLeafWithCpu(63)\n\tif newc62.parent == newc63.parent {\n\t\tt.Errorf(\"expected: 62 and 63 have different parents (physical cores), but they have the same %s\", newc62.parent)\n\t}\n\tif newc62.parent.parent == newc63.parent.parent {\n\t\tt.Errorf(\"expected: 62 and 63 have different grand parents (numa subclasses), but they have the same: %s\", newc62.parent.parent)\n\t}\n\tif newc62.parent.parent.parent != newc63.parent.parent.parent {\n\t\tt.Errorf(\"expected: 62 and 63 have the same great grand parents (numa), but they differ: %s and %s\", newc62.parent.parent.parent, newc63.parent.parent.parent)\n\t}\n\tif t.Failed() {\n\t\tt.Logf(\"newRoot:\\n%s\\n\", newRoot.PrettyPrint())\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/fillmethod.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n)\n\n// FillMethod specifies the order in which balloon instances should be filled.\ntype FillMethod int\n\nconst (\n\tFillUnspecified FillMethod = iota\n\t// FillBalanced: put a container into the balloon with most\n\t// free CPU without changing the size of the balloon.\n\tFillBalanced\n\t// FillBalancedInflate: put a container into the balloon with\n\t// most free CPU when the balloon is inflated to the maximum\n\t// size.\n\tFillBalancedInflate\n\t// FillPacked: put a container into a balloon so that it\n\t// minimizes the amount of currently unused CPUs in the\n\t// balloon.\n\tFillPacked\n\t// FillPackedInflate: put a container into a balloon so that\n\t// it minimizes the amount of unused CPUs if the balloon is\n\t// inflated to the maximum size.\n\tFillPackedInflate\n\t// FillSameNamespace: put a container into a balloon that already\n\t// includes another container from the same namespace\n\tFillSameNamespace\n\t// FillSamePod: put a container into a balloon that already\n\t// includes another container from the same pod.\n\tFillSamePod\n\t// FillNewBalloon: create a new balloon, if possible, and put\n\t// a container into it.\n\tFillNewBalloon\n\t// FillNewBalloonMust: create a new balloon for a container,\n\t// but refuse to run the container if the balloon cannot be\n\t// created.\n\tFillNewBalloonMust\n\t// FillReservedBalloon: put a container into the reserved\n\t// balloon.\n\tFillReservedBalloon\n\t// FillDefaultBalloon: put a container into the default\n\t// balloon.\n\tFillDefaultBalloon\n)\n\nvar fillMethodNames = map[FillMethod]string{\n\tFillUnspecified:     \"unspecified\",\n\tFillBalanced:        \"balanced\",\n\tFillBalancedInflate: \"balanced-inflate\",\n\tFillPacked:          \"packed\",\n\tFillPackedInflate:   \"packed-inflate\",\n\tFillSameNamespace:   \"same-namespace\",\n\tFillSamePod:         \"same-pod\",\n\tFillNewBalloon:      \"new-balloon\",\n\tFillNewBalloonMust:  \"new-balloon-must\",\n\tFillDefaultBalloon:  \"default-balloon\",\n\tFillReservedBalloon: \"reserved-balloon\",\n}\n\n// String stringifies a FillMethod\nfunc (fm FillMethod) String() string {\n\tif fmn, ok := fillMethodNames[fm]; ok {\n\t\treturn fmn\n\t}\n\treturn fmt.Sprintf(\"#UNNAMED-FILLMETHOD(%d)\", int(fm))\n}\n\n// MarshalJSON marshals a FillMethod as a quoted json string\nfunc (fm FillMethod) MarshalJSON() ([]byte, error) {\n\tbuffer := bytes.NewBufferString(fmt.Sprintf(\"%q\", fm))\n\treturn buffer.Bytes(), nil\n}\n\n// UnmarshalJSON unmarshals a FillMethod quoted json string to the enum value\nfunc (fm *FillMethod) UnmarshalJSON(b []byte) error {\n\tvar fillMethodName string\n\terr := json.Unmarshal(b, &fillMethodName)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor fmID, fmName := range fillMethodNames {\n\t\tif fmName == fillMethodName {\n\t\t\t*fm = fmID\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn balloonsError(\"invalid fill method %q\", fillMethodName)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/flags.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"encoding/json\"\n\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n)\n\ntype BalloonsOptions balloonsOptionsWrapped\n\n// BalloonsOptions contains configuration options specific to this policy.\ntype balloonsOptionsWrapped struct {\n\t// PinCPU controls pinning containers to CPUs.\n\tPinCPU *bool `json:\"PinCPU,omitempty\"`\n\t// PinMemory controls pinning containers to memory nodes.\n\tPinMemory *bool `json:\"PinMemory,omitempty\"`\n\t// IdleCpuClass controls how unusded CPUs outside any a\n\t// balloons are (re)configured.\n\tIdleCpuClass string `json:\"IdleCPUClass\",omitempty\"`\n\t// ReservedPoolNamespaces is a list of namespace globs that\n\t// will be allocated to reserved CPUs.\n\tReservedPoolNamespaces []string `json:\"ReservedPoolNamespaces,omitempty\"`\n\t// If AllocatorTopologyBalancing is true, balloons are\n\t// allocated and resized so that all topology elements\n\t// (packages, dies, numa nodes, cores) have roughly same\n\t// amount of allocations. The default is false: balloons are\n\t// packed tightly to optimize power efficiency. The value set\n\t// here can be overridden with the balloon type specific\n\t// setting with the same name.\n\tAllocatorTopologyBalancing bool\n\t// PreferSpreadOnPhysicalCores prefers allocating logical CPUs\n\t// (possibly hyperthreads) for a balloon from separate physical CPU\n\t// cores. This prevents workloads in the balloon from interfering with\n\t// themselves as they do not compete on the resources of the same CPU\n\t// cores. On the other hand, it allows more interference between\n\t// workloads in different balloons. The default is false: balloons\n\t// are packed tightly to a minimum number of physical CPU cores. The\n\t// value set here is the default for all balloon types, but it can be\n\t// overridden with the balloon type specific setting with the same\n\t// name.\n\tPreferSpreadOnPhysicalCores bool `json:\"PreferSpreadOnPhysicalCores,omitempty\"`\n\t// BallonDefs contains balloon type definitions.\n\tBalloonDefs []*BalloonDef `json:\"BalloonTypes,omitempty\"`\n}\n\n// BalloonDef contains a balloon definition.\ntype BalloonDef struct {\n\t// Name of the balloon definition.\n\tName string `json:\"Name\"`\n\t// Namespaces control which namespaces are assigned into\n\t// balloon instances from this definition. This is used by\n\t// namespace assign methods.\n\tNamespaces []string `json:\"Namespaces\",omitempty`\n\t// MaxCpus specifies the maximum number of CPUs exclusively\n\t// usable by containers in a balloon. Balloon size will not be\n\t// inflated larger than MaxCpus.\n\tMaxCpus int `json:\"MaxCPUs\"`\n\t// MinCpus specifies the minimum number of CPUs exclusively\n\t// usable by containers in a balloon. When new balloon is created,\n\t// this will be the number of CPUs reserved for it even if a container\n\t// would request less.\n\tMinCpus int `json:\"MinCPUs\"`\n\t// AllocatorPriority (0: High, 1: Normal, 2: Low, 3: None)\n\t// This parameter is passed to CPU allocator when creating or\n\t// resizing a balloon. At init, balloons with highest priority\n\t// CPUs are allocated first.\n\tAllocatorPriority cpuallocator.CPUPriority `json:\"AllocatorPriority\"`\n\t// PreferSpreadOnPhysicalCores is the balloon type specific\n\t// parameter of the policy level parameter with the same name.\n\tPreferSpreadOnPhysicalCores *bool `json:\"PreferSpreadOnPhysicalCores,omitempty\"`\n\t// AllocatorTopologyBalancing is the balloon type specific\n\t// parameter of the policy level parameter with the same name.\n\tAllocatorTopologyBalancing *bool `json:\"AllocatorTopologyBalancing,omitempty\"`\n\t// CpuClass controls how CPUs of a balloon are (re)configured\n\t// whenever a balloon is created, inflated or deflated.\n\tCpuClass string `json:\"CpuClass\"`\n\t// MinBalloons is the number of balloon instances that always\n\t// exist even if they would become empty. At init this number\n\t// of instances will be created before assigning any\n\t// containers.\n\tMinBalloons int `json:\"MinBalloons\"`\n\t// MaxBalloons is the maximum number of balloon instances that\n\t// is allowed to co-exist. If reached, new balloons cannot be\n\t// created anymore.\n\tMaxBalloons int `json:\"MaxBalloons\"`\n\t// PreferSpreadingPods: containers of the same pod may be\n\t// placed on separate balloons. The default is false: prefer\n\t// placing containers of a pod to the same balloon(s).\n\tPreferSpreadingPods bool\n\t// PreferPerNamespaceBalloon: if true, containers in different\n\t// namespaces are preferrably placed in separate balloons,\n\t// even if the balloon type is the same for all of them. On\n\t// the other hand, containers in the same namespace will be\n\t// placed in the same balloon instances. The default is false:\n\t// namespaces have no effect on placement.\n\tPreferPerNamespaceBalloon bool\n\t// PreferNewBalloons: prefer creating new balloons over adding\n\t// containers to existing balloons. The default is false:\n\t// prefer using filling free capacity and possibly inflating\n\t// existing balloons before creating new ones.\n\tPreferNewBalloons bool\n\t// ShareIdleCpusInSame <topology-level>: if there are idle\n\t// CPUs, that is CPUs not in any balloon, in the same\n\t// <topology-level> as any CPU in the balloon, then allow\n\t// workloads to run on those (shared) CPUs in addition to the\n\t// (dedicated) CPUs of the balloon.\n\tShareIdleCpusInSame CPUTopologyLevel `json:\"ShareIdleCPUsInSame,omitempty\"`\n}\n\nvar defaultPinCPU bool = true\nvar defaultPinMemory bool = true\n\n// DeepCopy creates a deep copy of a BalloonsOptions\nfunc (bo *BalloonsOptions) DeepCopy() *BalloonsOptions {\n\toutBo := *bo\n\toutBo.ReservedPoolNamespaces = make([]string, len(bo.ReservedPoolNamespaces))\n\tcopy(outBo.ReservedPoolNamespaces, bo.ReservedPoolNamespaces)\n\toutBo.BalloonDefs = make([]*BalloonDef, len(bo.BalloonDefs))\n\tfor i := range bo.BalloonDefs {\n\t\toutBo.BalloonDefs[i] = bo.BalloonDefs[i].DeepCopy()\n\t}\n\treturn &outBo\n}\n\n// String stringifies a BalloonDef\nfunc (bdef BalloonDef) String() string {\n\treturn bdef.Name\n}\n\n// DeepCopy creates a deep copy of a BalloonDef\nfunc (bdef *BalloonDef) DeepCopy() *BalloonDef {\n\toutBdef := *bdef\n\toutBdef.Namespaces = make([]string, len(bdef.Namespaces))\n\tcopy(outBdef.Namespaces, bdef.Namespaces)\n\treturn &outBdef\n}\n\n// defaultBalloonsOptions returns a new BalloonsOptions instance, all initialized to defaults.\nfunc defaultBalloonsOptions() interface{} {\n\treturn &BalloonsOptions{\n\t\tReservedPoolNamespaces: []string{metav1.NamespaceSystem},\n\t\tPinCPU:                 &defaultPinCPU,\n\t\tPinMemory:              &defaultPinMemory,\n\t}\n}\n\n// Our runtime configuration.\nvar balloonsOptions = defaultBalloonsOptions().(*BalloonsOptions)\n\n// UnmarshalJSON makes sure all options from previous unmarshals get\n// cleared before unmarshaling new data to the same address.\nfunc (bo *BalloonsOptions) UnmarshalJSON(data []byte) error {\n\tbow := balloonsOptionsWrapped{}\n\tif err := json.Unmarshal(data, &bow); err != nil {\n\t\treturn err\n\t}\n\t*bo = BalloonsOptions(bow)\n\treturn nil\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tpkgcfg.Register(PolicyPath, PolicyDescription, balloonsOptions, defaultBalloonsOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/balloons/metrics.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage balloons\n\nimport (\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\n// Prometheus Metric descriptor indices and descriptor table\nconst (\n\tballoonsDesc = iota\n)\n\nvar descriptors = []*prometheus.Desc{\n\tballoonsDesc: prometheus.NewDesc(\n\t\t\"balloons\",\n\t\t\"CPUs\",\n\t\t[]string{\n\t\t\t\"balloon_type\",\n\t\t\t\"cpu_class\",\n\t\t\t\"cpus_min\",\n\t\t\t\"cpus_max\",\n\t\t\t\"balloon\",\n\t\t\t\"cpus\",\n\t\t\t\"cpus_count\",\n\t\t\t\"numas\",\n\t\t\t\"numas_count\",\n\t\t\t\"dies\",\n\t\t\t\"dies_count\",\n\t\t\t\"packages\",\n\t\t\t\"packages_count\",\n\t\t\t\"sharedidlecpus\",\n\t\t\t\"sharedidlecpus_count\",\n\t\t\t\"cpus_allowed\",\n\t\t\t\"cpus_allowed_count\",\n\t\t\t\"mems\",\n\t\t\t\"containers\",\n\t\t\t\"tot_req_millicpu\",\n\t\t}, nil,\n\t),\n}\n\n// Metrics defines the balloons-specific metrics from policy level.\ntype Metrics struct {\n\tBalloons []*BalloonMetrics\n}\n\n// BalloonMetrics define metrics of a balloon instance.\ntype BalloonMetrics struct {\n\t// Balloon type metrics\n\tDefName  string\n\tCpuClass string\n\tMinCpus  int\n\tMaxCpus  int\n\t// Balloon instance metrics\n\tPrettyName            string\n\tCpus                  cpuset.CPUSet\n\tCpusCount             int\n\tNumas                 []string\n\tNumasCount            int\n\tDies                  []string\n\tDiesCount             int\n\tPackages              []string\n\tPackagesCount         int\n\tSharedIdleCpus        cpuset.CPUSet\n\tSharedIdleCpusCount   int\n\tCpusAllowed           cpuset.CPUSet\n\tCpusAllowedCount      int\n\tMems                  string\n\tContainerNames        string\n\tContainerReqMilliCpus int\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data\n// descriptors.\nfunc (p *balloons) DescribeMetrics() []*prometheus.Desc {\n\treturn descriptors\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *balloons) PollMetrics() policy.Metrics {\n\tpolicyMetrics := &Metrics{}\n\tpolicyMetrics.Balloons = make([]*BalloonMetrics, len(p.balloons))\n\tfor index, bln := range p.balloons {\n\t\tcpuLoc := p.cpuTree.CpuLocations(bln.Cpus)\n\t\tbm := &BalloonMetrics{}\n\t\tpolicyMetrics.Balloons[index] = bm\n\t\tbm.DefName = bln.Def.Name\n\t\tbm.CpuClass = bln.Def.CpuClass\n\t\tbm.MinCpus = bln.Def.MinCpus\n\t\tbm.MaxCpus = bln.Def.MaxCpus\n\t\tbm.PrettyName = bln.PrettyName()\n\t\tbm.Cpus = bln.Cpus\n\t\tbm.CpusCount = bm.Cpus.Size()\n\t\tif len(cpuLoc) > 3 {\n\t\t\tbm.Numas = cpuLoc[3]\n\t\t\tbm.NumasCount = len(bm.Numas)\n\t\t\tbm.Dies = cpuLoc[2]\n\t\t\tbm.DiesCount = len(bm.Dies)\n\t\t\tbm.Packages = cpuLoc[1]\n\t\t\tbm.PackagesCount = len(bm.Packages)\n\t\t}\n\t\tbm.SharedIdleCpus = bln.SharedIdleCpus\n\t\tbm.SharedIdleCpusCount = bm.SharedIdleCpus.Size()\n\t\tbm.CpusAllowed = bm.Cpus.Union(bm.SharedIdleCpus)\n\t\tbm.CpusAllowedCount = bm.CpusAllowed.Size()\n\t\tbm.Mems = bln.Mems.String()\n\t\tcNames := []string{}\n\t\t// Get container names and total requested milliCPUs.\n\t\tfor _, containerIDs := range bln.PodIDs {\n\t\t\tfor _, containerID := range containerIDs {\n\t\t\t\tif c, ok := p.cch.LookupContainer(containerID); ok {\n\t\t\t\t\tcNames = append(cNames, c.PrettyName())\n\t\t\t\t\tbm.ContainerReqMilliCpus += p.containerRequestedMilliCpus(containerID)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tsort.Strings(cNames)\n\t\tbm.ContainerNames = strings.Join(cNames, \",\")\n\t}\n\n\treturn policyMetrics\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled\n// policy-specific metrics data.\nfunc (p *balloons) CollectMetrics(m policy.Metrics) ([]prometheus.Metric, error) {\n\tmetrics, ok := m.(*Metrics)\n\tif !ok {\n\t\treturn nil, balloonsError(\"type mismatch in balloons metrics\")\n\t}\n\tpromMetrics := make([]prometheus.Metric, len(metrics.Balloons))\n\tfor index, bm := range metrics.Balloons {\n\t\tpromMetrics[index] = prometheus.MustNewConstMetric(\n\t\t\tdescriptors[balloonsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(bm.Cpus.Size()),\n\t\t\tbm.DefName,\n\t\t\tbm.CpuClass,\n\t\t\tstrconv.Itoa(bm.MinCpus),\n\t\t\tstrconv.Itoa(bm.MaxCpus),\n\t\t\tbm.PrettyName,\n\t\t\tbm.Cpus.String(),\n\t\t\tstrconv.Itoa(bm.CpusCount),\n\t\t\tstrings.Join(bm.Numas, \",\"),\n\t\t\tstrconv.Itoa(bm.NumasCount),\n\t\t\tstrings.Join(bm.Dies, \",\"),\n\t\t\tstrconv.Itoa(bm.DiesCount),\n\t\t\tstrings.Join(bm.Packages, \",\"),\n\t\t\tstrconv.Itoa(bm.PackagesCount),\n\t\t\tbm.SharedIdleCpus.String(),\n\t\t\tstrconv.Itoa(bm.SharedIdleCpusCount),\n\t\t\tbm.CpusAllowed.String(),\n\t\t\tstrconv.Itoa(bm.CpusAllowedCount),\n\t\t\tbm.Mems,\n\t\t\tbm.ContainerNames,\n\t\t\tstrconv.Itoa(bm.ContainerReqMilliCpus))\n\t}\n\treturn promMetrics, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/dynamic-pools/cpu.go",
    "content": "package dyp\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n)\n\ntype cpuTimesStat struct {\n\tcpu       string  `json:\"cpu\"`\n\tuser      float64 `json:\"user\"`\n\tsystem    float64 `json:\"system\"`\n\tidle      float64 `json:\"idle\"`\n\tnice      float64 `json:\"nice\"`\n\tioWait    float64 `json:\"iowait\"`\n\tirq       float64 `json:\"irq\"`\n\tsoftirq   float64 `json:\"softirq\"`\n\tsteal     float64 `json:\"steal\"`\n\tguest     float64 `json:\"guest\"`\n\tguestNice float64 `json:\"guestNice\"`\n}\n\n// getCpuUtilization returns the utilization of each cpu in an interval\nfunc getCpuUtilization(interval time.Duration) ([]float64, error) {\n\tctx := context.Background()\n\tcpuTimesStat1, err := getCpuTimesStat(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := wait(ctx, interval); err != nil {\n\t\treturn nil, err\n\t}\n\tcpuTimesStat2, err := getCpuTimesStat(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn calculateAllCpusUtilization(cpuTimesStat1, cpuTimesStat2)\n}\n\nfunc getCpuTimesStat(ctx context.Context) ([]cpuTimesStat, error) {\n\tfilename := filepath.Join(\"/\", sysfs.SysRoot(), \"proc\", \"stat\")\n\tlines := []string{}\n\tcpuLines, err := readCpuLines(filename)\n\tif err != nil || len(cpuLines) == 0 {\n\t\treturn []cpuTimesStat{}, err\n\t}\n\n\tstat := make([]cpuTimesStat, 0, len(lines))\n\tfor _, l := range cpuLines {\n\t\toneStat, err := parseStatLine(l)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tstat = append(stat, *oneStat)\n\t}\n\treturn stat, nil\n}\n\nfunc wait(ctx context.Context, interval time.Duration) error {\n\ttimer := time.NewTimer(interval)\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tcase <-timer.C:\n\t\treturn nil\n\t}\n}\n\nfunc calculateAllCpusUtilization(cts1, cts2 []cpuTimesStat) ([]float64, error) {\n\tif len(cts1) != len(cts2) {\n\t\treturn nil, dynamicPoolsError(\"received two CPU counts: %d != %d\", len(cts1), len(cts2))\n\t}\n\tallCpusUtilization := make([]float64, len(cts1))\n\tfor i := 0; i < len(cts1); i++ {\n\t\tallCpusUtilization[i] = calculateOneCpuUtilization(cts1[i], cts2[i])\n\t}\n\treturn allCpusUtilization, nil\n}\n\n// readCpuLines skips the first line indicating the total CPU utilization.\nfunc readCpuLines(filename string) ([]string, error) {\n\tf, err := os.Open(filename)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer f.Close()\n\tvar statLines []string\n\treader := bufio.NewReader(f)\n\tfor {\n\t\tline, _, err := reader.ReadLine()\n\t\tif err == io.EOF {\n\t\t\tbreak\n\t\t}\n\t\tstatLines = append(statLines, string(line))\n\t}\n\n\tvar cpuLines []string\n\tif len(statLines) < 2 {\n\t\treturn nil, nil\n\t}\n\tfor _, line := range statLines[1:] {\n\t\tif !strings.HasPrefix(line, \"cpu\") {\n\t\t\tbreak\n\t\t}\n\t\tcpuLines = append(cpuLines, line)\n\t}\n\treturn cpuLines, nil\n}\n\n// parseStatLine is to parse cpuLine into cpuTimesStat.\nfunc parseStatLine(cpuLine string) (*cpuTimesStat, error) {\n\tvalues := strings.Fields(cpuLine)\n\tif len(values) == 0 || len(values) < 8 {\n\t\treturn nil, dynamicPoolsError(\"Stat does not contain cpu info.\")\n\t}\n\tcpu := values[0]\n\tuser, err := strconv.ParseFloat(values[1], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnice, err := strconv.ParseFloat(values[2], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsystem, err := strconv.ParseFloat(values[3], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tidle, err := strconv.ParseFloat(values[4], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tioWait, err := strconv.ParseFloat(values[5], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tirq, err := strconv.ParseFloat(values[6], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsoftirq, err := strconv.ParseFloat(values[7], 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcts := &cpuTimesStat{\n\t\tcpu:     cpu,\n\t\tuser:    user,\n\t\tnice:    nice,\n\t\tsystem:  system,\n\t\tidle:    idle,\n\t\tioWait:  ioWait,\n\t\tirq:     irq,\n\t\tsoftirq: softirq,\n\t}\n\tif len(values) > 8 { // Linux >= 2.6.11\n\t\tsteal, err := strconv.ParseFloat(values[8], 64)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tcts.steal = steal\n\t}\n\tif len(values) > 9 { // Linux >= 2.6.24\n\t\tguest, err := strconv.ParseFloat(values[9], 64)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tcts.guest = guest\n\t}\n\tif len(values) > 10 { // Linux >= 3.2.0\n\t\tguestNice, err := strconv.ParseFloat(values[10], 64)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tcts.guestNice = guestNice\n\t}\n\treturn cts, nil\n}\n\n// calculateOneCpuUtilization returns the utilization of one cpu in an interval\nfunc calculateOneCpuUtilization(cts1, cts2 cpuTimesStat) float64 {\n\tcts1Total, cts1Busy := getBusyTime(cts1)\n\tcts2Total, cts2Busy := getBusyTime(cts2)\n\tif cts2Busy <= cts1Busy {\n\t\treturn 0\n\t}\n\tif cts2Total <= cts1Total {\n\t\treturn 100\n\t}\n\treturn math.Min(100, math.Max(0, (cts2Busy-cts1Busy)/(cts2Total-cts1Total)*100))\n}\n\nfunc getBusyTime(cts cpuTimesStat) (float64, float64) {\n\ttotal := cts.user + cts.system + cts.idle + cts.nice + cts.ioWait + cts.irq +\n\t\tcts.softirq + cts.steal + cts.guest + cts.guestNice\n\tif runtime.GOOS == \"linux\" {\n\t\ttotal -= cts.guest     // Linux 2.6.24+\n\t\ttotal -= cts.guestNice // Linux 3.2.0+\n\t}\n\tbusy := total - cts.idle - cts.ioWait\n\treturn total, busy\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/dynamic-pools/dyp.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dyp\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"time\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tcpucontrol \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/cpu\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tpolicyapi \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy.\n\tPolicyName = \"dynamic-pools\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"The cpuset of the dynamic pools can be dynamically changed based on workload.\"\n\t// PolicyPath is the path of this policy in the configuration hierarchy.\n\tPolicyPath = \"policy.\" + PolicyName\n\t// dynamicPoolKey is a pod annotation key, the value is a pod dynamicPool name.\n\tdynamicPoolKey = \"dynamic-pool.\" + PolicyName + \".\" + kubernetes.ResmgrKeyNamespace\n\t// reservedDynamicPoolDefName is the name in the reserved dynamicPool definition.\n\treservedDynamicPoolDefName = \"reserved\"\n\t// sharedDynamicPoolDefName is the name in the shared dynamicPool definition.\n\tsharedDynamicPoolDefName = \"shared\"\n)\n\n// dynamicPools contains configuration and runtime attributes of the dynamic-pools policy\ntype dynamicPools struct {\n\toptions   *policyapi.BackendOptions // configuration common to all policies\n\tdpoptions DynamicPoolsOptions       // dynamicPool-specific configuration\n\tcch       cache.Cache               // cri-resmgr cache\n\tallowed   cpuset.CPUSet             // bounding set of CPUs we're allowed to use\n\treserved  cpuset.CPUSet             // system-/kube-reserved CPUs\n\tfreeCpus  cpuset.CPUSet             // CPUs to be included in growing dynamicPools\n\n\treservedDynamicPoolDef *DynamicPoolDef // built-in definition of the reserved dynamicPool\n\tsharedDynamicPoolDef   *DynamicPoolDef // built-in definition of the shared dynamicPool\n\tdynamicPools           []*DynamicPool  // dynamicPool instances: reserved, shared and user-defined\n\n\tcpuAllocator cpuallocator.CPUAllocator // CPU allocator used by the policy\n}\n\n// DynamicPool contains attributes of a dynamicPool\ntype DynamicPool struct {\n\t// Def is the definition from which this dynamicPool is created.\n\tDef *DynamicPoolDef\n\t// Cpus is the set of CPUs exclusive to this dynamicPool only.\n\tCpus cpuset.CPUSet\n\t// Mems is the set of memory nodes with minimal access delay from CPUs.\n\tMems idset.IDSet\n\t// PodIDs maps pod ID to list of container IDs.\n\t// - len(PodIDs) is the number of pods in the dynamicPool.\n\t// - len(PodIDs[podID]) is the number of containers of podID currently assigned to the dynamicPool.\n\tPodIDs map[string][]string\n}\n\nvar log logger.Logger = logger.NewLogger(\"policy\")\n\n// String is a stringer for a dynamicPool.\nfunc (dp DynamicPool) String() string {\n\treturn fmt.Sprintf(\"%s{Cpus:%s, Mems:%s}\", dp.PrettyName(), dp.Cpus, dp.Mems)\n}\n\n// PrettyName returns a unique name for a dynamicPool.\nfunc (dp DynamicPool) PrettyName() string {\n\treturn dp.Def.Name\n}\n\n// ContainerIDs returns IDs of containers assigned in a dynamicPool.\n// (Using cache.Container.GetCacheID()'s)\nfunc (dp DynamicPool) ContainerIDs() []string {\n\tcIDs := []string{}\n\tfor _, ctrIDs := range dp.PodIDs {\n\t\tcIDs = append(cIDs, ctrIDs...)\n\t}\n\treturn cIDs\n}\n\n// ContainerCount returns the number of containers in a dynamicPool.\nfunc (dp DynamicPool) ContainerCount() int {\n\tcount := 0\n\tfor _, ctrIDs := range dp.PodIDs {\n\t\tcount += len(ctrIDs)\n\t}\n\treturn count\n}\n\n// AvailMilliCpus returns the number of CPUs in a dynamicPool.\nfunc (dp DynamicPool) AvailMilliCpus() int {\n\treturn dp.Cpus.Size() * 1000\n}\n\n// updateRealCpuUsed returns cpu utilization of a dynamicPool.\nfunc (dp *DynamicPool) updateRealCpuUsed(cpuInfo []float64) (float64, error) {\n\tif dp.Cpus.Size() == 0 {\n\t\tlog.Debug(\"dynamic pool %s cpuset is 0\", dp.Def.Name)\n\t\treturn 0, nil\n\t}\n\tcpus := dp.Cpus.UnsortedList()\n\tvar sum float64\n\tfor i := 0; i < len(cpus); i++ {\n\t\tsum += cpuInfo[cpus[i]]\n\t}\n\tlog.Debug(\"dynamic pool %s cpuset: %s,  cpu utilization: %v\", dp.Def.Name, dp.Cpus, sum)\n\treturn sum, nil\n}\n\n// calculateAllPoolWeights returns weights of all dynamicPools and the sum of weights.\n// Use dynamicPool's cpu utilization as its weight.\nfunc (p *dynamicPools) calculateAllPoolWeights() (map[*DynamicPool]float64, float64, error) {\n\tcpuInfo, _ := getCpuUtilization(time.Duration(time.Second))\n\tweight := make(map[*DynamicPool]float64)\n\tsumWeight := 0.0\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\t// If there is no container in a dynamic pool, there is no need to calculate its weight, that is, there is no need to allocate CPUs to it.\n\t\tif dp.ContainerCount() == 0 {\n\t\t\tweight[dp] = 0.0\n\t\t} else {\n\t\t\trealCpuUsed, err := dp.updateRealCpuUsed(cpuInfo)\n\t\t\tif err != nil {\n\t\t\t\treturn weight, sumWeight, dynamicPoolsError(\"The actual cpu usage of the dynamic pool %s cannot be obtained: %w\",\n\t\t\t\t\tdp.PrettyName(), err)\n\t\t\t}\n\t\t\tweight[dp] = realCpuUsed\n\t\t\tsumWeight += weight[dp]\n\t\t}\n\t\tlog.Debug(\"dynamic pool: %s, weight: %v\", dp, weight[dp])\n\t}\n\tlog.Debug(\"sum weight: %v\", sumWeight)\n\treturn weight, sumWeight, nil\n}\n\n// calculateAllPoolRequests returns the sum of the requests of containers in each dynamicPool and remaining free cpu.\n// remainFree = allowed cpu - reserved cpu - sum(requests of containers in each dynamicPool)\nfunc (p *dynamicPools) calculateAllPoolRequests() (map[*DynamicPool]int, int) {\n\trequestCpu := make(map[*DynamicPool]int)\n\tremainFree := p.allowed.Difference(p.reserved).Size()\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\trequestCpu[dp] = (p.requestedMinMilliCpus(dp) + 999) / 1000\n\t\tremainFree -= requestCpu[dp]\n\t\tlog.Debug(\"dynamic pool %s request cpu %d\", dp, requestCpu[dp])\n\t}\n\tlog.Debug(\"sum remain free cpu %d\", remainFree)\n\treturn requestCpu, remainFree\n}\n\nfunc (p *dynamicPools) containerPinPool(dp *DynamicPool) {\n\tdp.Mems = p.closestMems(dp.Cpus)\n\tfor _, cID := range dp.ContainerIDs() {\n\t\tif c, ok := p.cch.LookupContainer(cID); ok {\n\t\t\tp.pinCpuMem(c, dp.Cpus, dp.Mems)\n\t\t}\n\t}\n}\n\n// calculatePoolCpuset returns the cpus that dynamic pools need to allocate.\nfunc (p *dynamicPools) calculatePoolCpuset(requestCpu map[*DynamicPool]int, remainFree int, weight map[*DynamicPool]float64, sumWeight float64) map[*DynamicPool]int {\n\tusedCpu := 0\n\t// If there are containers in the shared dynamic pool, allocate at least one CPU to it,\n\t// otherwise there is no need to allocate a CPU to it.\n\t// Ensure that there is at least one cpu in the shared dynamicPool.\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == sharedDynamicPoolDefName && dp.ContainerCount() > 0 && sumWeight != 0 {\n\t\t\taddCpu := int(float64(remainFree) * weight[dp] / sumWeight)\n\t\t\tif requestCpu[dp]+addCpu < 1 {\n\t\t\t\trequestCpu[dp] = 1\n\t\t\t\tremainFree -= 1\n\t\t\t}\n\t\t}\n\t}\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\trequestCpu[dp] = dp.Cpus.Size()\n\t\t}\n\t\tif sumWeight != 0 {\n\t\t\taddCpu := int(float64(remainFree) * weight[dp] / sumWeight)\n\t\t\trequestCpu[dp] += addCpu\n\t\t\tusedCpu += addCpu\n\t\t}\n\t\tlog.Info(\"The cpu that dynamic pool %s needs to allocate is %d, remain free cpu %d\", dp, requestCpu[dp], remainFree-usedCpu)\n\t}\n\tif usedCpu < remainFree {\n\t\t// If there is still cpus, give the dynamicPool with the highest cpu utilization.\n\t\ttmp := p.dynamicPools[1]\n\t\tfor _, dp := range p.dynamicPools {\n\t\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif weight[dp] > weight[tmp] {\n\t\t\t\ttmp = dp\n\t\t\t}\n\t\t}\n\t\trequestCpu[tmp] += (remainFree - usedCpu)\n\t\tlog.Info(\"The cpu that dynamic pool %s needs to allocate is %d, remain free cpu %d\", tmp, requestCpu[tmp], 0)\n\t}\n\treturn requestCpu\n}\n\n// isNeedReallocate returns whether the cpus need to be reallocated.\nfunc (p *dynamicPools) isNeedReallocate(newPoolCpu map[*DynamicPool]int) bool {\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif dp.Cpus.Size() != newPoolCpu[dp] {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// updatePoolCpuset updates the cpuset of the dynamicPools.\nfunc (p *dynamicPools) updatePoolCpuset() error {\n\trequestCpu, remainFree := p.calculateAllPoolRequests()\n\tweight, sumWeight, err := p.calculateAllPoolWeights()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif remainFree >= 1 {\n\t\trequestCpu = p.calculatePoolCpuset(requestCpu, remainFree, weight, sumWeight)\n\t}\n\n\t// If the number of newly allocated CPUs is the same as the number of existing CPUs in the pool,\n\t// it means that there is no need to re-allocate\n\tif !p.isNeedReallocate(requestCpu) {\n\t\tlog.Info(\"The number of CPUs required by the pools is the same as the number of CPUs already in the pools, so there is no need to reallocate.\")\n\t\tfor _, dp := range p.dynamicPools {\n\t\t\tp.containerPinPool(dp)\n\t\t}\n\t\treturn nil\n\t}\n\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif dp.Cpus.Size() == 0 {\n\t\t\tcontinue\n\t\t}\n\t\toldCpus := dp.Cpus.Clone()\n\t\tkeptCpus, err := p.cpuAllocator.ReleaseCpus(&oldCpus, dp.Cpus.Size(), dp.Def.AllocatorPriority)\n\t\tif err != nil || keptCpus.Size() != 0 {\n\t\t\treturn dynamicPoolsError(\"releasing %d CPUs from %s failed: %w (kept: %s)\", dp.Cpus.Size(), dp, err, keptCpus)\n\t\t}\n\t\tp.freeCpus = p.freeCpus.Union(dp.Cpus)\n\t}\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def.Name == reservedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tnewCpus, err := p.cpuAllocator.AllocateCpus(&p.freeCpus, requestCpu[dp], dp.Def.AllocatorPriority)\n\t\tif err != nil {\n\t\t\treturn dynamicPoolsError(\"allocating %d CPUs for %s failed: %w\", requestCpu[dp], dp, err)\n\t\t}\n\t\tdp.Cpus = newCpus\n\t\tlog.Debugf(\"resize successful for container: %s, new Cpus: %#s\", dp.PrettyName(), dp.Cpus)\n\t\tp.containerPinPool(dp)\n\t\tp.useCpuClass(dp)\n\t}\n\treturn nil\n}\n\n// CreateDynamicPoolsPolicy creates a new policy instance.\nfunc CreateDynamicPoolsPolicy(policyOptions *policy.BackendOptions) policy.Backend {\n\tp := &dynamicPools{\n\t\toptions:      policyOptions,\n\t\tcch:          policyOptions.Cache,\n\t\tcpuAllocator: cpuallocator.NewCPUAllocator(policyOptions.System),\n\t}\n\tlog.Info(\"creating %s policy...\", PolicyName)\n\t// Handle common policy options: AvailableResources and ReservedResources.\n\t// p.allowed: CPUs available for the policy\n\tif allowed, ok := policyOptions.Available[policyapi.DomainCPU]; ok {\n\t\tp.allowed = allowed.(cpuset.CPUSet)\n\t} else {\n\t\t// Available CPUs not specified, default to all on-line CPUs.\n\t\tp.allowed = policyOptions.System.CPUSet().Difference(policyOptions.System.Offlined())\n\t}\n\t// p.reserved: CPUs reserved for kube-system pods, subset of p.allowed.\n\tp.reserved = cpuset.New()\n\tif reserved, ok := p.options.Reserved[policyapi.DomainCPU]; ok {\n\t\tswitch v := reserved.(type) {\n\t\tcase cpuset.CPUSet:\n\t\t\tp.reserved = p.allowed.Intersection(v)\n\t\tcase resapi.Quantity:\n\t\t\treserveCnt := (int(v.MilliValue()) + 999) / 1000\n\t\t\tcpus, err := p.cpuAllocator.AllocateCpus(&p.allowed, reserveCnt, cpuallocator.PriorityNone)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(\"failed to allocate reserved CPUs: %s\", err)\n\t\t\t}\n\t\t\tp.reserved = cpus\n\t\t\tp.allowed = p.allowed.Union(cpus)\n\t\t}\n\t}\n\tif p.reserved.IsEmpty() {\n\t\tlog.Fatal(\"%s cannot run without reserved CPUs that are also AvailableResources\", PolicyName)\n\t}\n\t// Handle policy-specific options\n\tlog.Debug(\"creating %s configuration\", PolicyName)\n\tif err := p.setConfig(dynamicPoolsOptions); err != nil {\n\t\tlog.Fatal(\"failed to create %s policy: %v\", PolicyName, err)\n\t}\n\tpkgcfg.GetModule(PolicyPath).AddNotify(p.configNotify)\n\n\treturn p\n}\n\n// Name returns the name of this policy.\nfunc (p *dynamicPools) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (p *dynamicPools) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (p *dynamicPools) Start(add []cache.Container, del []cache.Container) error {\n\tlog.Info(\"%s policy started\", PolicyName)\n\treturn p.Sync(p.cch.GetContainers(), nil)\n}\n\n// Sync synchronizes the active policy state.\nfunc (p *dynamicPools) Sync(add []cache.Container, del []cache.Container) error {\n\tlog.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\tp.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\tp.AllocateResources(c)\n\t}\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (p *dynamicPools) AllocateResources(c cache.Container) error {\n\tlog.Debug(\"allocating resources for container %s...\", c.PrettyName())\n\tdp, err := p.allocateDynamicPool(c)\n\tif err != nil {\n\t\treturn dynamicPoolsError(\"dynamicPool allocation for container %s failed: %w\", c.PrettyName(), err)\n\t}\n\tif dp == nil {\n\t\treturn dynamicPoolsError(\"no suitable dynamicPools found for container %s\", c.PrettyName())\n\t}\n\tlog.Info(\"assigning container %s to dynamicPool %s\", c.PrettyName(), dp)\n\tpodID := c.GetPodID()\n\tdp.PodIDs[podID] = append(dp.PodIDs[podID], c.GetCacheID())\n\tif dp.Cpus.Equals(p.reserved) {\n\t\tp.assignContainer(c, dp)\n\t\tlog.Debugf(\"if dynamic pool is reserved, do not updatePoolCpuset.\")\n\t} else {\n\t\tp.updatePoolCpuset()\n\t}\n\tif log.DebugEnabled() {\n\t\tlog.Debug(p.dumpDynamicPool(dp))\n\t}\n\treturn nil\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (p *dynamicPools) ReleaseResources(c cache.Container) error {\n\tlog.Debug(\"releasing container %s...\", c.PrettyName())\n\tdp := p.dynamicPoolByContainer(c)\n\tif dp == nil {\n\t\tlog.Debug(\"ReleaseResources: dynamicPool-less container %s, nothing to release\", c.PrettyName())\n\t\treturn nil\n\t}\n\tp.dismissContainer(c, dp)\n\tif dp.Cpus.Equals(p.reserved) {\n\t\tlog.Debugf(\"if dynamic pool is reserved, do not updatePoolCpuset.\")\n\t} else {\n\t\tp.updatePoolCpuset()\n\t}\n\tif log.DebugEnabled() {\n\t\tlog.Debug(p.dumpDynamicPool(dp))\n\t}\n\treturn nil\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (p *dynamicPools) UpdateResources(c cache.Container) error {\n\tlog.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (p *dynamicPools) Rebalance() (bool, error) {\n\tlog.Debug(\"rebalancing containers...\")\n\terr := p.updatePoolCpuset()\n\treturn true, err\n}\n\n// HandleEvent handles policy-specific events.\nfunc (p *dynamicPools) HandleEvent(*events.Policy) (bool, error) {\n\tlog.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (p *dynamicPools) ExportResourceData(c cache.Container) map[string]string {\n\treturn nil\n}\n\n// Introspect provides data for external introspection.\nfunc (p *dynamicPools) Introspect(*introspect.State) {\n\treturn\n}\n\n// dynamicPoolByContainer returns a dynamicPool that contains a container.\nfunc (p *dynamicPools) dynamicPoolByContainer(c cache.Container) *DynamicPool {\n\tpodID := c.GetPodID()\n\tcID := c.GetCacheID()\n\tfor _, dp := range p.dynamicPools {\n\t\tfor _, ctrID := range dp.PodIDs[podID] {\n\t\t\tif ctrID == cID {\n\t\t\t\treturn dp\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// dynamicPoolsByDef returns a dynamicPool instantiated from a dynamicPool definition.\nfunc (p *dynamicPools) dynamicPoolByDef(dpDef *DynamicPoolDef) *DynamicPool {\n\tfor _, dp := range p.dynamicPools {\n\t\tif dp.Def == dpDef {\n\t\t\treturn dp\n\t\t}\n\t}\n\treturn nil\n}\n\n// dynamicPoolDefByName returns a dynamicPool definition with a name.\nfunc (p *dynamicPools) dynamicPoolDefByName(defName string) *DynamicPoolDef {\n\tif defName == reservedDynamicPoolDefName {\n\t\treturn p.reservedDynamicPoolDef\n\t}\n\tif defName == sharedDynamicPoolDefName {\n\t\treturn p.sharedDynamicPoolDef\n\t}\n\tfor _, dpDef := range p.dpoptions.DynamicPoolDefs {\n\t\tif dpDef.Name == defName {\n\t\t\treturn dpDef\n\t\t}\n\t}\n\treturn nil\n}\n\n// chooseDynamicPoolDef returns the dynamicPoolDef selected by the container\nfunc (p *dynamicPools) chooseDynamicPoolDef(c cache.Container) (*DynamicPoolDef, error) {\n\tvar dpDef *DynamicPoolDef\n\t// If the requests and limits of container are 0, they are assigned to the shared dynamicPool.\n\tif !namespaceMatches(c.GetNamespace(), append(p.dpoptions.ReservedPoolNamespaces, metav1.NamespaceSystem)) &&\n\t\tp.containerRequestedMilliCpus(c.GetCacheID()) == 0 && p.containerLimitedMilliCpus(c.GetCacheID()) == 0 {\n\t\treturn p.sharedDynamicPoolDef, nil\n\t}\n\n\t// DynamicPoolDef is defined by annotation?\n\tif dpDefName, ok := c.GetEffectiveAnnotation(dynamicPoolKey); ok {\n\t\tdpDef = p.dynamicPoolDefByName(dpDefName)\n\t\tif dpDef == nil {\n\t\t\treturn nil, dynamicPoolsError(\"no dynamicPool for annotation %q\", dpDefName)\n\t\t}\n\t\treturn dpDef, nil\n\t}\n\n\t// DynamicPoolDef is defined by a special namespace (kube-system +\n\t// ReservedPoolNamespaces)?\n\tif namespaceMatches(c.GetNamespace(), append(p.dpoptions.ReservedPoolNamespaces, metav1.NamespaceSystem)) {\n\t\treturn p.dynamicPools[0].Def, nil\n\t}\n\n\t// DynamicPoolDef is defined by the namespace?\n\tfor _, dpDef := range append([]*DynamicPoolDef{p.reservedDynamicPoolDef, p.sharedDynamicPoolDef},\n\t\tp.dpoptions.DynamicPoolDefs...) {\n\t\tif namespaceMatches(c.GetNamespace(), dpDef.Namespaces) {\n\t\t\treturn dpDef, nil\n\t\t}\n\t}\n\t// Fallback to the shared dynamicPool.\n\treturn p.sharedDynamicPoolDef, nil\n}\n\nfunc (p *dynamicPools) containerRequestedMilliCpus(contID string) int {\n\tcont, ok := p.cch.LookupContainer(contID)\n\tif !ok {\n\t\treturn 0\n\t}\n\treqCpu, ok := cont.GetResourceRequirements().Requests[corev1.ResourceCPU]\n\tif !ok {\n\t\treturn 0\n\t}\n\treturn int(reqCpu.MilliValue())\n}\n\nfunc (p *dynamicPools) containerLimitedMilliCpus(contID string) int {\n\tcont, ok := p.cch.LookupContainer(contID)\n\tif !ok {\n\t\treturn 0\n\t}\n\tlimitCpu, ok := cont.GetResourceRequirements().Limits[corev1.ResourceCPU]\n\tif !ok {\n\t\treturn 0\n\t}\n\treturn int(limitCpu.MilliValue())\n}\n\n// requestedMaxMilliCpus sums up and returns CPU limits of all\n// containers assigned to a dynamicPool.\nfunc (p *dynamicPools) requestedMaxMilliCpus(dp *DynamicPool) int {\n\tcpuRequested := 0\n\tfor _, cID := range dp.ContainerIDs() {\n\t\tcpuRequested += p.containerLimitedMilliCpus(cID)\n\t}\n\treturn cpuRequested\n}\n\n// requestedMinMilliCpus sums up and returns CPU requests of all\n// containers assigned to a dynamicPool.\nfunc (p *dynamicPools) requestedMinMilliCpus(dp *DynamicPool) int {\n\tcpuRequested := 0\n\tfor _, cID := range dp.ContainerIDs() {\n\t\tcpuRequested += p.containerRequestedMilliCpus(cID)\n\t}\n\treturn cpuRequested\n}\n\n// useCpuClass configures CPUs of a dynamicPool.\nfunc (p *dynamicPools) useCpuClass(dp *DynamicPool) error {\n\t// Usual inputs:\n\t// - CPUs that cpuallocator has reserved for this dynamicPool:\n\t//   dp.Cpus (cpuset.CPUSet).\n\t// - User-defined CPU configuration for CPUs of dynamicPool of this type:\n\t//   dp.Def.CpuClass (string).\n\t// - Current configuration(?): feel free to add data\n\t//   structure for this. For instance policy-global p.cpuConfs,\n\t//   or dynamicPool-local dp.cpuConfs.\n\t//\n\t// Other input examples, if needed:\n\t// - Requested CPU resources by all containers in the dynamicPool:\n\t//   p.requestedMilliCpus(dp).\n\t// - Free CPU resources in the dynamicPool: p.freeMilliCpus(dp).\n\t// - Number of assigned containers: dp.ContainerCount().\n\t// - Container details: access p.cch with dp.ContainerIDs().\n\t// - User-defined CPU AllocatorPriority: dp.Def.AllocatorPriority.\n\t// - All existing dynamicPool instances: p.dynamicPools.\n\t// - CPU configurations by user: dp.Def.CpuClass (for dp in p.dynamicPools)\n\tcpucontrol.Assign(p.cch, dp.Def.CpuClass, dp.Cpus.UnsortedList()...)\n\tlog.Debugf(\"useCpuClass Cpus: %s; CpuClass: %s\", dp.Cpus, dp.Def.CpuClass)\n\treturn nil\n}\n\nfunc (p *dynamicPools) newDynamicPool(dpDef *DynamicPoolDef, confCpus bool) (*DynamicPool, error) {\n\tvar cpus cpuset.CPUSet\n\tvar err error\n\tif dpDef == p.reservedDynamicPoolDef {\n\t\tcpus = p.reserved\n\t} else {\n\t\tcpus, err = p.cpuAllocator.AllocateCpus(&p.freeCpus, 0, dpDef.AllocatorPriority)\n\n\t\tif err != nil {\n\t\t\treturn nil, dynamicPoolsError(\"could not allocate Cpus for dynamicPool %s: %w\", dpDef.Name, err)\n\t\t}\n\t}\n\tdp := &DynamicPool{\n\t\tDef:    dpDef,\n\t\tPodIDs: make(map[string][]string),\n\t\tCpus:   cpus,\n\t\tMems:   p.closestMems(cpus),\n\t}\n\tif confCpus {\n\t\tif err = p.useCpuClass(dp); err != nil {\n\t\t\tlog.Errorf(\"failed to apply CPU configuration to new dynamicPool %s (cpus: %s): %w\", dpDef.Name, cpus, err)\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn dp, nil\n}\n\nfunc namespaceMatches(namespace string, patterns []string) bool {\n\tfor _, pattern := range patterns {\n\t\tret, err := filepath.Match(pattern, namespace)\n\t\tif err == nil && ret {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// allocateDynamicPool returns a dynamicPool allocated for a container.\nfunc (p *dynamicPools) allocateDynamicPool(c cache.Container) (*DynamicPool, error) {\n\tdpDef, err := p.chooseDynamicPoolDef(c)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif dpDef == nil {\n\t\treturn nil, dynamicPoolsError(\"no applicable dynamicPool type found\")\n\t}\n\tdynamicPool := p.dynamicPoolByDef(dpDef)\n\tif dynamicPool == nil {\n\t\treturn nil, dynamicPoolsError(\"no suitable dynamicPool instance available\")\n\t}\n\treturn dynamicPool, err\n}\n\n// dumpDynamicPool dumps dynamicPool contents in detail.\nfunc (p *dynamicPools) dumpDynamicPool(dp *DynamicPool) string {\n\tconts := []string{}\n\tpods := []string{}\n\tfor podID, contIDs := range dp.PodIDs {\n\t\tpodName := podID\n\t\tif pod, ok := p.cch.LookupPod(podID); ok {\n\t\t\tpodName = pod.GetName()\n\t\t}\n\t\tpods = append(pods, podName)\n\t\tfor _, contID := range contIDs {\n\t\t\tif cont, ok := p.cch.LookupContainer(contID); ok {\n\t\t\t\tconts = append(conts, cont.PrettyName())\n\t\t\t} else {\n\t\t\t\tconts = append(conts, podName+\".\"+contID)\n\t\t\t}\n\t\t}\n\t}\n\ts := fmt.Sprintf(\"DynamicPool %s{Cpus: %s; Mems: %s; mCPU requests: %d; mCPU limits: %d; capacity: %d; pods: %s; conts: %s}\",\n\t\tdp.PrettyName(),\n\t\tdp.Cpus,\n\t\tdp.Mems,\n\t\tp.requestedMinMilliCpus(dp),\n\t\tp.requestedMaxMilliCpus(dp),\n\t\tdp.AvailMilliCpus(),\n\t\tpods,\n\t\tconts)\n\treturn s\n}\n\n// changesDynamicPools returns true if two dynamicPools policy configurations\n// may lead into different dynamicPools or workload assignment.\nfunc changesDynamicPools(opts0, opts1 *DynamicPoolsOptions) bool {\n\tif opts0 == nil && opts1 == nil {\n\t\treturn false\n\t}\n\tif opts0 == nil || opts1 == nil {\n\t\treturn true\n\t}\n\tif len(opts0.DynamicPoolDefs) != len(opts1.DynamicPoolDefs) {\n\t\treturn true\n\t}\n\to0 := opts0.DeepCopy()\n\to1 := opts1.DeepCopy()\n\t// Ignore differences in CPU class names. Every other change\n\t// potentially changes dynamicPools or workloads.\n\tfor i := range o0.DynamicPoolDefs {\n\t\to0.DynamicPoolDefs[i].CpuClass = \"\"\n\t\to1.DynamicPoolDefs[i].CpuClass = \"\"\n\t}\n\treturn utils.DumpJSON(o0) != utils.DumpJSON(o1)\n}\n\n// changesCpuClasses returns true if two dynamicPools policy\n// configurations can lead to using different CPU classes on\n// corresponding dynamicPool instances. Calling changesCpuClasses(o0, o1)\n// makes sense only if changesDynamicPools(o0, o1) has returned false.\nfunc changesCpuClasses(opts0, opts1 *DynamicPoolsOptions) bool {\n\tif opts0 == nil && opts1 == nil {\n\t\treturn false\n\t}\n\tif opts0 == nil || opts1 == nil {\n\t\treturn true\n\t}\n\tif len(opts0.DynamicPoolDefs) != len(opts1.DynamicPoolDefs) {\n\t\treturn true\n\t}\n\tfor i := range opts0.DynamicPoolDefs {\n\t\tif opts0.DynamicPoolDefs[i].CpuClass != opts1.DynamicPoolDefs[i].CpuClass {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// configNotify applies new configuration.\nfunc (p *dynamicPools) configNotify(event pkgcfg.Event, source pkgcfg.Source) error {\n\tlog.Info(\"configuration %s\", event)\n\tdefer log.Debug(\"effective configuration:\\n%s\\n\", utils.DumpJSON(p.dpoptions))\n\tnewDynamicPoolsOptions := dynamicPoolsOptions.DeepCopy()\n\tif !changesDynamicPools(&p.dpoptions, newDynamicPoolsOptions) {\n\t\tif !changesCpuClasses(&p.dpoptions, newDynamicPoolsOptions) {\n\t\t\tlog.Info(\"no configuration changes\")\n\t\t} else {\n\t\t\tlog.Info(\"configuration changes only on CPU classes\")\n\t\t\t// Update new CPU classes to existing DynamicPool\n\t\t\t// definitions. The same DynamicPoolDef instances\n\t\t\t// must be kept in use, because each dynamicPool\n\t\t\t// instance holds a direct reference to its\n\t\t\t// DynamicPoolDef.\n\t\t\tfor i := range p.dpoptions.DynamicPoolDefs {\n\t\t\t\tp.dpoptions.DynamicPoolDefs[i].CpuClass = newDynamicPoolsOptions.DynamicPoolDefs[i].CpuClass\n\t\t\t}\n\t\t\t// (Re)configures all CPUs in DynamicPools.\n\t\t\tfor _, dp := range p.dynamicPools {\n\t\t\t\tp.useCpuClass(dp)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t}\n\tif err := p.setConfig(newDynamicPoolsOptions); err != nil {\n\t\tlog.Error(\"config update failed: %v\", err)\n\t\treturn err\n\t}\n\tlog.Info(\"config updated successfully\")\n\tp.Sync(p.cch.GetContainers(), p.cch.GetContainers())\n\treturn nil\n}\n\n// applyDynamicPoolDef creates user-defined dynamicPools or reconfigures built-in\n// dynamicPools according to the dpDef. Does not initialize dynamicPool CPUs.\nfunc (p *dynamicPools) applyDynamicPoolDef(dynamicPools *[]*DynamicPool, dpDef *DynamicPoolDef) error {\n\tif len(*dynamicPools) < 2 {\n\t\treturn dynamicPoolsError(\"internal error: reserved and shared dynamicPools missing, cannot apply dynamicPool definitions\")\n\t}\n\treservedDynamicPool := (*dynamicPools)[0]\n\tsharedDynamicPool := (*dynamicPools)[1]\n\t// Every dynamicPoolDef does one of the following:\n\t// 1. reconfigures the \"reserved\" dynamicPool (most restricted)\n\t// 2. reconfigures the \"shared\" dynamicPool (somewhat restricted)\n\t// 3. defines new user-defined dynamicPool.\n\tswitch dpDef.Name {\n\tcase \"\":\n\t\t// Case 0: bad name\n\t\treturn dynamicPoolsError(\"undefined or empty dynamicPool name\")\n\tcase reservedDynamicPool.Def.Name:\n\t\t// Case 1: reconfigure the \"reserved\" dynamicPool.\n\t\tp.reservedDynamicPoolDef.AllocatorPriority = dpDef.AllocatorPriority\n\t\tp.reservedDynamicPoolDef.CpuClass = dpDef.CpuClass\n\t\tp.reservedDynamicPoolDef.Namespaces = dpDef.Namespaces\n\tcase sharedDynamicPool.Def.Name:\n\t\t// Case 2: reconfigure the \"shared\" dynamicPool.\n\t\tp.sharedDynamicPoolDef.AllocatorPriority = dpDef.AllocatorPriority\n\t\tp.sharedDynamicPoolDef.CpuClass = dpDef.CpuClass\n\t\tp.sharedDynamicPoolDef.Namespaces = dpDef.Namespaces\n\tdefault:\n\t\t// Case 3: create each user-defined dynamicPool without CPU.\n\t\tnewdp, err := p.newDynamicPool(dpDef, false)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t*dynamicPools = append(*dynamicPools, newdp)\n\t}\n\treturn nil\n}\n\n// setConfig takes new dynamicPool configuration into use.\nfunc (p *dynamicPools) setConfig(dpoptions *DynamicPoolsOptions) error {\n\t// Create the default reserved and shared dynamicPool\n\t// definitions. Some properties of these definitions may be\n\t// altered by user configuration.\n\tp.reservedDynamicPoolDef = &DynamicPoolDef{\n\t\tName:              reservedDynamicPoolDefName,\n\t\tAllocatorPriority: 3,\n\t}\n\tp.sharedDynamicPoolDef = &DynamicPoolDef{\n\t\tName:              sharedDynamicPoolDefName,\n\t\tAllocatorPriority: 3,\n\t}\n\tp.dynamicPools = []*DynamicPool{}\n\tp.freeCpus = p.allowed.Clone()\n\tp.freeCpus = p.freeCpus.Difference(p.reserved)\n\t// Instantiate built-in reserved and shared dynamicPool.\n\treservedDynamicPool, err := p.newDynamicPool(p.reservedDynamicPoolDef, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tp.dynamicPools = append(p.dynamicPools, reservedDynamicPool)\n\tsharedDynamicPool, err := p.newDynamicPool(p.sharedDynamicPoolDef, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tp.dynamicPools = append(p.dynamicPools, sharedDynamicPool)\n\t// First apply customizations to built-in dynamicPools: \"reserved\"\n\t// and \"shared\".\n\tfor _, dpDef := range dpoptions.DynamicPoolDefs {\n\t\tif dpDef.Name != reservedDynamicPoolDefName && dpDef.Name != sharedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif err := p.applyDynamicPoolDef(&p.dynamicPools, dpDef); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Apply all user dynamicPool definitions, skip already customized\n\t// \"reserved\" and \"shared\" dynamicPools.\n\tfor _, dpDef := range dpoptions.DynamicPoolDefs {\n\t\tif dpDef.Name == reservedDynamicPoolDefName || dpDef.Name == sharedDynamicPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif err := p.applyDynamicPoolDef(&p.dynamicPools, dpDef); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Finish dynamicPool initialization.\n\tlog.Info(\"%s policy dynamicPools:\", PolicyName)\n\tfor dpIdx, dp := range p.dynamicPools {\n\t\tlog.Info(\"- dynamicPool %d: %s\", dpIdx, dp)\n\t}\n\t// No errors in dynamicPool creation, take new configuration into use.\n\tp.dpoptions = *dpoptions\n\t// (Re)configures all CPUs in dynamicPools.\n\tfor _, dp := range p.dynamicPools {\n\t\tp.useCpuClass(dp)\n\t}\n\treturn nil\n}\n\n// closestMems returns memory node IDs good for pinning containers\n// that run on given CPUs.\nfunc (p *dynamicPools) closestMems(cpus cpuset.CPUSet) idset.IDSet {\n\tmems := idset.NewIDSet()\n\tsys := p.options.System\n\tfor _, nodeID := range sys.NodeIDs() {\n\t\tif !cpus.Intersection(sys.Node(nodeID).CPUSet()).IsEmpty() {\n\t\t\tmems.Add(nodeID)\n\t\t}\n\t}\n\treturn mems\n}\n\n// assignContainer adds a container to a dynamicPool.\nfunc (p *dynamicPools) assignContainer(c cache.Container, dp *DynamicPool) {\n\tlog.Info(\"assigning container %s to dynamicPool %s\", c.PrettyName(), dp)\n\tpodID := c.GetPodID()\n\tdp.PodIDs[podID] = append(dp.PodIDs[podID], c.GetCacheID())\n\tp.pinCpuMem(c, dp.Cpus, dp.Mems)\n}\n\n// dismissContainer removes a container from a dynamicPool.\nfunc (p *dynamicPools) dismissContainer(c cache.Container, dp *DynamicPool) {\n\tpodID := c.GetPodID()\n\tdp.PodIDs[podID] = removeString(dp.PodIDs[podID], c.GetCacheID())\n\tif len(dp.PodIDs[podID]) == 0 {\n\t\tdelete(dp.PodIDs, podID)\n\t}\n}\n\n// pinCpuMem pins container to CPUs and memory nodes if flagged.\nfunc (p *dynamicPools) pinCpuMem(c cache.Container, cpus cpuset.CPUSet, mems idset.IDSet) {\n\tif p.dpoptions.PinCPU == nil || *p.dpoptions.PinCPU {\n\t\tlog.Debug(\"  - pinning %s to cpuset: %s\", c.PrettyName(), cpus)\n\t\tc.SetCpusetCpus(cpus.String())\n\t\tif reqCpu, ok := c.GetResourceRequirements().Requests[corev1.ResourceCPU]; ok {\n\t\t\tmCpu := int(reqCpu.MilliValue())\n\t\t\tc.SetCPUShares(int64(cache.MilliCPUToShares(int64(mCpu))))\n\t\t}\n\t}\n\tif p.dpoptions.PinMemory == nil || *p.dpoptions.PinMemory {\n\t\tlog.Debug(\"  - pinning %s to memory %s\", c.PrettyName(), mems)\n\t\tc.SetCpusetMems(mems.String())\n\t}\n}\n\n// dynamicPoolsError formats an error from this policy.\nfunc dynamicPoolsError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n\n// removeString returns the first occurrence of a string from string slice.\nfunc removeString(strings []string, element string) []string {\n\tfor index, s := range strings {\n\t\tif s == element {\n\t\t\tstrings[index] = strings[len(strings)-1]\n\t\t\treturn strings[:len(strings)-1]\n\t\t}\n\t}\n\treturn strings\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, CreateDynamicPoolsPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/dynamic-pools/dyp_test.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dyp\n\nimport (\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nfunc TestChangesDynamicPools(t *testing.T) {\n\ttcases := []struct {\n\t\tname          string\n\t\topts1         *DynamicPoolsOptions\n\t\topts2         *DynamicPoolsOptions\n\t\texpectedValue bool\n\t}{\n\t\t{\n\t\t\tname:          \"both options are nil\",\n\t\t\texpectedValue: false,\n\t\t},\n\t\t{\n\t\t\tname:          \"one option is nil\",\n\t\t\topts2:         &DynamicPoolsOptions{},\n\t\t\texpectedValue: true,\n\t\t},\n\t\t{\n\t\t\tname: \"reserved pool namespaces differ by len\",\n\t\t\topts1: &DynamicPoolsOptions{\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t},\n\t\t\topts2: &DynamicPoolsOptions{\n\t\t\t\tReservedPoolNamespaces: []string{},\n\t\t\t},\n\t\t\texpectedValue: true,\n\t\t},\n\t\t{\n\t\t\tname: \"reserved pool namespaces differ by content\",\n\t\t\topts1: &DynamicPoolsOptions{\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t},\n\t\t\topts2: &DynamicPoolsOptions{\n\t\t\t\tReservedPoolNamespaces: []string{\"ns1\"},\n\t\t\t},\n\t\t\texpectedValue: true,\n\t\t},\n\t\t{\n\t\t\tname: \"dynamic-pool defs differ\",\n\t\t\topts1: &DynamicPoolsOptions{\n\t\t\t\tReservedPoolNamespaces: []string{\"ns0\"},\n\t\t\t\tDynamicPoolDefs:        []*DynamicPoolDef{},\n\t\t\t},\n\t\t\topts2: &DynamicPoolsOptions{\n\t\t\t\tReservedPoolNamespaces: []string{\"ns1\"},\n\t\t\t\tDynamicPoolDefs:        []*DynamicPoolDef{},\n\t\t\t},\n\t\t\texpectedValue: true,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tvalue := changesDynamicPools(tc.opts1, tc.opts2)\n\t\t\tif value != tc.expectedValue {\n\t\t\t\tt.Errorf(\"Expected return value %v but got %v\", tc.expectedValue, value)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestIsNeedReallocate(t *testing.T) {\n\tp := &dynamicPools{\n\t\tdynamicPools: []*DynamicPool{\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: reservedDynamicPoolDefName,\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(1, 2),\n\t\t\t},\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: sharedDynamicPoolDefName,\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(3, 4, 5, 6),\n\t\t\t},\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: \"poo1\",\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(7, 8, 9, 10, 11, 12),\n\t\t\t},\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: \"poo2\",\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(0),\n\t\t\t},\n\t\t},\n\t}\n\ttcases := []struct {\n\t\tname          string\n\t\tnewPoolCpu    map[*DynamicPool]int\n\t\texpectedValue bool\n\t}{\n\t\t{\n\t\t\tname: \"no need to reallocate\",\n\t\t\tnewPoolCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 4,\n\t\t\t\tp.dynamicPools[2]: 6,\n\t\t\t\tp.dynamicPools[3]: 1,\n\t\t\t},\n\t\t\texpectedValue: false,\n\t\t},\n\t\t{\n\t\t\tname: \"need to reallocate\",\n\t\t\tnewPoolCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 6,\n\t\t\t\tp.dynamicPools[2]: 4,\n\t\t\t\tp.dynamicPools[3]: 1,\n\t\t\t},\n\t\t\texpectedValue: true,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tvalue := p.isNeedReallocate(tc.newPoolCpu)\n\t\t\tif value != tc.expectedValue {\n\t\t\t\tt.Errorf(\"Expected return value %v but got %v\", tc.expectedValue, value)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestCalculatePoolCpuset(t *testing.T) {\n\tp := &dynamicPools{\n\t\tallowed:  cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),\n\t\treserved: cpuset.New(1, 2),\n\t\tdynamicPools: []*DynamicPool{\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: reservedDynamicPoolDefName,\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(1, 2),\n\t\t\t},\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: sharedDynamicPoolDefName,\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(3, 4, 5, 6),\n\t\t\t},\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: \"poo1\",\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(7, 8, 9, 10, 11, 12, 13),\n\t\t\t},\n\t\t\t{\n\t\t\t\tDef: &DynamicPoolDef{\n\t\t\t\t\tName: \"poo2\",\n\t\t\t\t},\n\t\t\t\tCpus: cpuset.New(0),\n\t\t\t},\n\t\t},\n\t}\n\ttcases := []struct {\n\t\tname          string\n\t\trequestCpu    map[*DynamicPool]int\n\t\tremainFree    int\n\t\tweight        map[*DynamicPool]float64\n\t\tsumWeight     float64\n\t\texpectedValue map[*DynamicPool]int\n\t}{\n\t\t{\n\t\t\tname:       \"The requests and weight of the dynamic pools are both nil\",\n\t\t\trequestCpu: map[*DynamicPool]int{},\n\t\t\tremainFree: 12,\n\t\t\tweight:     map[*DynamicPool]float64{},\n\t\t\tsumWeight:  0.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 12,\n\t\t\t\tp.dynamicPools[2]: 0,\n\t\t\t\tp.dynamicPools[3]: 0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"The requests of the dynamic pools is not nil, and the requests of the shared dynamic pools is 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 1,\n\t\t\t\tp.dynamicPools[1]: 0,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t\tremainFree: 8,\n\t\t\tweight:     map[*DynamicPool]float64{},\n\t\t\tsumWeight:  0.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 8,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"The requests of the dynamic pools is not nil, and the requests of the shared dynamic pools is not 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 1,\n\t\t\t\tp.dynamicPools[1]: 2,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t\tremainFree: 6,\n\t\t\tweight:     map[*DynamicPool]float64{},\n\t\t\tsumWeight:  0.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 8,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"The weight of the dynamic pools is not nil, and the weight of the shared dynamic pools is not 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{},\n\t\t\tremainFree: 12,\n\t\t\tweight: map[*DynamicPool]float64{\n\t\t\t\tp.dynamicPools[0]: 10.0,\n\t\t\t\tp.dynamicPools[1]: 100.0,\n\t\t\t\tp.dynamicPools[2]: 200.0,\n\t\t\t\tp.dynamicPools[3]: 100.0,\n\t\t\t},\n\t\t\tsumWeight: 400.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 3,\n\t\t\t\tp.dynamicPools[2]: 6,\n\t\t\t\tp.dynamicPools[3]: 3,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:       \"The weight of the dynamic pools is not nil, and the weight of the shared dynamic pools is 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{},\n\t\t\tremainFree: 12,\n\t\t\tweight: map[*DynamicPool]float64{\n\t\t\t\tp.dynamicPools[0]: 10.0,\n\t\t\t\tp.dynamicPools[1]: 0.0,\n\t\t\t\tp.dynamicPools[2]: 200.0,\n\t\t\t\tp.dynamicPools[3]: 100.0,\n\t\t\t},\n\t\t\tsumWeight: 300.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 0,\n\t\t\t\tp.dynamicPools[2]: 8,\n\t\t\t\tp.dynamicPools[3]: 4,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"The requests and weight of the dynamic pools are not nil, and the requests of the shared dynamic pools is 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 1,\n\t\t\t\tp.dynamicPools[1]: 0,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t\tremainFree: 8,\n\t\t\tweight: map[*DynamicPool]float64{\n\t\t\t\tp.dynamicPools[0]: 10.0,\n\t\t\t\tp.dynamicPools[1]: 100.0,\n\t\t\t\tp.dynamicPools[2]: 200.0,\n\t\t\t\tp.dynamicPools[3]: 100.0,\n\t\t\t},\n\t\t\tsumWeight: 400.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 2,\n\t\t\t\tp.dynamicPools[2]: 6,\n\t\t\t\tp.dynamicPools[3]: 4,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"The requests and weight of the dynamic pools are not nil, and the weight of the shared dynamic pools is 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 1,\n\t\t\t\tp.dynamicPools[1]: 1,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t\tremainFree: 7,\n\t\t\tweight: map[*DynamicPool]float64{\n\t\t\t\tp.dynamicPools[0]: 10.0,\n\t\t\t\tp.dynamicPools[1]: 0.0,\n\t\t\t\tp.dynamicPools[2]: 200.0,\n\t\t\t\tp.dynamicPools[3]: 100.0,\n\t\t\t},\n\t\t\tsumWeight: 300.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 1,\n\t\t\t\tp.dynamicPools[2]: 7,\n\t\t\t\tp.dynamicPools[3]: 4,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"The requests and weight of the dynamic pools are not nil, and the requests and weight of the shared dynamic pools are both 0\",\n\t\t\trequestCpu: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 1,\n\t\t\t\tp.dynamicPools[1]: 0,\n\t\t\t\tp.dynamicPools[2]: 2,\n\t\t\t\tp.dynamicPools[3]: 2,\n\t\t\t},\n\t\t\tremainFree: 8,\n\t\t\tweight: map[*DynamicPool]float64{\n\t\t\t\tp.dynamicPools[0]: 10.0,\n\t\t\t\tp.dynamicPools[1]: 0.0,\n\t\t\t\tp.dynamicPools[2]: 200.0,\n\t\t\t\tp.dynamicPools[3]: 100.0,\n\t\t\t},\n\t\t\tsumWeight: 300.0,\n\t\t\texpectedValue: map[*DynamicPool]int{\n\t\t\t\tp.dynamicPools[0]: 2,\n\t\t\t\tp.dynamicPools[1]: 0,\n\t\t\t\tp.dynamicPools[2]: 8,\n\t\t\t\tp.dynamicPools[3]: 4,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tvalue := p.calculatePoolCpuset(tc.requestCpu, tc.remainFree, tc.weight, tc.sumWeight)\n\t\t\tfor k, v := range value {\n\t\t\t\tif v != tc.expectedValue[k] {\n\t\t\t\t\tt.Errorf(\"dynamic pool %v Expected return value %v but got %v\", k, tc.expectedValue[k], v)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/dynamic-pools/flags.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dyp\n\nimport (\n\t\"encoding/json\"\n\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n)\n\ntype DynamicPoolsOptions dynamicPoolsOptionsWrapped\n\n// dynamicPoolsOptions contains configuration options specific to this policy.\ntype dynamicPoolsOptionsWrapped struct {\n\t// PinCPU controls pinning containers to CPUs.\n\tPinCPU *bool `json:\"PinCPU,omitempty\"`\n\t// PinMemory controls pinning containers to memory nodes.\n\tPinMemory *bool `json:\"PinMemory,omitempty\"`\n\t// ReservedPoolNamespaces is a list of namespace globs that\n\t// will be allocated to reserved CPUs.\n\tReservedPoolNamespaces []string `json:\"ReservedPoolNamespaces,omitempty\"`\n\t// DynamicPoolDefs contains dynamicPool type definitions.\n\tDynamicPoolDefs []*DynamicPoolDef `json:\"DynamicPoolTypes,omitempty\"`\n}\n\n// DynamicPoolDef contains a dynamicPool definition.\ntype DynamicPoolDef struct {\n\t// Name of the dynamicPool definition.\n\tName       string   `json:\"Name\"`\n\tNamespaces []string `json:\"Namespaces\",omitempty`\n\tCpuClass   string   `json:\"CpuClass\"`\n\t// AllocatorPriority (0: High, 1: Normal, 2: Low, 3: None)\n\t// This parameter is passed to CPU allocator when creating or\n\t// resizing a dynamicPool. At init, dynamicPools with highest priority\n\t// CPUs are allocated first.\n\tAllocatorPriority cpuallocator.CPUPriority `json:\"AllocatorPriority\"`\n}\n\nvar defaultPinCPU bool = true\nvar defaultPinMemory bool = true\n\n// DeepCopy creates a deep copy of a DynamicPoolsOptions\nfunc (dpo *DynamicPoolsOptions) DeepCopy() *DynamicPoolsOptions {\n\toutDpo := *dpo\n\toutDpo.ReservedPoolNamespaces = make([]string, len(dpo.ReservedPoolNamespaces))\n\tcopy(outDpo.ReservedPoolNamespaces, dpo.ReservedPoolNamespaces)\n\toutDpo.DynamicPoolDefs = make([]*DynamicPoolDef, len(dpo.DynamicPoolDefs))\n\tfor i := range dpo.DynamicPoolDefs {\n\t\toutDpo.DynamicPoolDefs[i] = dpo.DynamicPoolDefs[i].DeepCopy()\n\t}\n\treturn &outDpo\n}\n\n// String stringifies a DynamicPoolsDef\nfunc (dpDef DynamicPoolDef) String() string {\n\treturn dpDef.Name\n}\n\n// DeepCopy creates a deep copy of a DynamicPoolDef\nfunc (bdef *DynamicPoolDef) DeepCopy() *DynamicPoolDef {\n\toutBdef := *bdef\n\toutBdef.Namespaces = make([]string, len(bdef.Namespaces))\n\tcopy(outBdef.Namespaces, bdef.Namespaces)\n\treturn &outBdef\n}\n\n// defaultDynamicPoolsOptions returns a new DynamicPoolsOptions instance, all initialized to defaults.\nfunc defaultDynamicPoolsOptions() interface{} {\n\treturn &DynamicPoolsOptions{\n\t\tReservedPoolNamespaces: []string{metav1.NamespaceSystem},\n\t\tPinCPU:                 &defaultPinCPU,\n\t\tPinMemory:              &defaultPinMemory,\n\t}\n}\n\n// Our runtime configuration.\nvar dynamicPoolsOptions = defaultDynamicPoolsOptions().(*DynamicPoolsOptions)\n\n// UnmarshalJSON makes sure all options from previous unmarshals get\n// cleared before unmarshaling new data to the same address.\nfunc (bo *DynamicPoolsOptions) UnmarshalJSON(data []byte) error {\n\tbow := dynamicPoolsOptionsWrapped{}\n\tif err := json.Unmarshal(data, &bow); err != nil {\n\t\treturn err\n\t}\n\t*bo = DynamicPoolsOptions(bow)\n\treturn nil\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tpkgcfg.Register(PolicyPath, PolicyDescription, dynamicPoolsOptions, defaultDynamicPoolsOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/dynamic-pools/metrics.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dyp\n\nimport (\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\n// Prometheus Metric descriptor indices and descriptor table\nconst (\n\tdynamicPoolsDesc = iota\n)\n\nvar descriptors = []*prometheus.Desc{\n\tdynamicPoolsDesc: prometheus.NewDesc(\n\t\t\"DynamicPools\",\n\t\t\"CPUs\",\n\t\t[]string{\n\t\t\t\"dynamicPool_type\",\n\t\t\t\"cpu_class\",\n\t\t\t\"dynamicPool\",\n\t\t\t\"cpus\",\n\t\t\t\"mems\",\n\t\t\t\"containers\",\n\t\t\t\"tot_req_millicpu\",\n\t\t\t\"tot_limit_millicpu\",\n\t\t}, nil,\n\t),\n}\n\n// Metrics defines the dynamicPools-specific metrics from policy level.\ntype Metrics struct {\n\tDynamicPools []*DynamicPoolMetrics\n}\n\n// DynamicPoolMetrics define metrics of a dynamicPool instance.\ntype DynamicPoolMetrics struct {\n\t// dynamicPool type metrics\n\tDefName  string\n\tCpuClass string\n\t// DynamicPool instance metrics\n\tPrettyName              string\n\tCpus                    cpuset.CPUSet\n\tMems                    string\n\tContainerNames          string\n\tContainerReqMilliCpus   int\n\tContainerLimitMilliCpus int\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data\n// descriptors.\nfunc (p *dynamicPools) DescribeMetrics() []*prometheus.Desc {\n\treturn descriptors\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *dynamicPools) PollMetrics() policy.Metrics {\n\tpolicyMetrics := &Metrics{}\n\tpolicyMetrics.DynamicPools = make([]*DynamicPoolMetrics, len(p.dynamicPools))\n\tfor index, dp := range p.dynamicPools {\n\t\tdm := &DynamicPoolMetrics{}\n\t\tpolicyMetrics.DynamicPools[index] = dm\n\t\tdm.DefName = dp.Def.Name\n\t\tdm.CpuClass = dp.Def.CpuClass\n\t\tdm.PrettyName = dp.PrettyName()\n\t\tdm.Cpus = dp.Cpus\n\t\tdm.Mems = dp.Mems.String()\n\t\tcNames := []string{}\n\t\t// Get container names, total requested milliCPUs and total limit milliCPUs.\n\t\tfor _, containerIDs := range dp.PodIDs {\n\t\t\tfor _, containerID := range containerIDs {\n\t\t\t\tif c, ok := p.cch.LookupContainer(containerID); ok {\n\t\t\t\t\tcNames = append(cNames, c.PrettyName())\n\t\t\t\t\tdm.ContainerReqMilliCpus += p.containerRequestedMilliCpus(containerID)\n\t\t\t\t\tdm.ContainerLimitMilliCpus += p.containerLimitedMilliCpus(containerID)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tsort.Strings(cNames)\n\t\tdm.ContainerNames = strings.Join(cNames, \",\")\n\t}\n\n\treturn policyMetrics\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled\n// policy-specific metrics data.\nfunc (p *dynamicPools) CollectMetrics(m policy.Metrics) ([]prometheus.Metric, error) {\n\tmetrics, ok := m.(*Metrics)\n\tif !ok {\n\t\treturn nil, dynamicPoolsError(\"type mismatch in dynamicPools metrics\")\n\t}\n\tpromMetrics := make([]prometheus.Metric, len(metrics.DynamicPools))\n\tfor index, dm := range metrics.DynamicPools {\n\t\tpromMetrics[index] = prometheus.MustNewConstMetric(\n\t\t\tdescriptors[dynamicPoolsDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tfloat64(dm.Cpus.Size()),\n\t\t\tdm.DefName,\n\t\t\tdm.CpuClass,\n\t\t\tdm.PrettyName,\n\t\t\tdm.Cpus.String(),\n\t\t\tdm.Mems,\n\t\t\tdm.ContainerNames,\n\t\t\tstrconv.Itoa(dm.ContainerReqMilliCpus),\n\t\t\tstrconv.Itoa(dm.ContainerLimitMilliCpus))\n\t}\n\treturn promMetrics, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/none/none-policy.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage none\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy implementation.\n\tPolicyName = policy.NonePolicy\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"A no-op policy, doing pretty much nothing.\"\n)\n\ntype none struct {\n\tlogger.Logger\n\tcch cache.Cache\n}\n\nvar _ policy.Backend = &none{}\n\n// CreateNonePolicy creates a new policy instance.\nfunc CreateNonePolicy(opts *policy.BackendOptions) policy.Backend {\n\tn := &none{Logger: logger.NewLogger(PolicyName)}\n\tn.Info(\"creating policy...\")\n\treturn n\n}\n\n// Name returns the name of this policy.\nfunc (n *none) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (n *none) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (n *none) Start(add []cache.Container, del []cache.Container) error {\n\tn.Debug(\"got started...\")\n\treturn nil\n}\n\n// Sync synchronizes the active policy state.\nfunc (n *none) Sync(add []cache.Container, del []cache.Container) error {\n\tn.Debug(\"(not) synchronizing policy state\")\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (n *none) AllocateResources(c cache.Container) error {\n\tn.Debug(\"(not) allocating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (n *none) ReleaseResources(c cache.Container) error {\n\tn.Debug(\"(not) releasing container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (n *none) UpdateResources(c cache.Container) error {\n\tn.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (n *none) Rebalance() (bool, error) {\n\tn.Debug(\"(not) rebalancing containers...\")\n\treturn false, nil\n}\n\n// HandleEvent handles policy-specific events.\nfunc (n *none) HandleEvent(*events.Policy) (bool, error) {\n\tn.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (n *none) ExportResourceData(c cache.Container) map[string]string {\n\treturn nil\n}\n\n// Introspect provides data for external introspection.\nfunc (n *none) Introspect(*introspect.State) {\n\treturn\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *none) PollMetrics() policy.Metrics {\n\treturn nil\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *none) DescribeMetrics() []*prometheus.Desc {\n\treturn nil\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *none) CollectMetrics(policy.Metrics) ([]prometheus.Metric, error) {\n\treturn nil, nil\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, CreateNonePolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/podpools/flags.go",
    "content": "// Copyright 2020-2021 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage podpools\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// PodpoolsOptions contains configuration options specific to this policy.\ntype PodpoolsOptions struct {\n\t// PinCPU controls pinning containers to CPUs.\n\tPinCPU bool `json:\"PinCPU,omitempty\"`\n\t// PinMemory controls pinning containers to memory nodes.\n\tPinMemory bool `json:\"PinMemory,omitempty\"`\n\t// PoolDefs contains pool definitions\n\tPoolDefs []*PoolDef `json:\"Pools,omitempty\"`\n}\n\n// PoolDef contains a pool definition.\ntype PoolDef struct {\n\t// Name is the name of the pool, or name prefix of\n\t// multi-instance pools.\n\tName string `json:\"Name\"`\n\t// CPU specifies the number of CPUs exclusively usable by\n\t// pods in the pool.\n\tCPU string `json:\"CPU\"`\n\t// MaxPods specifies the maximum number of pods assigned to\n\t// the pool. 0 (the default) means unlimited. -1 means no\n\t// pods.\n\tMaxPods int `json:\"MaxPods\"`\n\t// Instances specifies the number of multi-instance pools,\n\t// either directly or as CPU (count/percentage) reserved for\n\t// instances. The default is 1.\n\tInstances string `json:\"Instances,omitempty\"`\n\t// FillOrder specifies how multi-instance pools are filled.\n\tFillOrder FillOrder `json:\"FillOrder\"`\n\t// For the future: when enabling dynamic (on-demand) pool\n\t// instantiation, consider different ways of handling the case\n\t// of MaxPods>1, FillOrder==Balanced. Creating underloaded\n\t// pool instances will consume CPUs from other pool instances,\n\t// in a bad case causing workload migrations between memory\n\t// controllers when rearranging pool load is needed for\n\t// creation of new pools.\n}\n\n// FillOrder specifies the order in which pool instances should be filled.\ntype FillOrder int\n\nconst (\n\tFillBalanced FillOrder = iota\n\tFillPacked\n\tFillFirstFree\n)\n\nvar fillOrderNames = map[FillOrder]string{\n\tFillBalanced:  \"Balanced\",\n\tFillPacked:    \"Packed\",\n\tFillFirstFree: \"FirstFree\",\n}\n\n// String stringifies a FillOrder\nfunc (fo FillOrder) String() string {\n\tif fon, ok := fillOrderNames[fo]; ok {\n\t\treturn fon\n\t}\n\treturn fmt.Sprintf(\"#UNNAMED-FILLORDER(%d)\", int(fo))\n}\n\n// MarshalJSON marshals a FillOrder as a quoted json string\nfunc (fo FillOrder) MarshalJSON() ([]byte, error) {\n\tbuffer := bytes.NewBufferString(fmt.Sprintf(\"%q\", fo))\n\treturn buffer.Bytes(), nil\n}\n\n// UnmarshalJSON unmarshals a FillOrder quoted json string to the enum value\nfunc (fo *FillOrder) UnmarshalJSON(b []byte) error {\n\tvar fillOrderName string\n\terr := json.Unmarshal(b, &fillOrderName)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor foID, foName := range fillOrderNames {\n\t\tif foName == fillOrderName {\n\t\t\t*fo = foID\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn podpoolsError(\"invalid fill order %q\", fillOrderName)\n}\n\n// defaultPodpoolsOptions returns a new PodpoolsOptions instance, all initialized to defaults.\nfunc defaultPodpoolsOptions() interface{} {\n\treturn &PodpoolsOptions{\n\t\tPinCPU:    true,\n\t\tPinMemory: true,\n\t}\n}\n\n// Our runtime configuration.\nvar podpoolsOptions = defaultPodpoolsOptions().(*PodpoolsOptions)\n\n// Register us for configuration handling.\nfunc init() {\n\tpkgcfg.Register(PolicyPath, PolicyDescription, podpoolsOptions, defaultPodpoolsOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/podpools/metrics.go",
    "content": "// Copyright 2020-2021 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage podpools\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/procstats\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\n// Metrics defines the podpools-specific metrics from policy level.\ntype Metrics struct {\n\tPoolMetrics map[string]*PoolMetrics\n}\n\n// PoolMetrics defines the podpools-specific metrics from pool level.\ntype PoolMetrics struct {\n\tDefName        string\n\tPrettyName     string\n\tCPUs           cpuset.CPUSet\n\tCPUIds         []int\n\tMilliCPUs      string\n\tMemory         string\n\tContainerNames string\n\tPodNames       string\n}\n\n// Prometheus Metric descriptor indices and descriptor table\nconst (\n\tcpuUsageDesc = iota\n\tpoolCPUUsageDesc\n)\n\nvar descriptors = []*prometheus.Desc{\n\tcpuUsageDesc: prometheus.NewDesc(\n\t\t\"cpu_usage\",\n\t\t\"CPU usage per logical processor\",\n\t\t[]string{\n\t\t\t\"cpu\",\n\t\t}, nil,\n\t),\n\tpoolCPUUsageDesc: prometheus.NewDesc(\n\t\t\"pool_cpu_usage\",\n\t\t\"CPU usage for a given pool\",\n\t\t[]string{\n\t\t\t\"policy\",\n\t\t\t\"pretty_name\",\n\t\t\t\"def_name\",\n\t\t\t\"CPUs\",\n\t\t\t\"memory\",\n\t\t\t\"pool_size\",\n\t\t\t\"pod_name\",\n\t\t\t\"container_name\",\n\t\t}, nil,\n\t),\n}\n\nvar cpuTimeStat *procstats.CPUTimeStat\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *podpools) DescribeMetrics() []*prometheus.Desc {\n\treturn descriptors\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *podpools) PollMetrics() policy.Metrics {\n\tif p.pools == nil || len(p.pools) <= 0 {\n\t\tlog.Error(\"Failed to pull metrics.\")\n\t\treturn nil\n\t}\n\tpolicyMetrics := &Metrics{}\n\tpolicyMetrics.PoolMetrics = make(map[string]*PoolMetrics, len(p.pools))\n\n\tfor _, pool := range p.pools {\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()] = &PoolMetrics{}\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].DefName = pool.Def.Name\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].PrettyName = pool.PrettyName()\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].CPUs = pool.CPUs\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].CPUIds = pool.CPUs.List()\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].MilliCPUs = strconv.Itoa(pool.CPUs.Size() * 1000)\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].Memory = pool.Mems.String()\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].ContainerNames = \"\"\n\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].PodNames = \"\"\n\t\tif len(pool.PodIDs) > 0 {\n\t\t\tpodIds := make([]string, 0, len(pool.PodIDs))\n\t\t\tfor podId := range pool.PodIDs {\n\t\t\t\tpodIds = append(podIds, podId)\n\t\t\t}\n\t\t\tsort.Sort(sort.StringSlice(podIds))\n\t\t\tfor _, podId := range podIds {\n\t\t\t\tfor _, containerId := range pool.PodIDs[podId] {\n\t\t\t\t\tif container, ok := p.cch.LookupContainer(containerId); ok {\n\t\t\t\t\t\tcontainerName := container.PrettyName()\n\t\t\t\t\t\tif policyMetrics.PoolMetrics[pool.PrettyName()].ContainerNames == \"\" {\n\t\t\t\t\t\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].ContainerNames = containerName\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].ContainerNames = fmt.Sprintf(\"%s,%s\", policyMetrics.PoolMetrics[pool.PrettyName()].ContainerNames, containerName)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif pod, ok := p.cch.LookupPod(podId); ok {\n\t\t\t\t\tpodName := pod.GetName()\n\t\t\t\t\tif policyMetrics.PoolMetrics[pool.PrettyName()].PodNames == \"\" {\n\t\t\t\t\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].PodNames = podName\n\t\t\t\t\t} else {\n\t\t\t\t\t\tpolicyMetrics.PoolMetrics[pool.PrettyName()].PodNames = fmt.Sprintf(\"%s,%s\", policyMetrics.PoolMetrics[pool.PrettyName()].PodNames, podName)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn policyMetrics\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *podpools) CollectMetrics(m policy.Metrics) ([]prometheus.Metric, error) {\n\tmetrics, ok := m.(*Metrics)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"Wrong podpools metrics.\")\n\t}\n\tif cpuTimeStat == nil {\n\t\tif initSys, err := sysfs.DiscoverSystem(); err != nil {\n\t\t\treturn nil, err\n\t\t} else {\n\t\t\tcpuCount := len(initSys.CPUIDs())\n\t\t\tcpuTimeStat = &procstats.CPUTimeStat{\n\t\t\t\tPrevIdleTime:       make([]uint64, cpuCount),\n\t\t\t\tPrevTotalTime:      make([]uint64, cpuCount),\n\t\t\t\tCurIdleTime:        make([]uint64, cpuCount),\n\t\t\t\tCurTotalTime:       make([]uint64, cpuCount),\n\t\t\t\tDeltaIdleTime:      make([]uint64, cpuCount),\n\t\t\t\tDeltaTotalTime:     make([]uint64, cpuCount),\n\t\t\t\tCPUUsage:           make([]float64, cpuCount),\n\t\t\t\tIsGetCPUUsageBegin: false,\n\t\t\t}\n\t\t}\n\t}\n\terr := cpuTimeStat.GetCPUTimeStat()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcpuMetrics, err := updateCPUUsageMetrics()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tpoolCPUMetrics, err := updatePoolCPUUsageMetrics(metrics)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn append(cpuMetrics, poolCPUMetrics...), nil\n}\n\n// updateCPUUsageMetrics collects the CPU usage per logical processor.\nfunc updateCPUUsageMetrics() ([]prometheus.Metric, error) {\n\tcpuTimeStat.RLock()\n\tdefer cpuTimeStat.RUnlock()\n\tsys, err := sysfs.DiscoverSystem()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tonlined := sys.CPUSet().Difference(sys.Offlined())\n\tonlinedUsage := make([]prometheus.Metric, onlined.Size())\n\tfor i, j := range onlined.List() {\n\t\tonlinedUsage[i] = prometheus.MustNewConstMetric(\n\t\t\tdescriptors[cpuUsageDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tcpuTimeStat.CPUUsage[j],\n\t\t\tstrconv.Itoa(j),\n\t\t)\n\t}\n\treturn onlinedUsage, nil\n}\n\n// updatePoolCPUUsageMetrics collects the CPU usage of pools defined by podpools-policy.\nfunc updatePoolCPUUsageMetrics(ppm *Metrics) ([]prometheus.Metric, error) {\n\tif ppm == nil {\n\t\treturn nil, fmt.Errorf(\"Podpools metrics used to count pool CPU usage is missing.\")\n\t}\n\t// Sort the pool metrics.\n\tpoolNames := make([]string, 0, len(ppm.PoolMetrics))\n\tfor poolName := range ppm.PoolMetrics {\n\t\tpoolNames = append(poolNames, poolName)\n\t}\n\tsort.Sort(sort.StringSlice(poolNames))\n\n\t// Calculate the CPU usage of a pool and send to prometheus.\n\tpoolCPUUsageMetrics := make([]prometheus.Metric, len(poolNames))\n\tpoolCPUUsageList := make(map[string]float64, len(poolNames))\n\tcpuTimeStat.RLock()\n\tdefer cpuTimeStat.RUnlock()\n\tfor index, poolName := range poolNames {\n\t\tpoolDeltaIdleTime := uint64(0)\n\t\tpoolDeltaTotalTime := uint64(0)\n\t\tfor _, cpuId := range ppm.PoolMetrics[poolName].CPUIds {\n\t\t\tpoolDeltaIdleTime += cpuTimeStat.DeltaIdleTime[cpuId]\n\t\t\tpoolDeltaTotalTime += cpuTimeStat.DeltaTotalTime[cpuId]\n\t\t}\n\t\tpoolCPUUsageList[poolName] = 0.0\n\t\tif poolDeltaTotalTime != 0 {\n\t\t\tsys, err := sysfs.DiscoverSystem()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tpoolCPUOnlined := ppm.PoolMetrics[poolName].CPUs.Difference(sys.Offlined())\n\t\t\tpoolCPUUsageList[poolName] = (1.0 - float64(poolDeltaIdleTime)/float64(poolDeltaTotalTime)) * 100.0 * float64(len(poolCPUOnlined.List()))\n\t\t}\n\t\tpoolCPUUsageMetrics[index] = prometheus.MustNewConstMetric(\n\t\t\tdescriptors[poolCPUUsageDesc],\n\t\t\tprometheus.GaugeValue,\n\t\t\tpoolCPUUsageList[poolName],\n\t\t\tPolicyName,\n\t\t\tpoolName,\n\t\t\tppm.PoolMetrics[poolName].DefName,\n\t\t\tppm.PoolMetrics[poolName].CPUs.String(),\n\t\t\tppm.PoolMetrics[poolName].Memory,\n\t\t\tppm.PoolMetrics[poolName].MilliCPUs,\n\t\t\tppm.PoolMetrics[poolName].PodNames,\n\t\t\tppm.PoolMetrics[poolName].ContainerNames,\n\t\t)\n\t}\n\treturn poolCPUUsageMetrics, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/podpools/podpools-policy.go",
    "content": "// Copyright 2020-2021 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage podpools\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tpolicyapi \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy.\n\tPolicyName = \"podpools\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"Pod-granularity workload placement\"\n\t// PolicyPath is the path of this policy in the configuration hierarchy.\n\tPolicyPath = \"policy.\" + PolicyName\n\t// podpoolKey is a pod annotation key, the value is a pod pool name.\n\tpodpoolKey = \"pool.\" + PolicyName + \".\" + kubernetes.ResmgrKeyNamespace\n\t// reservedPoolDefName is the name in the reserved pool definition.\n\treservedPoolDefName = \"reserved\"\n\t// defaultPoolDefName is the name in the default pool definition.\n\tdefaultPoolDefName = \"default\"\n\t// podMilliCPUErrorMargin is the maximum error in requested vs\n\t// allocated mCPUs per pod. For instance, 10 mCPU error margin\n\t// allows error of magnitude of +-0.5 mCPU/container up to 20\n\t// containers/pod.\n\tpodMilliCPUErrorMargin = int64(10)\n)\n\n// podpools contains configuration and runtime attributes of the podpools policy\ntype podpools struct {\n\toptions         *policyapi.BackendOptions // configuration common to all policies\n\tppoptions       PodpoolsOptions           // podpools-specific configuration\n\tcch             cache.Cache               // cri-resmgr cache\n\tallowed         cpuset.CPUSet             // bounding set of CPUs we're allowed to use\n\treserved        cpuset.CPUSet             // system-/kube-reserved CPUs\n\treservedPoolDef *PoolDef                  // built-in definition of the reserved pool\n\tdefaultPoolDef  *PoolDef                  // built-in definition of the default pool\n\tpools           []*Pool                   // pools for pods: reserved, default and user-defined\n\tpodMaxMilliCPU  map[string]int64          // maximum total MilliCPUs requested by containers of pods in pools\n\tcpuAllocator    cpuallocator.CPUAllocator // CPU allocator used by the policy\n}\n\n// Pool contains attributes of a pool instance\ntype Pool struct {\n\t// Def is the definition from which this pool instance is created.\n\tDef *PoolDef\n\t// Instance is the index of this pool instance, starting from\n\t// zero for every pool definition.\n\tInstance int\n\t// CPUs is the set of CPUs exclusive to this pool instance only.\n\tCPUs cpuset.CPUSet\n\t// Mems is the set of memory nodes with minimal access delay\n\t// from CPUs.\n\tMems idset.IDSet\n\t// PodIDs maps pod ID to list of container IDs.\n\t// - len(PodIDs) is the number of pods in the pool.\n\t// - len(PodIDs[podID]) is the number of containers of podID\n\t//   currently assigned to the pool.\n\t// - Def.MaxPods - len(PodIDs) is free pod capacity.\n\tPodIDs map[string][]string\n}\n\nvar log logger.Logger = logger.NewLogger(\"policy\")\n\n// String is a stringer for a pool.\nfunc (pool Pool) String() string {\n\tpodCount := len(pool.PodIDs)\n\tcontCount := 0\n\tfor _, contIDs := range pool.PodIDs {\n\t\tcontCount += len(contIDs)\n\t}\n\ts := fmt.Sprintf(\"%s{cpus:%s, mems:%s, pods:%d/%d, containers:%d}\",\n\t\tpool.PrettyName(), pool.CPUs, pool.Mems, podCount, pool.Def.MaxPods, contCount)\n\treturn s\n}\n\n// PrettyName returns unique name for a pool.\nfunc (pool Pool) PrettyName() string {\n\treturn fmt.Sprintf(\"%s[%d]\", pool.Def.Name, pool.Instance)\n}\n\n// CreatePodpoolsPolicy creates a new policy instance.\nfunc CreatePodpoolsPolicy(policyOptions *policy.BackendOptions) policy.Backend {\n\tp := &podpools{\n\t\toptions: policyOptions,\n\t\tcch:     policyOptions.Cache,\n\t\treservedPoolDef: &PoolDef{\n\t\t\tName:    reservedPoolDefName,\n\t\t\tMaxPods: 0,\n\t\t},\n\t\tdefaultPoolDef: &PoolDef{\n\t\t\tName:    defaultPoolDefName,\n\t\t\tMaxPods: 0,\n\t\t},\n\t\tpodMaxMilliCPU: make(map[string]int64),\n\t\tcpuAllocator:   cpuallocator.NewCPUAllocator(policyOptions.System),\n\t}\n\tlog.Info(\"creating %s policy...\", PolicyName)\n\t// Handle common policy options: AvailableResources and ReservedResources.\n\t// p.allowed: CPUs available for the policy\n\tif allowed, ok := policyOptions.Available[policyapi.DomainCPU]; ok {\n\t\tp.allowed = allowed.(cpuset.CPUSet)\n\t} else {\n\t\t// Available CPUs not specified, default to all on-line CPUs.\n\t\tp.allowed = policyOptions.System.CPUSet().Difference(policyOptions.System.Offlined())\n\t}\n\t// p.reserved: CPUs reserved for kube-system pods, subset of p.allowed.\n\tp.reserved = cpuset.New()\n\tif reserved, ok := p.options.Reserved[policyapi.DomainCPU]; ok {\n\t\tswitch v := reserved.(type) {\n\t\tcase cpuset.CPUSet:\n\t\t\tp.reserved = p.allowed.Intersection(v)\n\t\tcase resapi.Quantity:\n\t\t\treserveCnt := (int(v.MilliValue()) + 999) / 1000\n\t\t\tcpus, err := p.cpuAllocator.AllocateCpus(&p.allowed, reserveCnt, cpuallocator.PriorityNone)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(\"failed to allocate reserved CPUs: %s\", err)\n\t\t\t}\n\t\t\tp.reserved = cpus\n\t\t\tp.allowed = p.allowed.Union(cpus)\n\t\t}\n\t}\n\tif p.reserved.IsEmpty() {\n\t\tlog.Fatal(\"%s cannot run without reserved CPUs that are also AvailableResources\", PolicyName)\n\t}\n\t// Handle policy-specific options\n\tlog.Debug(\"creating %s configuration\", PolicyName)\n\tif err := p.setConfig(podpoolsOptions); err != nil {\n\t\tlog.Fatal(\"failed to create %s policy: %v\", PolicyName, err)\n\t}\n\n\tpkgcfg.GetModule(PolicyPath).AddNotify(p.configNotify)\n\n\treturn p\n}\n\n// Name returns the name of this policy.\nfunc (p *podpools) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (p *podpools) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (p *podpools) Start(add []cache.Container, del []cache.Container) error {\n\tlog.Info(\"%s policy started\", PolicyName)\n\treturn p.Sync(p.cch.GetContainers(), del)\n}\n\n// Sync synchronizes the active policy state.\nfunc (p *podpools) Sync(add []cache.Container, del []cache.Container) error {\n\tlog.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\tp.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\tp.AllocateResources(c)\n\t}\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (p *podpools) AllocateResources(c cache.Container) error {\n\tlog.Debug(\"allocating container %s...\", c.PrettyName())\n\t// Assign container to correct pool.\n\tpod, ok := c.GetPod()\n\tif !ok {\n\t\treturn podpoolsError(\"cannot find pod of container %s from the cache\", c.PrettyName())\n\t}\n\tif pool := p.allocatePool(pod); pool != nil {\n\t\tp.assignContainer(c, pool)\n\t\tp.trackPodCPU(pod, pool)\n\t\tif log.DebugEnabled() {\n\t\t\tlog.Debug(p.dumpPool(pool))\n\t\t}\n\t} else {\n\t\t// Cannot assign container to any of the pooled CPUs.\n\t\treturn podpoolsError(\"cannot find CPUs to run container %s - no default or reserved CPUs available\", c.PrettyName())\n\t}\n\treturn nil\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (p *podpools) ReleaseResources(c cache.Container) error {\n\tlog.Debug(\"releasing container %s...\", c.PrettyName())\n\tpod, ok := c.GetPod()\n\tif !ok {\n\t\treturn podpoolsError(\"cannot find pod of container %s from the cache\", c.PrettyName())\n\t}\n\tif pool := p.allocatedPool(pod); pool != nil {\n\t\tp.dismissContainer(c, pool)\n\t\tif log.DebugEnabled() {\n\t\t\tlog.Debug(p.dumpPool(pool))\n\t\t}\n\t\tif p.containersInPool(pod, pool) == 0 {\n\t\t\tlog.Debug(\"all containers removed, free pool allocation %s for pod %q\", pool.PrettyName(), pod.GetName())\n\t\t\tp.validatePodCPU(pod, pool)\n\t\t\tp.freePool(pod, pool)\n\t\t}\n\t} else {\n\t\tlog.Debug(\"ReleaseResources: pool-less container %s, nothing to release\", c.PrettyName())\n\t}\n\treturn nil\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (p *podpools) UpdateResources(c cache.Container) error {\n\tlog.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (p *podpools) Rebalance() (bool, error) {\n\tlog.Debug(\"(not) rebalancing containers...\")\n\treturn false, nil\n}\n\n// HandleEvent handles policy-specific events.\nfunc (p *podpools) HandleEvent(*events.Policy) (bool, error) {\n\tlog.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (p *podpools) ExportResourceData(c cache.Container) map[string]string {\n\treturn nil\n}\n\n// Introspect provides data for external introspection.\nfunc (p *podpools) Introspect(*introspect.State) {\n\treturn\n}\n\n// allocatedPool returns a pool already allocated for a pod.\nfunc (p *podpools) allocatedPool(pod cache.Pod) *Pool {\n\tpodID := pod.GetID()\n\tpools := filterPools(p.pools,\n\t\tfunc(pl *Pool) bool { _, ok := pl.PodIDs[podID]; return ok })\n\tif len(pools) == 0 {\n\t\treturn nil\n\t}\n\treturn pools[0]\n}\n\n// allocatePool returns a pool allocated for a pod.\nfunc (p *podpools) allocatePool(pod cache.Pod) *Pool {\n\tif pool := p.allocatedPool(pod); pool != nil {\n\t\treturn pool\n\t}\n\tpoolDef := p.getPoolDef(pod)\n\tif poolDef == nil {\n\t\treturn nil\n\t}\n\t// Try to find a suitable pool and allocate it for the pod.\n\tpools := filterPools(p.pools,\n\t\tfunc(pl *Pool) bool {\n\t\t\treturn poolDef.Name == pl.Def.Name && (pl.Def.MaxPods > len(pl.PodIDs) || pl.Def.MaxPods == 0)\n\t\t})\n\t// Sort pools according to pool type fill order so that the\n\t// first pool in the list is the preferred one.\n\tswitch poolDef.FillOrder {\n\tcase FillBalanced:\n\t\tsort.Slice(pools, func(i, j int) bool {\n\t\t\treturn len(pools[i].PodIDs) < len(pools[j].PodIDs)\n\t\t})\n\tcase FillPacked:\n\t\tsort.Slice(pools, func(i, j int) bool {\n\t\t\treturn len(pools[i].PodIDs) > len(pools[j].PodIDs)\n\t\t})\n\tcase FillFirstFree:\n\t\t// FirstFree is already the first of the pools list.\n\t}\n\tif len(pools) == 0 {\n\t\tlog.Error(\"cannot find free %q pool for pod %q, falling back to %q\", poolDef.Name, pod.GetName(), defaultPoolDefName)\n\t\tpools = []*Pool{p.pools[1]}\n\t}\n\t// Found a suitable pool. Allocate it for the pod.\n\tpodID := pod.GetID()\n\tpool := pools[0]\n\tpool.PodIDs[podID] = []string{}\n\tlog.Debug(\"allocated pool %s[%d] for pod %q\", pool.Def.Name, pool.Instance, pod.GetName())\n\treturn pool\n}\n\n// containersInPool returns the number of containers of a pod in a pool.\nfunc (p *podpools) containersInPool(pod cache.Pod, pool *Pool) int {\n\tif cnts, ok := pool.PodIDs[pod.GetID()]; ok {\n\t\treturn len(cnts)\n\t}\n\treturn 0\n}\n\n// dumpPool dumps pool contents in detail.\nfunc (p *podpools) dumpPool(pool *Pool) string {\n\tconts := []string{}\n\tpods := []string{}\n\tfor podID, contIDs := range pool.PodIDs {\n\t\tpodName := podID\n\t\tif pod, ok := p.cch.LookupPod(podID); ok {\n\t\t\tpodName = pod.GetName()\n\t\t}\n\t\tpods = append(pods, fmt.Sprintf(\"%s (mCPU: %d, max=%d)\", podName, p.getPodMilliCPU(podID), p.podMaxMilliCPU[podID]))\n\t\tfor _, contID := range contIDs {\n\t\t\tif cont, ok := p.cch.LookupContainer(contID); ok {\n\t\t\t\tconts = append(conts, cont.PrettyName())\n\t\t\t} else {\n\t\t\t\tconts = append(conts, podName+\":\"+contID)\n\t\t\t}\n\t\t}\n\t}\n\ts := fmt.Sprintf(\"Pool{Def.Name: %q, Instance: %d, CPUs: %s, Mems: %s, Def.MaxPods: %d, pods: %v, containers:%v}\",\n\t\tpool.Def.Name, pool.Instance, pool.CPUs, pool.Mems, pool.Def.MaxPods, pods, conts)\n\treturn s\n}\n\n// freePool removes an empty pod from a pool\nfunc (p *podpools) freePool(pod cache.Pod, pool *Pool) {\n\tpodID := pod.GetID()\n\tdelete(pool.PodIDs, podID)\n\tdelete(p.podMaxMilliCPU, podID)\n}\n\n// trackPodCPU keeps track on pod's CPU requests.\nfunc (p *podpools) trackPodCPU(pod cache.Pod, pool *Pool) {\n\t// As we do not have direct information on total CPU resources\n\t// requested by a pod, we gather the information indirectly by\n\t// tracking the sum of requested CPUs of its running\n\t// containers. This enables reacting to misalignment between\n\t// CPU resources per pod in a pool and CPU resource requests\n\t// visible to the kube-scheduler.\n\tpodID := pod.GetID()\n\tcurrent := p.getPodMilliCPU(podID)\n\tif max, ok := p.podMaxMilliCPU[podID]; ok {\n\t\tif max < current {\n\t\t\tp.podMaxMilliCPU[podID] = current\n\t\t}\n\t} else {\n\t\tp.podMaxMilliCPU[podID] = current\n\t}\n\t// Check overbooking\n\tif cpuAvail := p.availableMilliCPUs(pool); cpuAvail < 0 {\n\t\tlog.Error(\"overbooked pool %q, cpuset:%s: %dm / %dm CPUs used, %d mCPU available\", pool.PrettyName(), pool.CPUs, pool.CPUs.Size()*1000-int(cpuAvail), pool.CPUs.Size()*1000, cpuAvail)\n\t}\n}\n\n// validatePodCPU compares max CPU requests against pool CPU capacity per pod.\nfunc (p *podpools) validatePodCPU(pod cache.Pod, pool *Pool) {\n\t// Log pod configuration error if a pool has fixed amount of\n\t// CPUs per pod but the pod failed to request the correct\n\t// amount.\n\tpodID := pod.GetID()\n\tif podmCPU, ok := p.podMaxMilliCPU[podID]; ok {\n\t\tif pool.Def.MaxPods > 0 {\n\t\t\tpoolmCPUperPod := int64(pool.CPUs.Size() * 1000 / pool.Def.MaxPods)\n\t\t\tmCPUerr := podmCPU - poolmCPUperPod\n\t\t\t// Allow rounding errors (up and down) when\n\t\t\t// comparing the sum of containers' CPU usages\n\t\t\t// against milli-CPUs allocated per pod in its\n\t\t\t// pool.\n\t\t\tif mCPUerr < -podMilliCPUErrorMargin || mCPUerr > podMilliCPUErrorMargin {\n\t\t\t\tpodName := \"\"\n\t\t\t\tif pod, ok := p.cch.LookupPod(podID); ok {\n\t\t\t\t\tpodName = pod.GetName()\n\t\t\t\t}\n\t\t\t\tlog.Error(\"bad CPU requests: pod %q requested %d mCPUs, but in pool %q pods must request %d mCPUs.\", podName, podmCPU, pool.Def.Name, poolmCPUperPod)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// getPodMilliCPU returns mCPUs requested by podID.\nfunc (p *podpools) getPodMilliCPU(podID string) int64 {\n\tcpuRequested := int64(0)\n\tfor _, c := range p.cch.GetContainers() {\n\t\tif c.GetPodID() == podID {\n\t\t\tif reqCpu, ok := c.GetResourceRequirements().Requests[corev1.ResourceCPU]; ok {\n\t\t\t\tcpuRequested += reqCpu.MilliValue()\n\t\t\t}\n\t\t}\n\t}\n\treturn cpuRequested\n}\n\n// configNotify applies new configuration.\nfunc (p *podpools) configNotify(event pkgcfg.Event, source pkgcfg.Source) error {\n\tlog.Info(\"configuration %s\", event)\n\tif err := p.setConfig(podpoolsOptions); err != nil {\n\t\tlog.Error(\"config update failed: %v\", err)\n\t\treturn err\n\t}\n\tlog.Info(\"config updated successfully\")\n\tp.Sync(p.cch.GetContainers(), nil)\n\treturn nil\n}\n\n// getPoolDefName returns the name of the pool definition of a pod.\nfunc (p *podpools) getPoolDefName(pod cache.Pod) string {\n\tif poolDefName, ok := pod.GetEffectiveAnnotation(podpoolKey, \"\"); ok {\n\t\treturn poolDefName\n\t}\n\tif pod.GetNamespace() == \"kube-system\" {\n\t\treturn reservedPoolDefName\n\t}\n\treturn defaultPoolDefName\n}\n\n// getPoolDef returns the pool definition of a pod.\nfunc (p *podpools) getPoolDef(pod cache.Pod) *PoolDef {\n\tpoolDefName := p.getPoolDefName(pod)\n\tif poolDefName == reservedPoolDefName {\n\t\treturn p.reservedPoolDef\n\t}\n\tif poolDefName == defaultPoolDefName {\n\t\treturn p.defaultPoolDef\n\t}\n\tfor _, poolDef := range p.ppoptions.PoolDefs {\n\t\tif poolDef.Name == poolDefName {\n\t\t\treturn poolDef\n\t\t}\n\t}\n\tlog.Error(\"pod %q pool %q does not match any pool definition, falling back to %q\", pod.GetName(), poolDefName, p.defaultPoolDef.Name)\n\treturn p.defaultPoolDef\n}\n\n// applyPoolDef creates user-defined pools or reconfigures built-in\n// pools according to the poolDef.\nfunc (p *podpools) applyPoolDef(pools *[]*Pool, poolDef *PoolDef, freeCpus *cpuset.CPUSet, nonReservedCpuCount int) error {\n\tif len(*pools) < 2 {\n\t\treturn podpoolsError(\"internal error: reserved and default pools missing, cannot apply pool definitions\")\n\t}\n\treservedPool := (*pools)[0]\n\tdefaultPool := (*pools)[1]\n\t// Every PoolDef does one of the following:\n\t// 1. reconfigures the \"reserved\" pool (most restricted)\n\t// 2. reconfigutes the \"default\" pool (somewhat restricted)\n\t// 3. defines new user-defined pools.\n\tswitch poolDef.Name {\n\tcase \"\":\n\t\t// Case 0: bad name\n\t\treturn podpoolsError(\"undefined or empty pool name\")\n\n\tcase reservedPool.Def.Name:\n\t\t// Case 1: reconfigure the \"reserved\" pool.\n\t\t// Forbid redefinition of CPU and Instances.\n\t\tif poolDef.CPU != \"\" || poolDef.Instances != \"\" {\n\t\t\tpoolCount, cpusPerPool, err := parseInstancesCPUs(poolDef.Instances, poolDef.CPU, nonReservedCpuCount)\n\t\t\tif err != nil {\n\t\t\t\treturn podpoolsError(\"pool %q: %w\", poolDef.Name, err)\n\t\t\t}\n\t\t\tif poolCount != 1 {\n\t\t\t\treturn podpoolsError(\"pool %q: cannot change the number of instances\", poolDef.Name)\n\t\t\t}\n\t\t\tif cpusPerPool != reservedPool.CPUs.Size() {\n\t\t\t\treturn podpoolsError(\"pool %q: number of CPUs is conflicting ReservedResources CPUs\", poolDef.Name)\n\t\t\t}\n\t\t}\n\t\treservedPool.Def.MaxPods = poolDef.MaxPods\n\n\tcase defaultPool.Def.Name:\n\t\t// Case 2: reconfigure the \"default\" pool.\n\t\t// Allow redefinition of CPU but not Instances.\n\t\tif poolDef.CPU != \"\" || poolDef.Instances != \"\" {\n\t\t\tpoolCount, cpusPerPool, err := parseInstancesCPUs(poolDef.Instances, poolDef.CPU, nonReservedCpuCount)\n\t\t\tif err != nil {\n\t\t\t\treturn podpoolsError(\"pool %q: %w\", poolDef.Name, err)\n\t\t\t}\n\t\t\tif poolCount != 1 {\n\t\t\t\treturn podpoolsError(\"pool %q: cannot change the number of instances\", poolDef.Name)\n\t\t\t}\n\t\t\tcpus, err := p.cpuAllocator.AllocateCpus(freeCpus, cpusPerPool, cpuallocator.PriorityNormal)\n\t\t\tif err != nil {\n\t\t\t\treturn podpoolsError(\"could not allocate %d CPUs for pool %q: %w\", cpusPerPool, poolDef.Name, err)\n\t\t\t}\n\t\t\tdefaultPool.CPUs = cpus\n\t\t}\n\t\tdefaultPool.Def.MaxPods = poolDef.MaxPods\n\n\tdefault:\n\t\t// Case 3: create new user-defined pool(s).\n\t\tpoolCount, cpusPerPool, err := parseInstancesCPUs(poolDef.Instances, poolDef.CPU, nonReservedCpuCount)\n\t\tif err != nil {\n\t\t\treturn podpoolsError(\"pool %q: %w\", poolDef.Name, err)\n\t\t}\n\t\tif poolCount == 0 {\n\t\t\treturn podpoolsError(\"pool %q: insufficient CPUs to create any instances\", poolDef.Name)\n\t\t}\n\t\tif poolCount > 1 && poolDef.FillOrder == FillPacked && poolDef.MaxPods == 0 {\n\t\t\treturn podpoolsError(\"pool %q: %d pool(s) unreachable due to unlimited pod capacity and FillOrder: %s\", poolDef.Name, poolCount-1, poolDef.FillOrder)\n\t\t}\n\t\tlog.Debug(\"allocating %d out of %d non-reserved CPUs for %d %q pools\", poolCount*cpusPerPool, nonReservedCpuCount, poolCount, poolDef.Name)\n\t\tfor poolIndex := 0; poolIndex < poolCount; poolIndex++ {\n\t\t\tif cpusPerPool > freeCpus.Size() {\n\t\t\t\treturn podpoolsError(\"insufficient CPUs when trying to allocate %d CPUs for pool %s[%d]\", cpusPerPool, poolDef.Name, poolIndex)\n\t\t\t}\n\t\t\tcpus, err := p.cpuAllocator.AllocateCpus(freeCpus, cpusPerPool, cpuallocator.PriorityNormal)\n\t\t\tif err != nil {\n\t\t\t\treturn podpoolsError(\"could not allocate %d CPUs for instance %d of pool %q: %w\", cpusPerPool, poolIndex, poolDef.Name, err)\n\t\t\t}\n\t\t\tpool := Pool{\n\t\t\t\tDef:      poolDef,\n\t\t\t\tInstance: poolIndex,\n\t\t\t\tCPUs:     cpus,\n\t\t\t}\n\t\t\t*pools = append(*pools, &pool)\n\t\t}\n\t}\n\treturn nil\n}\n\n// setConfig takes new pool configuration into use.\nfunc (p *podpools) setConfig(ppoptions *PodpoolsOptions) error {\n\t// Instantiate pools for pods.\n\tpools := []*Pool{}\n\t// Built-in reserved pool.\n\treservedPool := Pool{\n\t\tDef:  p.reservedPoolDef,\n\t\tCPUs: p.reserved,\n\t}\n\tpools = append(pools, &reservedPool)\n\t// Built-in default pool.\n\t// The default pool will use reserved CPUs by default. If CPUs\n\t// are left over after constructing user-defined pools, those\n\t// will be used as the Default pool instead.\n\tdefaultPool := Pool{\n\t\tDef:  p.defaultPoolDef,\n\t\tCPUs: reservedPool.CPUs,\n\t}\n\tpools = append(pools, &defaultPool)\n\t// Apply pool definitions from configuration.\n\tfreeCpus := p.allowed.Clone()\n\tfreeCpus = freeCpus.Difference(p.reserved)\n\tnonReservedCpuCount := freeCpus.Size()\n\tuserPoolDefs := 0\n\t// First apply customizations to built-in pools: \"reserved\"\n\t// and \"default\".\n\tfor _, poolDef := range ppoptions.PoolDefs {\n\t\tif poolDef.Name != reservedPoolDefName && poolDef.Name != defaultPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif err := p.applyPoolDef(&pools, poolDef, &freeCpus, nonReservedCpuCount); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// Update nonReservedCount: if the default pool is customized\n\t// with its own CPUs, do not count those CPUs in the\n\t// \"Instances: 100%\" syntax of user-defined pools.\n\tnonReservedCpuCount = freeCpus.Size()\n\t// Apply all user pool definitions, skip \"reserved\" and \"default\".\n\tfor _, poolDef := range ppoptions.PoolDefs {\n\t\tif poolDef.Name == reservedPoolDefName || poolDef.Name == defaultPoolDefName {\n\t\t\tcontinue\n\t\t}\n\t\tif err := p.applyPoolDef(&pools, poolDef, &freeCpus, nonReservedCpuCount); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tuserPoolDefs += 1\n\t}\n\t// Check if there are unallocated CPUs.\n\tif freeCpus.Size() > 0 {\n\t\tif defaultPool.CPUs.Intersection(reservedPool.CPUs).IsEmpty() {\n\t\t\t// User has reallocated \"default\" pool CPUs\n\t\t\tlog.Debug(\"%d unused CPUs are added to the default pool.\", freeCpus.Size())\n\t\t\tdefaultPool.CPUs = defaultPool.CPUs.Union(freeCpus)\n\t\t} else {\n\t\t\tlog.Debug(\"%d unused CPUs are used as the default pool.\", freeCpus.Size())\n\t\t\tdefaultPool.CPUs = freeCpus\n\t\t}\n\t}\n\t// Finish pool instance initialization.\n\tlog.Info(\"%s policy pools:\", PolicyName)\n\tfor index, pool := range pools {\n\t\tpool.Mems = p.closestMems(pool.CPUs)\n\t\tpool.PodIDs = make(map[string][]string)\n\t\tlog.Info(\"- pool %d: %s\", index, pool)\n\t}\n\t// No errors in pool creation, take new configuration into use.\n\tlog.Debug(\"new %s configuration:\\n%s\", PolicyName, utils.DumpJSON(ppoptions))\n\tp.pools = pools\n\tp.ppoptions = *ppoptions\n\t// Warning on multiple user-defined pools.\n\tif userPoolDefs > 1 {\n\t\tlog.Warn(\"Multiple (%d) user-defined pool definitions on the node. kube-scheduler does not know which of the pools has CPUs left for new workloads, and may overbook pools on the node.\", userPoolDefs)\n\t}\n\treturn nil\n}\n\n// closestMems returns memory node IDs good for pinning containers\n// that run on given CPUs\nfunc (p *podpools) closestMems(cpus cpuset.CPUSet) idset.IDSet {\n\tmems := idset.NewIDSet()\n\tsys := p.options.System\n\tfor _, nodeID := range sys.NodeIDs() {\n\t\tif !cpus.Intersection(sys.Node(nodeID).CPUSet()).IsEmpty() {\n\t\t\tmems.Add(nodeID)\n\t\t}\n\t}\n\treturn mems\n}\n\n// filterPools returns pools for which the test function returns true\nfunc filterPools(pools []*Pool, test func(*Pool) bool) (ret []*Pool) {\n\tfor _, pool := range pools {\n\t\tif test(pool) {\n\t\t\tret = append(ret, pool)\n\t\t}\n\t}\n\treturn\n}\n\n// parseInstancesCPUs parses the number of pool instances and the\n// number of CPUs per pool instance from PoolDef Instances and CPUs\n// fields.\nfunc parseInstancesCPUs(is string, cs string, freeCpus int) (int, int, error) {\n\tif cs == \"\" {\n\t\treturn 0, 0, podpoolsError(\"missing CPUs\")\n\t}\n\tc64, err := strconv.ParseInt(cs, 0, 32)\n\tif err != nil || c64 <= 0 {\n\t\treturn 0, 0, podpoolsError(\"invalid CPUs per pool: %q, integer > 1 expected\", cs)\n\t}\n\tcpusPerPool := int(c64)\n\t// Supported Instances specifications:\n\t// 0. Instances is an empty string.\n\t//    Create 1 instance.\n\t// 1. Instances: N %\n\t//    Use at most N % of freeCpus for all PoolDef instances.\n\t//    The number of instances is floor(freeCpus * N/100 / cpusPerPool).\n\t// 2. Instances: N CPUs\n\t//    Use at most N CPUs for all PoolDef instances.\n\t//    The number of instances is floor(N / cpusPerPool).\n\t// 3. Instances: N\n\t//    Create N instances from PoolDef.\n\tvar instances int\n\tswitch {\n\tcase is == \"\":\n\t\tinstances = 1\n\tcase strings.HasSuffix(is, \"%\"):\n\t\ttis := strings.TrimSpace(strings.TrimSuffix(is, \"%\"))\n\t\ti64, err := strconv.ParseInt(tis, 0, 32)\n\t\tif err != nil || i64 < 0 {\n\t\t\treturn 0, 0, podpoolsError(\"invalid Instances: %q\", is)\n\t\t}\n\t\tinstances = freeCpus * int(i64) / 100 / cpusPerPool\n\tcase strings.HasSuffix(strings.ToLower(is), \"cpu\"):\n\t\t// All these are equivalent: N(cpu|cpus|CPU|CPUs|CPUS) for any N > 0.\n\t\t// Handling \"CPU\" suffix is an alias for \"CPUs\".\n\t\tis = strings.TrimSpace(strings.TrimSuffix(strings.ToLower(is), \"cpu\")) + \"cpus\"\n\t\tfallthrough\n\tcase strings.HasSuffix(strings.ToLower(is), \"cpus\"):\n\t\ttis := strings.TrimSpace(strings.TrimSuffix(strings.ToLower(is), \"cpus\"))\n\t\ti64, err := strconv.ParseInt(tis, 0, 32)\n\t\tif err != nil || i64 < 0 {\n\t\t\treturn 0, 0, podpoolsError(\"invalid Instances: %q\", is)\n\t\t}\n\t\tif i64 > int64(freeCpus) {\n\t\t\treturn 0, 0, podpoolsError(\"insufficient CPUs: %d required for instances but %d is available\", i64, freeCpus)\n\t\t}\n\t\tinstances = int(i64) / cpusPerPool\n\tdefault:\n\t\ti64, err := strconv.ParseInt(is, 0, 32)\n\t\tif err != nil || i64 < 0 {\n\t\t\treturn 0, 0, podpoolsError(\"invalid Instances: %q\", is)\n\t\t}\n\t\tinstances = int(i64)\n\t}\n\treturn instances, cpusPerPool, nil\n}\n\n// availableMilliCPU returns mCPUs available in a pool.\nfunc (p *podpools) availableMilliCPUs(pool *Pool) int64 {\n\tcpuAvail := int64(pool.CPUs.Size() * 1000)\n\tcpuRequested := int64(0)\n\tfor podID := range pool.PodIDs {\n\t\tcpuRequested += p.getPodMilliCPU(podID)\n\t}\n\treturn cpuAvail - cpuRequested\n}\n\n// assignContainer adds a container to a pool\nfunc (p *podpools) assignContainer(c cache.Container, pool *Pool) {\n\tlog.Info(\"assigning container %s to pool %s\", c.PrettyName(), pool)\n\tpodID := c.GetPodID()\n\tpool.PodIDs[podID] = append(pool.PodIDs[podID], c.GetCacheID())\n\tp.pinCpuMem(c, pool.CPUs, pool.Mems)\n}\n\n// dismissContainer removes a container from a pool\nfunc (p *podpools) dismissContainer(c cache.Container, pool *Pool) {\n\tpodID := c.GetPodID()\n\tpool.PodIDs[podID] = removeString(pool.PodIDs[podID], c.GetCacheID())\n}\n\n// pinCpuMem pins container to CPUs and memory nodes if flagged\nfunc (p *podpools) pinCpuMem(c cache.Container, cpus cpuset.CPUSet, mems idset.IDSet) {\n\tif p.ppoptions.PinCPU {\n\t\tlog.Debug(\"  - pinning to cpuset: %s\", cpus)\n\t\tc.SetCpusetCpus(cpus.String())\n\t\tif reqCpu, ok := c.GetResourceRequirements().Requests[corev1.ResourceCPU]; ok {\n\t\t\tmCpu := int(reqCpu.MilliValue())\n\t\t\tc.SetCPUShares(int64(cache.MilliCPUToShares(int64(mCpu))))\n\t\t}\n\t}\n\tif p.ppoptions.PinMemory {\n\t\tlog.Debug(\"  - pinning to memory %s\", mems)\n\t\tc.SetCpusetMems(mems.String())\n\t}\n}\n\n// podpoolsError formats an error from this policy.\nfunc podpoolsError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n\n// removeString returns the first occurrence of a string from string slice.\nfunc removeString(strings []string, element string) []string {\n\tfor index, s := range strings {\n\t\tif s == element {\n\t\t\tstrings[index] = strings[len(strings)-1]\n\t\t\treturn strings[:len(strings)-1]\n\t\t}\n\t}\n\treturn strings\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, CreatePodpoolsPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/podpools/podpools-policy_test.go",
    "content": "// Copyright 2020-2021 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage podpools\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nfunc validateError(t *testing.T, expectedError string, err error) bool {\n\tif expectedError != \"\" {\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error containing %q, did not get any error\", expectedError)\n\t\t\treturn false\n\t\t} else if !strings.Contains(err.Error(), expectedError) {\n\t\t\tt.Errorf(\"Expected error containing %q, but got %q\", expectedError, err.Error())\n\t\t\treturn false\n\t\t}\n\t} else {\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Unexpected error %s\", err)\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc assertEqualPools(t *testing.T, expectedPool, gotPool Pool) bool {\n\tif expectedPool.String() != gotPool.String() {\n\t\t// Compares Def.Name, Def.Instance, .CPUs, .Mems, Def.MaxPods\n\t\t// and assigned pods/containers.\n\t\tt.Errorf(\"expected pool %s, got %s\", expectedPool, gotPool)\n\t\treturn false\n\t}\n\tif expectedPool.Def.Instances != gotPool.Def.Instances {\n\t\tt.Errorf(\"pools %s: PoolDef.Instances differ: expected %q, got %q\", expectedPool, expectedPool.Def.Instances, gotPool.Def.Instances)\n\t\treturn false\n\t}\n\tif expectedPool.Def.FillOrder != gotPool.Def.FillOrder {\n\t\tt.Errorf(\"pools %s: PoolDef.FillOrder differ: expected %s, got %s\", expectedPool, expectedPool.Def.FillOrder, gotPool.Def.FillOrder)\n\t\treturn false\n\t}\n\treturn true\n}\n\ntype mockCpuAllocator struct{}\n\nfunc (mca *mockCpuAllocator) AllocateCpus(from *cpuset.CPUSet, cnt int, dontcare cpuallocator.CPUPriority) (cpuset.CPUSet, error) {\n\tswitch {\n\tcase from.Size() < cnt:\n\t\treturn cpuset.New(), fmt.Errorf(\"cpuset %s does not have %d CPUs\", from, cnt)\n\tcase from.Size() == cnt:\n\t\tresult := from.Clone()\n\t\t*from = cpuset.New()\n\t\treturn result, nil\n\tdefault:\n\t\tresult := cpuset.New()\n\t\tfor _, cpu := range from.List() {\n\t\t\tif result.Size() >= cnt {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tresult = result.Union(cpuset.New(cpu))\n\t\t}\n\t\t*from = from.Difference(result)\n\t\treturn result, nil\n\t}\n}\n\nfunc (mca *mockCpuAllocator) ReleaseCpus(*cpuset.CPUSet, int, cpuallocator.CPUPriority) (cpuset.CPUSet, error) {\n\treturn cpuset.New(), nil\n}\n\nfunc TestApplyPoolDef(t *testing.T) {\n\treservedCpus1 := cpuset.CPUSet{}\n\treservedPoolDef := PoolDef{\n\t\tName: reservedPoolDefName,\n\t}\n\tdefaultPoolDef := PoolDef{\n\t\tName: defaultPoolDefName,\n\t}\n\treservedPool := Pool{\n\t\tDef:  &reservedPoolDef,\n\t\tCPUs: reservedCpus1,\n\t}\n\tdefaultPool := Pool{\n\t\tDef:  &defaultPoolDef,\n\t\tCPUs: reservedCpus1,\n\t}\n\tnormalPoolsAtStart := []Pool{reservedPool, defaultPool}\n\tsinglecpuSingleInstance := PoolDef{\n\t\tName: \"singlecpu\",\n\t\tCPU:  \"1\",\n\t}\n\tquadcpuDualInstance := PoolDef{\n\t\tName:      \"quadcpu\",\n\t\tCPU:       \"4\",\n\t\tInstances: \"8 CPUs\",\n\t}\n\tquadcpuMultiInstance := PoolDef{\n\t\tName:      \"quadcpu\",\n\t\tCPU:       \"4\",\n\t\tInstances: \"100%\",\n\t}\n\ttcases := []struct {\n\t\tname             string\n\t\tpools            *[]Pool\n\t\tpoolDef          PoolDef\n\t\tfreeCpus         string // example: \"0-2\"\n\t\texpectedFreeCpus string // \"\": no check, \"-\": assert empty\n\t\texpectedError    string // \"\": error is not allowed, otherwise expected error substring\n\t\texpectedPools    *[]Pool\n\t}{\n\t\t// negative tests\n\t\t{\n\t\t\tname:          \"call apply without built-in pools\",\n\t\t\tpools:         &([]Pool{}),\n\t\t\tpoolDef:       singlecpuSingleInstance,\n\t\t\tfreeCpus:      \"0-3\",\n\t\t\texpectedError: \"pools missing\",\n\t\t},\n\t\t{\n\t\t\tname: \"bad reserved CPUs\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName: \"reserved\",\n\t\t\t\tCPU:  \"two\",\n\t\t\t},\n\t\t\texpectedError: \"invalid CPUs\",\n\t\t},\n\t\t{\n\t\t\tname: \"bad reserved Instances\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:      \"reserved\",\n\t\t\t\tCPU:       \"1\",\n\t\t\t\tInstances: \"0x\",\n\t\t\t},\n\t\t\texpectedError: \"invalid Instances\",\n\t\t},\n\t\t{\n\t\t\tname: \"bad default CPUs\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName: \"default\",\n\t\t\t\tCPU:  \"2500m\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-8\",\n\t\t\texpectedError: \"invalid CPUs\",\n\t\t},\n\t\t{\n\t\t\tname: \"bad default Instances\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:      \"default\",\n\t\t\t\tCPU:       \"0xf\",\n\t\t\t\tInstances: \"100 % CPUs\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-95\",\n\t\t\texpectedError: \"invalid Instances\",\n\t\t},\n\t\t{\n\t\t\tname: \"bad user-defined CPUs\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName: \"mypool\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-8\",\n\t\t\texpectedError: \"missing CPUs\",\n\t\t},\n\t\t{\n\t\t\tname: \"too many CPUs on user-defined Instances\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:      \"user pool\",\n\t\t\t\tCPU:       \"1\",\n\t\t\t\tInstances: \"100 CPUs\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-95\",\n\t\t\texpectedError: \"insufficient CPUs\",\n\t\t},\n\t\t{\n\t\t\tname: \"unnamed pool\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tCPU:     \"1\",\n\t\t\t\tMaxPods: 1,\n\t\t\t},\n\t\t\tfreeCpus:      \"0-3\",\n\t\t\texpectedError: \"undefined or empty pool name\",\n\t\t},\n\t\t{\n\t\t\tname: \"unreachable pools\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:      \"unlimited capacity\",\n\t\t\t\tCPU:       \"3\",\n\t\t\t\tMaxPods:   0,\n\t\t\t\tFillOrder: FillPacked,\n\t\t\t\tInstances: \"3\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-95\",\n\t\t\texpectedError: \"2 pool(s) unreachable\",\n\t\t},\n\t\t// redefine the reserved pool\n\t\t{\n\t\t\tname: \"redefine reserved CPUs\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName: \"reserved\",\n\t\t\t\tCPU:  \"2\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-3\",\n\t\t\texpectedError: \"conflicting ReservedResources CPUs\",\n\t\t},\n\t\t{\n\t\t\tname: \"redefine reserved instances\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:      \"reserved\",\n\t\t\t\tCPU:       \"1\",\n\t\t\t\tInstances: \"2\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-3\",\n\t\t\texpectedError: \"cannot change the number of instances\",\n\t\t},\n\t\t{\n\t\t\tname: \"redefine reserved MaxPods\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:    \"reserved\",\n\t\t\t\tMaxPods: 42,\n\t\t\t},\n\t\t\tfreeCpus: \"0-3\",\n\t\t\texpectedPools: &[]Pool{\n\t\t\t\t{\n\t\t\t\t\tDef: &PoolDef{\n\t\t\t\t\t\tName:    reservedPoolDefName,\n\t\t\t\t\t\tMaxPods: 42,\n\t\t\t\t\t},\n\t\t\t\t\tCPUs: reservedPool.CPUs,\n\t\t\t\t},\n\t\t\t\tdefaultPool,\n\t\t\t},\n\t\t},\n\t\t// redefine the default pool\n\t\t{\n\t\t\tname: \"redefine default CPUs\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName: \"default\",\n\t\t\t\tCPU:  \"2\",\n\t\t\t},\n\t\t\tfreeCpus:         \"0-3\",\n\t\t\texpectedFreeCpus: \"2-3\",\n\t\t\texpectedPools: &[]Pool{\n\t\t\t\treservedPool,\n\t\t\t\t{\n\t\t\t\t\tDef: &PoolDef{\n\t\t\t\t\t\tName: defaultPoolDefName,\n\t\t\t\t\t},\n\t\t\t\t\tCPUs: cpuset.MustParse(\"0-1\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"redefine default instances\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:      \"default\",\n\t\t\t\tCPU:       \"1\",\n\t\t\t\tInstances: \"2\",\n\t\t\t},\n\t\t\tfreeCpus:      \"0-3\",\n\t\t\texpectedError: \"cannot change the number of instances\",\n\t\t},\n\t\t{\n\t\t\tname: \"redefine default MaxPods\",\n\t\t\tpoolDef: PoolDef{\n\t\t\t\tName:    \"default\",\n\t\t\t\tMaxPods: 52,\n\t\t\t},\n\t\t\tfreeCpus: \"0-3\",\n\t\t\texpectedPools: &[]Pool{\n\t\t\t\treservedPool,\n\t\t\t\t{\n\t\t\t\t\tDef: &PoolDef{\n\t\t\t\t\t\tName:    defaultPoolDefName,\n\t\t\t\t\t\tMaxPods: 52,\n\t\t\t\t\t},\n\t\t\t\t\tCPUs: defaultPool.CPUs,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t// user-defined pools\n\t\t{\n\t\t\tname:          \"use one CPUs - insufficient\",\n\t\t\tpoolDef:       singlecpuSingleInstance,\n\t\t\texpectedError: \"insufficient CPUs\",\n\t\t},\n\t\t{\n\t\t\tname:             \"use one CPU\",\n\t\t\tfreeCpus:         \"0-3\",\n\t\t\tpoolDef:          singlecpuSingleInstance,\n\t\t\texpectedFreeCpus: \"1-3\",\n\t\t\texpectedPools: &[]Pool{\n\t\t\t\treservedPool,\n\t\t\t\tdefaultPool,\n\t\t\t\t{\n\t\t\t\t\tDef:      &singlecpuSingleInstance,\n\t\t\t\t\tInstance: 0,\n\t\t\t\t\tCPUs:     cpuset.MustParse(\"0\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:             \"use the only CPU\",\n\t\t\tfreeCpus:         \"0\",\n\t\t\tpoolDef:          singlecpuSingleInstance,\n\t\t\texpectedFreeCpus: \"-\",\n\t\t},\n\t\t{\n\t\t\tname:          \"use 2x4 CPUs - insufficient\",\n\t\t\tfreeCpus:      \"0-6\",\n\t\t\tpoolDef:       quadcpuDualInstance,\n\t\t\texpectedError: \"insufficient CPUs\",\n\t\t},\n\t\t{\n\t\t\tname:             \"use 2x4 CPUs - consume all\",\n\t\t\tfreeCpus:         \"0-7\",\n\t\t\tpoolDef:          quadcpuDualInstance,\n\t\t\texpectedFreeCpus: \"-\",\n\t\t},\n\t\t{\n\t\t\tname:             \"use 2x4 CPUs - CPUs left\",\n\t\t\tfreeCpus:         \"0-8\",\n\t\t\tpoolDef:          quadcpuDualInstance,\n\t\t\texpectedFreeCpus: \"8\",\n\t\t},\n\t\t{\n\t\t\tname:          \"use all cpus - but insufficient\",\n\t\t\tfreeCpus:      \"0-2\",\n\t\t\tpoolDef:       quadcpuMultiInstance,\n\t\t\texpectedError: \"insufficient CPUs\",\n\t\t},\n\t\t{\n\t\t\tname:             \"use all cpus - partial\",\n\t\t\tfreeCpus:         \"0-6\",\n\t\t\tpoolDef:          quadcpuMultiInstance,\n\t\t\texpectedFreeCpus: \"4-6\",\n\t\t\texpectedPools: &[]Pool{\n\t\t\t\treservedPool,\n\t\t\t\tdefaultPool,\n\t\t\t\t{\n\t\t\t\t\tDef:      &quadcpuMultiInstance,\n\t\t\t\t\tInstance: 0,\n\t\t\t\t\tCPUs:     cpuset.MustParse(\"0-3\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:             \"use all cpus - every single one\",\n\t\t\tfreeCpus:         \"0-7\",\n\t\t\tpoolDef:          quadcpuMultiInstance,\n\t\t\texpectedFreeCpus: \"-\",\n\t\t\texpectedPools: &[]Pool{\n\t\t\t\treservedPool,\n\t\t\t\tdefaultPool,\n\t\t\t\t{\n\t\t\t\t\tDef:      &quadcpuMultiInstance,\n\t\t\t\t\tInstance: 0,\n\t\t\t\t\tCPUs:     cpuset.MustParse(\"0-3\"),\n\t\t\t\t}, {\n\t\t\t\t\tDef:      &quadcpuMultiInstance,\n\t\t\t\t\tInstance: 1,\n\t\t\t\t\tCPUs:     cpuset.MustParse(\"4-7\"),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t// Tests should not change original pools/pooldefs/freeCpus\n\t\t\t// Create copies before calling the function.\n\t\t\tpools := []*Pool{}\n\t\t\tif tc.pools == nil {\n\t\t\t\ttc.pools = &normalPoolsAtStart\n\t\t\t}\n\t\t\tfor i := range *tc.pools {\n\t\t\t\tcopyOfPool := (*tc.pools)[i]\n\t\t\t\tpools = append(pools, &copyOfPool)\n\t\t\t}\n\t\t\tfreeCpus := cpuset.New()\n\t\t\tif tc.freeCpus != \"\" {\n\t\t\t\tfreeCpus = cpuset.MustParse(tc.freeCpus)\n\t\t\t}\n\t\t\tp := &podpools{\n\t\t\t\tcpuAllocator: &mockCpuAllocator{},\n\t\t\t}\n\t\t\terr := p.applyPoolDef(&pools, &tc.poolDef, &freeCpus, freeCpus.Size())\n\t\t\tif ok := validateError(t, tc.expectedError, err); ok {\n\t\t\t\t// check freeCpus modified by applyPoolDef\n\t\t\t\tif tc.expectedFreeCpus != \"\" {\n\t\t\t\t\texpectedFreeCpus := cpuset.New()\n\t\t\t\t\tif tc.expectedFreeCpus != \"-\" {\n\t\t\t\t\t\texpectedFreeCpus = cpuset.MustParse(tc.expectedFreeCpus)\n\t\t\t\t\t}\n\t\t\t\t\tif expectedFreeCpus.Size() != freeCpus.Size() {\n\t\t\t\t\t\tt.Errorf(\"unexpected number of free CPUs left, expected %d, got %d\", expectedFreeCpus.Size(), freeCpus.Size())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// check pools modified by applyPoolDef\n\t\t\t\tif tc.expectedPools != nil {\n\t\t\t\t\tif len(pools) != len(*tc.expectedPools) {\n\t\t\t\t\t\tt.Errorf(\"unexpected number of new pools, expected %d got %d\", len(pools), len(*tc.expectedPools))\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tfor i := 0; i < len(pools); i++ {\n\t\t\t\t\t\tif !assertEqualPools(t, (*tc.expectedPools)[i], *pools[i]) {\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseInstancesCPUs(t *testing.T) {\n\ttcases := []struct {\n\t\tname              string\n\t\tinstances         string\n\t\tcpus              string\n\t\tfreeCpus          int\n\t\texpectedInstances int\n\t\texpectedCPUs      int\n\t\texpectedError     string\n\t}{\n\t\t{\n\t\t\tname:          \"empty CPUs\",\n\t\t\texpectedError: \"missing CPUs\",\n\t\t},\n\t\t{\n\t\t\tname:          \"bad CPUs\",\n\t\t\tcpus:          \"55%\",\n\t\t\texpectedError: \"> 1 expected\",\n\t\t},\n\t\t{\n\t\t\tname:          \"zero CPUs\",\n\t\t\tcpus:          \"0\",\n\t\t\texpectedError: \"> 1 expected\",\n\t\t},\n\t\t{\n\t\t\tname:          \"negative CPUs\",\n\t\t\tcpus:          \"-1\",\n\t\t\texpectedError: \"> 1 expected\",\n\t\t},\n\t\t{\n\t\t\tname:              \"42 CPUs, empty instances defaults to 1\",\n\t\t\tcpus:              \"42\",\n\t\t\texpectedCPUs:      42,\n\t\t\texpectedInstances: 1,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: 0\",\n\t\t\tinstances:         \"0\",\n\t\t\tcpus:              \"2\",\n\t\t\tfreeCpus:          100,\n\t\t\texpectedInstances: 0,\n\t\t\texpectedCPUs:      2,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: N\",\n\t\t\tinstances:         \"10\",\n\t\t\tcpus:              \"2\",\n\t\t\tfreeCpus:          100,\n\t\t\texpectedInstances: 10,\n\t\t\texpectedCPUs:      2,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: N CPUs\",\n\t\t\tinstances:         \"10 CPUs\",\n\t\t\tcpus:              \"2\",\n\t\t\tfreeCpus:          100,\n\t\t\texpectedInstances: 10 / 2,\n\t\t\texpectedCPUs:      2,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: 1 CPUS\",\n\t\t\tinstances:         \"1 CPUS\",\n\t\t\tcpus:              \"1\",\n\t\t\tfreeCpus:          1,\n\t\t\texpectedInstances: 1,\n\t\t\texpectedCPUs:      1,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: 1 cpu\",\n\t\t\tinstances:         \"1 cpu\",\n\t\t\tcpus:              \"1\",\n\t\t\tfreeCpus:          2,\n\t\t\texpectedInstances: 1,\n\t\t\texpectedCPUs:      1,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: 8cpu\",\n\t\t\tinstances:         \"8cpu\",\n\t\t\tcpus:              \"2\",\n\t\t\tfreeCpus:          9,\n\t\t\texpectedInstances: 4,\n\t\t\texpectedCPUs:      2,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: N %\",\n\t\t\tinstances:         \"90 %\",\n\t\t\tcpus:              \"2\",\n\t\t\tfreeCpus:          10,\n\t\t\texpectedInstances: 4, // 10 * (90/100) / 2\n\t\t\texpectedCPUs:      2,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: N%\",\n\t\t\tinstances:         \"90%\",\n\t\t\tcpus:              \"90\",\n\t\t\tfreeCpus:          100,\n\t\t\texpectedInstances: 1,\n\t\t\texpectedCPUs:      90,\n\t\t},\n\t\t{\n\t\t\tname:              \"instances: N %, not enough for any pools\",\n\t\t\tinstances:         \"10 %\",\n\t\t\tcpus:              \"2\",\n\t\t\tfreeCpus:          10,\n\t\t\texpectedInstances: 0, // 10 * (10/100) / 2\n\t\t\texpectedCPUs:      2,\n\t\t},\n\t\t{\n\t\t\tname:          \"instances: -N\",\n\t\t\tinstances:     \"-10\",\n\t\t\tcpus:          \"2\",\n\t\t\texpectedError: \"invalid Instances\",\n\t\t},\n\t\t{\n\t\t\tname:          \"instances: -N CPUs\",\n\t\t\tinstances:     \"-10 CPUs\",\n\t\t\tcpus:          \"2\",\n\t\t\texpectedError: \"invalid Instances\",\n\t\t},\n\t\t{\n\t\t\tname:          \"instances: N CPUs CPU\",\n\t\t\tinstances:     \"2 CPUs CPU\",\n\t\t\tcpus:          \"2\",\n\t\t\texpectedError: \"invalid Instances\",\n\t\t},\n\t\t{\n\t\t\tname:          \"instances: -N %\",\n\t\t\tinstances:     \"-10 %\",\n\t\t\tcpus:          \"2\",\n\t\t\texpectedError: \"invalid Instances\",\n\t\t},\n\t\t{\n\t\t\tname:          \"instances: N CPUs, N < cpus\",\n\t\t\tinstances:     \"3 CPUs\",\n\t\t\tcpus:          \"4\",\n\t\t\texpectedError: \"insufficient CPUs\",\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tinstances, cpus, err := parseInstancesCPUs(tc.instances, tc.cpus, tc.freeCpus)\n\t\t\tif ok := validateError(t, tc.expectedError, err); ok {\n\t\t\t\tif instances != tc.expectedInstances || cpus != tc.expectedCPUs {\n\t\t\t\t\tt.Errorf(\"Expected (instances, cpus) (%v, %v), but got (%v, %v)\", tc.expectedInstances, tc.expectedCPUs, instances, cpus)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static/flags.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage static\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"sigs.k8s.io/yaml\"\n)\n\n// Options captures our configurable policy parameters.\ntype options struct {\n\t// Relax exclusive isolated CPU allocation criteria\n\tRelaxedIsolation bool `json:\"RelaxedIsolation\"`\n\t// Control whether containers are assigned to RDT classes by this policy.\n\tRdt Tristate `json:\"Rdt\"`\n}\n\n// Tristate is boolean-like value with 3 states: on, off, automatically-determined.\ntype Tristate int\n\nconst (\n\t// TristateOff is unconditional boolean false\n\tTristateOff = iota\n\t// TristateOn is unconditional boolean true\n\tTristateOn\n\t// TristateAuto indicates boolean value should be inferred using other data.\n\tTristateAuto\n)\n\n// Our runtime configuration.\nvar opt = defaultOptions().(*options)\n\n// UnmarshalJSON implements the unmarshaller function for \"encoding/json\"\nfunc (t *Tristate) UnmarshalJSON(data []byte) error {\n\tvar value interface{}\n\tif err := yaml.Unmarshal(data, &value); err != nil {\n\t\treturn policyError(\"invalid Tristate value '%s': %v\", string(data), err)\n\t}\n\n\tswitch value.(type) {\n\tcase bool:\n\t\t*t = map[bool]Tristate{false: TristateOff, true: TristateOn}[value.(bool)]\n\t\treturn nil\n\tcase string:\n\t\tif value.(string) == \"auto\" {\n\t\t\t*t = TristateAuto\n\t\t\treturn nil\n\t\t}\n\t}\n\n\treturn policyError(\"invalid Tristate value %v of type %T\", value, value)\n}\n\n// MarshalJSON implements the marshaller function for \"encoding/json\"\nfunc (t Tristate) MarshalJSON() ([]byte, error) {\n\tswitch t {\n\tcase TristateOff:\n\t\treturn []byte(\"false\"), nil\n\tcase TristateOn:\n\t\treturn []byte(\"true\"), nil\n\tcase TristateAuto:\n\t\treturn []byte(\"\\\"auto\\\"\"), nil\n\t}\n\treturn nil, policyError(\"invalid tristate value %v\", t)\n}\n\n// String returns the value of Tristate as a string\nfunc (t *Tristate) String() string {\n\tswitch *t {\n\tcase TristateOff:\n\t\treturn \"false\"\n\tcase TristateOn:\n\t\treturn \"true\"\n\t}\n\treturn \"auto\"\n}\n\n// defaultOptions returns a new options instance, all initialized to defaults.\nfunc defaultOptions() interface{} {\n\treturn &options{Rdt: TristateAuto}\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tconfig.Register(PolicyPath, PolicyDescription, opt, defaultOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static/static-policy.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage static\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy implementation.\n\tPolicyName = \"static\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"A reimplementation of the static CPU Manager policy.\"\n\t// PolicyPath is the path of this policy in the configuration hierarchy.\n\tPolicyPath = \"policy.\" + PolicyName\n)\n\ntype static struct {\n\tlogger.Logger\n\n\tavailable     policy.ConstraintSet      // resource availability constraints\n\treserved      policy.ConstraintSet      // system/kube-reservation constraints\n\treservedCpus  cpuset.CPUSet             // CPUs reserved for system- and kube-tasks\n\tavailableCpus cpuset.CPUSet             // CPUs free usable by this policy\n\tisolatedCpus  cpuset.CPUSet             // available CPUs isolated from normal scheduling\n\tsys           sysfs.System              // system/topology information\n\tnumHT         int                       // number of hyperthreads per core\n\tstate         cache.Cache               // policy/state cache\n\tcpuAllocator  cpuallocator.CPUAllocator // CPU allocator used by the policy\n}\n\n// Make sure static implements the policy backend interface.\nvar _ policy.Backend = &static{}\n\nconst (\n\t// keyPreferIsolated is the annotation used to mark pods preferring isolated CPUs.\n\tkeyPreferIsolated = \"prefer-isolated-cpus\"\n)\n\n// NewStaticPolicy creates a new policy instance.\nfunc NewStaticPolicy(opts *policy.BackendOptions) policy.Backend {\n\ts := &static{\n\t\tLogger:       logger.NewLogger(PolicyName),\n\t\tstate:        opts.Cache,\n\t\tsys:          opts.System,\n\t\tavailable:    opts.Available,\n\t\treserved:     opts.Reserved,\n\t\tcpuAllocator: cpuallocator.NewCPUAllocator(opts.System),\n\t}\n\n\ts.Info(\"creating policy...\")\n\n\ts.numHT = s.sys.CPU(idset.ID(0)).ThreadCPUSet().Size()\n\n\tif err := s.checkConstraints(); err != nil {\n\t\ts.Fatal(\"cannot start with given constraints: %v\", err)\n\t}\n\n\tconfig.GetModule(PolicyPath).AddNotify(s.configNotify)\n\n\treturn s\n}\n\n// Name returns the name of this policy.\nfunc (s *static) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (s *static) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (s *static) Start(add []cache.Container, del []cache.Container) error {\n\ts.Debug(\"starting up...\")\n\n\tif err := s.allocateReserved(); err != nil {\n\t\treturn policyError(\"failed allocate reserved CPUs: %v\", err)\n\t}\n\n\ts.Info(\"using reserved CPUs: %s\", s.reservedCpus.String())\n\ts.Info(\"using available CPUs: %s\", s.availableCpus.String())\n\n\tif err := s.validateState(s.state); err != nil {\n\t\treturn policyError(\"failed to start with given cache/state: %v\", err)\n\t}\n\n\ts.validateAssignments()\n\n\treturn s.Sync(add, del)\n}\n\n// Sync synchronizes the active policy state.\nfunc (s *static) Sync(add []cache.Container, del []cache.Container) error {\n\ts.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\ts.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\ts.AllocateResources(c)\n\t}\n\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (s *static) AllocateResources(c cache.Container) error {\n\ts.Info(\"allocating resource for container %s...\", c.PrettyName())\n\n\tcontainer := c\n\tcontainerID := c.GetCacheID()\n\tpod, found := c.GetPod()\n\tif !found {\n\t\treturn policyError(\"can't find pod for container %s\", containerID)\n\t}\n\n\terr := s.AddContainer(pod, container, containerID)\n\n\treturn err\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (s *static) ReleaseResources(c cache.Container) error {\n\ts.Info(\"releasing resources of container %s...\", c.PrettyName())\n\n\tcontainerID := c.GetCacheID()\n\terr := s.RemoveContainer(containerID)\n\n\treturn err\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (s *static) UpdateResources(c cache.Container) error {\n\ts.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (s *static) Rebalance() (bool, error) {\n\ts.Debug(\"(not) rebalancing containers...\")\n\treturn false, nil\n}\n\n// HandleEvent handles policy-specific events.\nfunc (s *static) HandleEvent(*events.Policy) (bool, error) {\n\ts.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (s *static) ExportResourceData(c cache.Container) map[string]string {\n\tdata := map[string]string{}\n\n\tif cset, ok := s.GetCPUSet(c.GetCacheID()); !ok {\n\t\tcset = s.GetDefaultCPUSet()\n\t\tdata[policy.ExportSharedCPUs] = cset.String()\n\t} else {\n\t\tisolated := cset.Intersection(s.sys.Isolated()).String()\n\t\tif isolated != \"\" {\n\t\t\tdata[policy.ExportIsolatedCPUs] = isolated\n\t\t}\n\t\texclusive := cset.Difference(s.sys.Isolated()).String()\n\t\tif exclusive != \"\" {\n\t\t\tdata[policy.ExportExclusiveCPUs] = exclusive\n\t\t}\n\t}\n\n\treturn data\n}\n\n// Introspect provides data for external introspection.\nfunc (s *static) Introspect(*introspect.State) {\n\treturn\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *static) DescribeMetrics() []*prometheus.Desc {\n\treturn nil\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *static) PollMetrics() policy.Metrics {\n\treturn nil\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *static) CollectMetrics(policy.Metrics) ([]prometheus.Metric, error) {\n\treturn nil, nil\n}\n\nfunc (s *static) configNotify(event config.Event, source config.Source) error {\n\ts.Info(\"configuration %s\", event)\n\n\tif opt.RelaxedIsolation {\n\t\ts.Info(\"isolated exclusive CPUs: globally preferred (all pods)\")\n\t} else {\n\t\ts.Info(\"isolated exclusive CPUs: per-pod (by annotation '%s')\",\n\t\t\tkubernetes.ResmgrKey(keyPreferIsolated))\n\t}\n\n\ts.Info(\"rdt support set to %v\", opt.Rdt)\n\n\treturn nil\n}\n\n// assignableCPUs returns the set of unassigned CPUs minus the reserved set.\nfunc (s *static) assignableCPUs(numCPUs int) cpuset.CPUSet {\n\tcset := s.GetDefaultCPUSet().Difference(s.reservedCpus)\n\n\tif cset.Size() < numCPUs && s.isolatedCpus.Size() > 0 {\n\t\ts.Warn(\"not enough non-isolated CPUs (%d) left for request (%d)\",\n\t\t\tcset.Size(), numCPUs)\n\t\tcset = cset.Union(s.isolatedCpus)\n\t}\n\n\treturn cset\n}\n\n// AddContainer is the CPU Manager static policy AddContainer function.\nfunc (s *static) AddContainer(pod cache.Pod, container cache.Container, containerID string) error {\n\tif numCPUs := s.guaranteedCPUs(pod, container); numCPUs != 0 {\n\t\ts.Info(\"[cpumanager] static policy: AddContainer (pod: %s, container: %s, container id: %s)\", pod.GetName(), container.GetName(), containerID)\n\t\t// container belongs in an exclusively allocated pool\n\n\t\tif _, ok := s.GetCPUSet(containerID); ok {\n\t\t\ts.Info(\"[cpumanager] static policy: container already present in state, skipping (container: %s, container id: %s)\", container.GetName(), containerID)\n\t\t\treturn nil\n\t\t}\n\n\t\tcpuset, err := s.allocateCPUs(numCPUs, containerID)\n\t\tif err != nil {\n\t\t\ts.Error(\"[cpumanager] unable to allocate %d CPUs (container id: %s, error: %v)\", numCPUs, containerID, err)\n\t\t\treturn err\n\t\t}\n\t\ts.Debug(\"setting cpuset of %s to allocated %s\", containerID, cpuset)\n\t\ts.SetCPUSet(containerID, cpuset)\n\t}\n\t// container belongs in the shared pool (nothing to do; use default cpuset)\n\treturn nil\n}\n\n// RemoveContainer is the CPU Manager static policy RemoveContainer function.\nfunc (s *static) RemoveContainer(containerID string) error {\n\ts.Info(\"[cpumanager] static policy: RemoveContainer (container id: %s)\", containerID)\n\tif toRelease, ok := s.GetCPUSet(containerID); ok {\n\t\ts.Delete(containerID)\n\t\tisolated := toRelease.Intersection(s.sys.Isolated())\n\t\tordinary := toRelease.Difference(isolated)\n\n\t\t// Mutate the shared pool, adding released cpus.\n\t\ts.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(ordinary))\n\t\ts.isolatedCpus = s.isolatedCpus.Union(isolated)\n\t}\n\treturn nil\n}\n\n// Notes:\n//   By default we assume workloads are not isolation-aware. We\n//   only allocate isolated CPUs exclusively to containers if\n//\n//     - we globally prefer isolated exclusive CPUs, or\n//     - the pod prefers isolated exclusive CPUs, or\n//     - the container asks a single hyperthread worth of CPU\n\n// cpuPreference checks if isolated CPUs should be tried and are preferred for an allocation.\nfunc (s *static) cpuPreference(containerID string, numCPUs int) (bool, bool) {\n\tvar try, prefer bool\n\n\t// Check if we prefer isolated CPUs (globally of per this containers pod).\n\tif opt.RelaxedIsolation {\n\t\tprefer = true\n\t} else {\n\t\tif c, ok := s.state.LookupContainer(containerID); ok {\n\t\t\tp, found := c.GetPod()\n\t\t\tif !found {\n\t\t\t\ts.Warn(\"can't find pod for container %s\", c.GetID())\n\t\t\t\treturn false, false\n\t\t\t}\n\n\t\t\tif value, ok := p.GetResmgrAnnotation(keyPreferIsolated); ok {\n\t\t\t\tif isolated, err := strconv.ParseBool(value); isolated {\n\t\t\t\t\tprefer = true\n\t\t\t\t} else {\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\ts.Error(\"invalid annotation '%s' on container %s, expecting boolean: %v\",\n\t\t\t\t\t\t\tkeyPreferIsolated, c.PrettyName(), err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Try isolated cpus when explicitly asked, or, if a single HT of CPU is requested\n\tif prefer || (numCPUs == 1 && s.isolatedCpus.Size() >= 1) {\n\t\ttry = true\n\t}\n\n\treturn try, prefer\n}\n\n// allocateOrdinaryCPUs tries to take a number of non-isolated CPUs.\nfunc (s *static) allocateOrdinaryCPUs(numCPUs int) (cpuset.CPUSet, error) {\n\tassignable := s.assignableCPUs(numCPUs)\n\tresult, err := s.takeByTopology(assignable, numCPUs, cpuallocator.PriorityHigh)\n\n\tif err != nil {\n\t\treturn cpuset.New(), err\n\t}\n\n\ts.Info(\"allocated %d ordinary CPUs: %s\", numCPUs, result.String())\n\n\treturn result, nil\n}\n\n// allocateIsolatedCPUs tries to take a number of isolated CPUs, falling back to ordinary ones.\nfunc (s *static) allocateIsolatedCPUs(numCPUs int, prefer bool) (cpuset.CPUSet, error) {\n\tresult, err := s.takeByTopology(s.isolatedCpus, numCPUs, cpuallocator.PriorityHigh)\n\n\tswitch {\n\tcase err != nil:\n\t\ts.Info(\"falling back to %d ordinary CPUs\", numCPUs)\n\t\treturn s.allocateOrdinaryCPUs(numCPUs)\n\tcase numCPUs == 1 || prefer:\n\t\ts.Info(\"allocated %d isolated CPUs: %s\", numCPUs, result.String())\n\t\treturn result, nil\n\tdefault:\n\t\ts.Info(\"falling back to %d ordinary CPUs\", numCPUs)\n\t\treturn s.allocateOrdinaryCPUs(numCPUs)\n\t}\n}\n\n// allocateCPUs allocates the requested number of CPUs.\nfunc (s *static) allocateCPUs(numCPUs int, containerID string) (cpuset.CPUSet, error) {\n\tvar result cpuset.CPUSet\n\tvar err error\n\n\ts.Info(\"[cpumanager] allocateCpus: (numCPUs: %d)\", numCPUs)\n\n\tif try, prefer := s.cpuPreference(containerID, numCPUs); !try {\n\t\tresult, err = s.allocateOrdinaryCPUs(numCPUs)\n\t} else {\n\t\tresult, err = s.allocateIsolatedCPUs(numCPUs, prefer)\n\t}\n\n\tif err != nil {\n\t\treturn result, err\n\t}\n\n\t// Remove allocated CPUs from the shared and/or isolated CPUSet.\n\ts.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result))\n\ts.isolatedCpus = s.isolatedCpus.Difference(result)\n\n\ts.Info(\"[cpumanager] allocateCPUs: returning \\\"%v\\\"\", result)\n\treturn result, nil\n}\n\nfunc (s *static) guaranteedCPUs(pod cache.Pod, container cache.Container) int {\n\tqos := pod.GetQOSClass()\n\n\ts.Debug(\"* QoS class for pod %s (%s) is %s\", pod.GetID(), pod.GetName(), qos)\n\n\tif qos != corev1.PodQOSGuaranteed {\n\t\treturn 0\n\t}\n\tcpuQuantity := container.GetResourceRequirements().Requests[corev1.ResourceCPU]\n\tif cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {\n\t\treturn 0\n\t}\n\t// Safe downcast to do for all systems with < 2.1 billion CPUs.\n\t// Per the language spec, `int` is guaranteed to be at least 32 bits wide.\n\t// https://golang.org/ref/spec#Numeric_types\n\treturn int(cpuQuantity.Value())\n}\n\n// Check our allocations constraints.\nfunc (s *static) checkConstraints() error {\n\tonline := s.sys.CPUSet().Difference(s.sys.Offlined())\n\tisolated := s.sys.Isolated().Intersection(online)\n\tonline = online.Difference(isolated)\n\n\tcpus, ok := s.available[policy.DomainCPU]\n\tif !ok {\n\t\ts.availableCpus = online\n\t} else {\n\t\tswitch cpus.(type) {\n\t\tcase cpuset.CPUSet:\n\t\t\ts.availableCpus = cpus.(cpuset.CPUSet).Intersection(online)\n\t\tdefault:\n\t\t\treturn policyError(\"invalid type for available CPU set: %T\", cpus)\n\t\t}\n\t}\n\n\ts.isolatedCpus = isolated\n\ts.Info(\"system isolated CPUs: %s\", s.isolatedCpus)\n\n\treturn nil\n}\n\n// Allocate the requested reserved cpus.\nfunc (s *static) allocateReserved() error {\n\tvar err error\n\tvar reserved cpuset.CPUSet\n\n\tcpus, ok := s.reserved[policy.DomainCPU]\n\tif !ok {\n\t\treturn policyError(\"static policy cannot start without reserved CPUs\")\n\t}\n\n\tswitch cpus.(type) {\n\tcase cpuset.CPUSet:\n\t\treserved = cpus.(cpuset.CPUSet)\n\t\tif !reserved.Intersection(s.availableCpus).Equals(reserved) {\n\t\t\treturn policyError(\"some reserved CPUs (%s) are unavailable\",\n\t\t\t\treserved.Difference(s.availableCpus).String())\n\t\t}\n\tcase resource.Quantity:\n\t\tqty := cpus.(resource.Quantity)\n\t\tcount := (int(qty.MilliValue()) + 999) / 1000\n\t\tfrom := s.availableCpus.Clone()\n\t\tif reserved, err = s.takeByTopology(from, count, cpuallocator.PriorityNormal); err != nil {\n\t\t\treturn policyError(\"failed to reserve %d CPUs: %v\", cpus.(int), err)\n\t\t}\n\t}\n\n\ts.reservedCpus = reserved\n\n\treturn nil\n}\n\n// Validate the cache/state supplied for starting.\nfunc (s *static) validateState(state cache.Cache) error {\n\ts.state = state\n\n\ttmpAssignments := s.GetCPUAssignments()\n\ttmpDefaultCPUset := s.GetDefaultCPUSet()\n\tallCPUs := s.availableCpus.Clone()\n\tisolated := s.isolatedCpus.Clone()\n\n\t// Default cpuset cannot be empty when assignments exist\n\tif tmpDefaultCPUset.IsEmpty() {\n\t\tif len(tmpAssignments) != 0 {\n\t\t\treturn fmt.Errorf(\"default cpuset cannot be empty\")\n\t\t}\n\n\t\t// state is empty initialize\n\t\ts.SetDefaultCPUSet(allCPUs)\n\n\t\treturn nil\n\t}\n\n\t// State has already been initialized from file (is not empty)\n\t// 1. Check if the reserved cpuset is not part of default cpuset because:\n\t// - kube/system reserved have changed (increased) - may lead to some containers not being able to start\n\t// - user tampered with file\n\tif !s.reservedCpus.Intersection(tmpDefaultCPUset).Equals(s.reservedCpus) {\n\t\treturn fmt.Errorf(\"not all reserved cpus: \\\"%s\\\" are present in defaultCpuSet: \\\"%s\\\"\",\n\t\t\ts.reservedCpus.String(), tmpDefaultCPUset.String())\n\t}\n\n\t// 2. Check if state for static policy is consistent\n\tfor cID, cset := range tmpAssignments {\n\t\t// None of the cpu in DEFAULT cset should be in s.assignments\n\t\tif !tmpDefaultCPUset.Intersection(cset).IsEmpty() {\n\t\t\treturn fmt.Errorf(\"container id: %s cpuset: \\\"%s\\\" overlaps with default cpuset \\\"%s\\\"\",\n\t\t\t\tcID, cset.String(), tmpDefaultCPUset.String())\n\t\t}\n\n\t\t// Remove any potentially taken isolated CPUs from the available isolated set.\n\t\ts.isolatedCpus = s.isolatedCpus.Difference(cset)\n\t}\n\n\ts.Info(\"available (unallocated) isolated CPUs: %s\", s.isolatedCpus)\n\n\t// 3. It's possible that the set of available CPUs has changed since\n\t// the state was written. This can be due to for example\n\t// offlining a CPU when kubelet is not running. If this happens,\n\t// CPU manager will run into trouble when later it tries to\n\t// assign non-existent CPUs to containers. Validate that the\n\t// topology that was received during CPU manager startup matches with\n\t// the set of CPUs stored in the state.\n\ttotalKnownCPUs := tmpDefaultCPUset.Clone()\n\n\tfor _, cset := range tmpAssignments {\n\t\ttotalKnownCPUs = totalKnownCPUs.Union(cset)\n\t}\n\tif !totalKnownCPUs.Equals(allCPUs) {\n\t\tif totalKnownCPUs.IsSubsetOf(allCPUs.Union(isolated)) {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"current available CPUs \\\"%s\\\" are not a superset of CPUs in state \\\"%s\\\"\",\n\t\t\tallCPUs.Union(isolated).String(), totalKnownCPUs.String())\n\t}\n\n\treturn nil\n}\n\n// Topology-aware-like allocation wrapper.\nfunc (s *static) takeByTopology(available cpuset.CPUSet, numCPUs int, preferredPrio cpuallocator.CPUPriority) (cpuset.CPUSet, error) {\n\tfrom := &available\n\tcset, err := s.cpuAllocator.AllocateCpus(from, numCPUs, preferredPrio)\n\tif err != nil {\n\t\treturn cset, err\n\t}\n\n\treturn cset, err\n}\n\n// Validate static assignments, purge stale ones.\nfunc (s *static) validateAssignments() {\n\t// Instead of relying/waiting for an external reconcilation loop to\n\t// clean up stale container/assignments, we do it ourselves upon startup.\n\n\tca := s.GetCPUAssignments()\n\tfor id, cset := range ca {\n\t\tif _, ok := s.state.LookupContainer(id); !ok {\n\t\t\ts.Info(\"Removing stale assignment of container %s (cpus %s)\",\n\t\t\t\tid, cset.String())\n\t\t\ts.RemoveContainer(id)\n\t\t}\n\t}\n}\n\n// policyError creates a policy-specific formatted error\nfunc policyError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n\n//\n// Kubelet CPU Manager / policy_static.go adaptation\n//\n// A set of rudimentary functions to get policy_static.go up and running\n// with small enough changes that the code (above) remains recognisable\n// for those who are already familiar with the original. These functions\n// basically implements a CPU Manager state-like interface on top of our\n// cache.\n\n// ContainerCPUAssignments assigns CPU sets per container id.\ntype ContainerCPUAssignments map[string]cpuset.CPUSet\n\n//\n// Cache keys for storing the default cpuset (one for containers\n// without exclusive allocations) and static assignments (cpusets\n// for containers with exclusive allocations).\n\nconst (\n\tkeyAssignments = \"CPUAssignments\"\n\tkeyDefaultCPUs = \"DefaultCPUSet\"\n)\n\n// GetCPUAssignments gets the current CPU assignments from our state.\nfunc (s *static) GetCPUAssignments() ContainerCPUAssignments {\n\tvar ca map[string]cpuset.CPUSet\n\n\tif !s.state.GetPolicyEntry(keyAssignments, &ca) {\n\t\ts.Error(\"no cached CPU assignments\")\n\t}\n\n\tif ca == nil {\n\t\tca = make(map[string]cpuset.CPUSet)\n\t\ts.state.SetPolicyEntry(keyAssignments, ca)\n\t}\n\n\treturn ca\n}\n\n// SetCPUAssginments sets the current CPU assignments in our state.\nfunc (s *static) SetCPUAssignments(ca ContainerCPUAssignments) {\n\ts.state.SetPolicyEntry(keyAssignments, map[string]cpuset.CPUSet(ca))\n}\n\n// GetDefaultCPUSet gets the current default CPUSet from our state.\nfunc (s *static) GetDefaultCPUSet() cpuset.CPUSet {\n\tvar cset cpuset.CPUSet\n\n\tif !s.state.GetPolicyEntry(keyDefaultCPUs, &cset) {\n\t\ts.Error(\"no cached default CPU set\")\n\t}\n\n\treturn cset\n}\n\n// SetDefaultCPUSet sets the current default CPUSet in our state.\nfunc (s *static) SetDefaultCPUSet(cset cpuset.CPUSet) {\n\ts.state.SetPolicyEntry(keyDefaultCPUs, cset)\n\n\t// update cpuset for containers with default assignment\n\tca := s.GetCPUAssignments()\n\tfor _, id := range s.state.GetContainerCacheIds() {\n\t\tif _, ok := ca[id]; !ok {\n\t\t\ts.SetCpusetCpus(id, cset.String())\n\t\t}\n\t}\n}\n\n// GetCPUSet gets the CPUSet for a container from our state.\nfunc (s *static) GetCPUSet(containerID string) (cpuset.CPUSet, bool) {\n\tca := s.GetCPUAssignments()\n\tcset, ok := ca[containerID]\n\n\treturn cset.Clone(), ok\n}\n\n// SetCPUSet sets the CPUSet for a container in our state.\nfunc (s *static) SetCPUSet(containerID string, cset cpuset.CPUSet) {\n\tca := s.GetCPUAssignments()\n\tca[containerID] = cset\n\n\ts.SetCPUAssignments(ca)\n\ts.SetCpusetCpus(containerID, cset.String())\n}\n\n// Delete deletes the given container from our state.\nfunc (s *static) Delete(containerID string) {\n\ts.Debug(\"deleting container %s from assignments\", containerID)\n\n\tca := s.GetCPUAssignments()\n\tdelete(ca, containerID)\n\n\ts.SetCPUAssignments(ca)\n}\n\n// SetCPUSetCpus updates cpuset.cpus for a container.\nfunc (s *static) SetCpusetCpus(id, value string) error {\n\tc, ok := s.state.LookupContainer(id)\n\tif !ok {\n\t\treturn policyError(\"can't find container '%s'\", id)\n\t}\n\n\tc.SetCpusetCpus(value)\n\ts.Info(\"container %s: CpusetCpus set to %s\", c.PrettyName(), value)\n\n\treturn nil\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, NewStaticPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static-plus/static-plus-policy.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage staticplus\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strconv\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy implementation.\n\tPolicyName = \"static-plus\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"A simple policy supporting exclusive/pinned and shared allocations.\"\n\t// Cache key for storing container resource allocations.\n\tkeyAllocations = \"allocations\"\n\t// Cache key for storing the shared pool.\n\tkeySharedPool = \"shared-pool\"\n\t// keyPreferIsolated is the annotation used to mark pods preferring isolated CPUs.\n\tkeyPreferIsolated = \"prefer-isolated-cpus\"\n)\n\n// Assignment tracks resource assignments for a single container.\ntype Assignment struct {\n\texclusive cpuset.CPUSet // exclusively allocated cpus\n\tshared    int           // milli-cpus to allocated from shared cpus\n}\n\n// Allocations track all resources allocations by the static+ policy.\ntype Allocations map[string]*Assignment\n\n// static-plus policy runtime state.\ntype staticplus struct {\n\tlogger.Logger\n\toffline      cpuset.CPUSet             // offlined cpus\n\tavailable    cpuset.CPUSet             // bounding set of cpus available for us\n\treserved     cpuset.CPUSet             // pool (primarily) for system-/kube-tasks\n\tisolated     cpuset.CPUSet             // primary pool for exclusive allocations\n\tallocations  Allocations               // container cpu allocations\n\tsys          sysfs.System              // system/topologu information\n\tcache        cache.Cache               // system state/cache\n\tshared       cpuset.CPUSet             // pool for fractional and shared allocations\n\tcpuAllocator cpuallocator.CPUAllocator // CPU allocator used by the policy\n}\n\n// Make sure staticplus implements the policy backend interface.\nvar _ policy.Backend = &staticplus{}\n\n// CreateStaticPlusPolicy creates a new policy instance.\nfunc CreateStaticPlusPolicy(opts *policy.BackendOptions) policy.Backend {\n\tp := &staticplus{\n\t\tLogger:       logger.NewLogger(PolicyName),\n\t\tcache:        opts.Cache,\n\t\tsys:          opts.System,\n\t\tcpuAllocator: cpuallocator.NewCPUAllocator(opts.System),\n\t}\n\n\tp.Info(\"creating policy...\")\n\n\tif err := p.setupPools(opts.Available, opts.Reserved); err != nil {\n\t\tp.Fatal(\"failed to set up cpu pools: %v\", err)\n\t}\n\n\tp.dumpPools()\n\n\treturn p\n}\n\n// Name returns the name of this policy.\nfunc (p *staticplus) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (p *staticplus) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (p *staticplus) Start(add []cache.Container, del []cache.Container) error {\n\tif err := p.restoreCache(); err != nil {\n\t\treturn policyError(\"failed to start: %v\", err)\n\t}\n\n\tif err := p.updatePools(); err != nil {\n\t\treturn policyError(\"failed to start: %v\", err)\n\t}\n\n\treturn p.Sync(add, del)\n}\n\n// Sync synchronizes the state ofd this policy.\nfunc (p *staticplus) Sync(add []cache.Container, del []cache.Container) error {\n\tp.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\tp.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\tp.AllocateResources(c)\n\t}\n\n\treturn nil\n}\n\n// AllocateResources allocates resources for the given container.\nfunc (p *staticplus) AllocateResources(c cache.Container) error {\n\tvar a *Assignment\n\n\tid := c.GetCacheID()\n\n\tp.Debug(\"allocating container %s...\", id)\n\n\tif _, ok := p.allocations[id]; ok {\n\t\treturn nil\n\t}\n\n\ta, err := p.assignCpus(c)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn p.addAssignment(c, a)\n}\n\n// ReleaseResources release resources assigned to the given container.\nfunc (p *staticplus) ReleaseResources(c cache.Container) error {\n\tid := c.GetCacheID()\n\n\tp.Debug(\"releasing container %s...\", id)\n\n\ta, ok := p.allocations[id]\n\tif !ok {\n\t\treturn nil\n\t}\n\n\treturn p.delAssignment(a, id)\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (p *staticplus) UpdateResources(c cache.Container) error {\n\tp.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (p *staticplus) Rebalance() (bool, error) {\n\tp.Debug(\"(not) rebalancing containers...\")\n\treturn false, nil\n}\n\n// HandleEvent handles policy-specific events.\nfunc (p *staticplus) HandleEvent(*events.Policy) (bool, error) {\n\tp.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (p *staticplus) ExportResourceData(c cache.Container) map[string]string {\n\ta, ok := p.allocations[c.GetCacheID()]\n\tif !ok {\n\t\t// Hmm...\n\t\tp.Warn(\"can't find allocation for container %s\", c.PrettyName())\n\t\treturn nil\n\t}\n\n\tdata := map[string]string{}\n\n\tif a.shared != 0 {\n\t\tdata[policy.ExportSharedCPUs] = p.shared.String()\n\t}\n\tif a != nil && !a.exclusive.IsEmpty() {\n\t\tisolated := a.exclusive.Intersection(p.sys.Isolated()).String()\n\t\tif isolated != \"\" {\n\t\t\tdata[policy.ExportIsolatedCPUs] = isolated\n\t\t}\n\t\texclusive := a.exclusive.Difference(p.sys.Isolated()).String()\n\t\tif exclusive != \"\" {\n\t\t\tdata[policy.ExportExclusiveCPUs] = exclusive\n\t\t}\n\t}\n\n\treturn data\n}\n\n// Introspect provides data for external introspection.\nfunc (p *staticplus) Introspect(*introspect.State) {\n\treturn\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *staticplus) DescribeMetrics() []*prometheus.Desc {\n\treturn nil\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *staticplus) PollMetrics() policy.Metrics {\n\treturn nil\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *staticplus) CollectMetrics(policy.Metrics) ([]prometheus.Metric, error) {\n\treturn nil, nil\n}\n\n// policyError creates a formatted policy-specific error.\nfunc policyError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n\n// setupPools sets up the pools we allocate resources from.\nfunc (p *staticplus) setupPools(available, reserved policy.ConstraintSet) error {\n\t// Set up three disjoint CPU pools for allocating CPU to containers. These\n\t// three pools are:\n\t//\n\t//   1) reserved pool: kube- and system-tasks\n\t//        Pods in the kube-system namespace are assigned to this pool. The\n\t//        size of this pool is the requested reservation rounded up to the\n\t//        closest integer. Any unused fractional part of this pool is used\n\t//        as a shared pool if the shared pool ever gets fully allocated.\n\t//\n\t//   2) isolated pool: primary exclusive allocations\n\t//        Exclusive CPU allocations are primarily done from this pool. Pods\n\t//        that request at least 1 full CPU get their exclusive (integer)\n\t//        CPU shares allocated from this pool unless the pool has already\n\t//        been exhausted (in which case we try to slice off exclusive CPUs\n\t//        from the shared pool).\n\t//\n\t//   3) shared pool: shared allocations, secondary exclusive allocations\n\t//        Shared CPU allocations are served from this pool. Pods fractional\n\t//        CPU shares are allocated from this pool. If the isolated pool has\n\t//        been exhausted exclusive allocations are sliced off from this\n\t//        pool. If this pool has been fully allocated, shared allocations\n\t//        are oversubscribed to the reserved pool.\n\n\tp.offline = p.sys.Offlined()\n\n\tcpus, ok := available[policy.DomainCPU]\n\tif !ok {\n\t\tp.available = p.sys.CPUSet().Difference(p.offline)\n\t} else {\n\t\tp.available = cpus.(cpuset.CPUSet).Difference(p.offline)\n\t}\n\n\tp.isolated = p.sys.Isolated().Intersection(p.available)\n\tp.available = p.available.Difference(p.isolated)\n\n\tcpus, ok = reserved[policy.DomainCPU]\n\tif !ok {\n\t\treturn policyError(\"cannot start without any reserved CPUs\")\n\t}\n\n\tswitch cpus.(type) {\n\tcase cpuset.CPUSet:\n\t\tp.reserved = cpus.(cpuset.CPUSet).Intersection(p.available)\n\t\tif !p.reserved.Equals(cpus.(cpuset.CPUSet)) {\n\t\t\treturn policyError(\"part of the reserved CPUs (%s) are not available: %s\",\n\t\t\t\tcpus.(cpuset.CPUSet).String(), cpus.(cpuset.CPUSet).Difference(p.available))\n\t\t}\n\t\tp.available = p.available.Difference(p.reserved)\n\n\tcase resource.Quantity:\n\t\tvar err error\n\t\tqty := cpus.(resource.Quantity)\n\t\tcount := (int(qty.MilliValue()) + 999) / 1000\n\t\tif count < 2 && p.available.Contains(0) {\n\t\t\tp.reserved = cpuset.New(0)\n\t\t\tp.available = p.available.Difference(p.reserved)\n\t\t} else {\n\t\t\tp.reserved, err = p.takeCPUs(&p.available, nil, count, cpuallocator.PriorityNormal)\n\t\t\tif err != nil {\n\t\t\t\treturn policyError(\"failed to reserve %d CPUs from %s: %v\",\n\t\t\t\t\tcount, p.available.String())\n\t\t\t}\n\t\t}\n\t}\n\n\tp.shared = p.available\n\n\treturn nil\n}\n\n// Restore saved policy state from the cache.\nfunc (p *staticplus) restoreCache() error {\n\tif !p.cache.GetPolicyEntry(keySharedPool, &p.shared) {\n\t\tp.Warn(\"initializing empty policy state...\")\n\n\t\tp.shared = p.available\n\t\tp.allocations = make(Allocations)\n\t\tp.cache.SetPolicyEntry(keySharedPool, &p.shared)\n\t\tp.cache.SetPolicyEntry(keyAllocations,\n\t\t\tcache.Cachable(&cachedAllocations{a: p.allocations}))\n\t} else {\n\t\tp.Info(\"restoring cached policy state...\")\n\n\t\tca := cachedAllocations{}\n\t\tif !p.cache.GetPolicyEntry(keyAllocations, &ca) {\n\t\t\treturn policyError(\"failed to restore state from cache, no allocations\")\n\t\t}\n\t\tp.allocations = ca.a\n\t}\n\n\tp.dumpPools()\n\tp.dumpAllocations()\n\n\treturn nil\n}\n\n// requestedCpus calculates the exclusive and shared cpu allocations for a container.\nfunc (p *staticplus) requestedCpus(c cache.Container) (int, int) {\n\tcpuReq, ok := c.GetResourceRequirements().Requests[corev1.ResourceCPU]\n\tif !ok {\n\t\treturn 0, 0\n\t}\n\n\tfull := int(cpuReq.MilliValue()) / 1000\n\tpart := int(cpuReq.MilliValue()) - 1000*full\n\n\treturn full, part\n}\n\n// optOutFromIsolated checks if a container prefers (to opt out from) isolated CPUs.\nfunc (p *staticplus) optOutFromIsolation(c cache.Container) bool {\n\tpreferIsolated := true\n\n\tif pod, found := c.GetPod(); !found {\n\t\tp.Warn(\"can't find pod for container %s\", c.PrettyName())\n\t} else {\n\t\tif value, ok := pod.GetResmgrAnnotation(keyPreferIsolated); ok {\n\t\t\tif isolated, err := strconv.ParseBool(value); !isolated {\n\t\t\t\tif err != nil {\n\t\t\t\t\tp.Error(\"invalid annotation '%s' on container %s, expecting boolean: %v\",\n\t\t\t\t\t\tkeyPreferIsolated, c.PrettyName(), err)\n\t\t\t\t} else {\n\t\t\t\t\tp.Info(\"container %s is opted-out from isolation\", c.PrettyName())\n\t\t\t\t}\n\t\t\t\tpreferIsolated = false\n\t\t\t} else {\n\t\t\t\tp.Info(\"container %s explicitly opted-in for isolation\", c.PrettyName())\n\t\t\t}\n\t\t} else {\n\t\t\tp.Info(\"container %s goes with default isolation\", c.PrettyName())\n\t\t}\n\t}\n\n\treturn !preferIsolated\n}\n\n// assignCpus allocates cpus for a containers.\nfunc (p *staticplus) assignCpus(c cache.Container) (*Assignment, error) {\n\tfull, part := p.requestedCpus(c)\n\n\t// system containers always share (the reserved) cpus\n\tif c.GetNamespace() == metav1.NamespaceSystem {\n\t\treturn &Assignment{shared: 1000*full + part}, nil\n\t}\n\n\t// assign to the shared pool if less than a single cpu was requested\n\tif full == 0 {\n\t\treturn &Assignment{shared: part}, nil\n\t}\n\n\t// if there is capacity in the isolated pool, slice cpus off from it\n\tif p.isolated.Size() >= full && !p.optOutFromIsolation(c) {\n\t\tcpus, err := p.takeCPUs(&p.isolated, nil, full, cpuallocator.PriorityHigh)\n\t\tif err != nil {\n\t\t\treturn nil, policyError(\"failed to allocate %d isolated CPUs: %v\",\n\t\t\t\tfull, err)\n\t\t}\n\t\treturn &Assignment{exclusive: cpus, shared: part}, nil\n\t}\n\n\t// otherwise, try to slice off cpus from the shared pool\n\tif p.shared.Size() >= full {\n\t\tcpus, err := p.takeCPUs(&p.shared, nil, full, cpuallocator.PriorityHigh)\n\t\tif err != nil {\n\t\t\treturn nil, policyError(\"failed to allocate %d exclusive CPUs: %v\",\n\t\t\t\tfull, err)\n\t\t}\n\t\treturn &Assignment{exclusive: cpus, shared: part}, nil\n\t}\n\n\t// we're screwed, not enough cpu in either isolated or shared pool\n\treturn nil, policyError(\"failed to allocate %d exclusive CPUs: %s\",\n\t\tfull, \"not enough capacity\")\n}\n\n// addAssignment updates container allocations for a newly added container assignment.\nfunc (p *staticplus) addAssignment(c cache.Container, a *Assignment) error {\n\tswitch {\n\t// always assign system containers to the reserved pool\n\tcase c.GetNamespace() == metav1.NamespaceSystem:\n\t\tc.SetCpusetCpus(p.reserved.String())\n\t\tc.SetCPUShares(int64(MilliCPUToShares(a.shared)))\n\n\t\tp.Info(\"system container %s allocated (%d mCPU) to reserved pool %s\",\n\t\t\tc.PrettyName(), a.shared, p.reserved.String())\n\n\t\t// for shared-only assignments, it's enough to update the container\n\tcase a.exclusive.IsEmpty():\n\t\tc.SetCpusetCpus(p.shared.String())\n\t\tc.SetCPUShares(int64(MilliCPUToShares(a.shared)))\n\n\t\tp.Info(\"container %s allocated (%d mCPU) to shared pool %s\",\n\t\t\tc.PrettyName(), a.shared, p.shared.String())\n\n\t\t// isolated, sliced-off exclusive, or mixed allocation\n\tdefault:\n\t\tvar kind string\n\t\tvar isolated bool\n\t\tif isolated = !a.exclusive.Intersection(p.sys.Isolated()).IsEmpty(); isolated {\n\t\t\tkind = \"isolated\"\n\t\t} else {\n\t\t\tkind = \"exclusive\"\n\t\t}\n\t\tif a.shared != 0 {\n\t\t\tc.SetCpusetCpus(a.exclusive.Union(p.shared).String())\n\t\t\tc.SetCPUShares(int64(MilliCPUToShares(a.shared)))\n\t\t\tp.Info(\"container %s allocated to %s (%s) and shared (%d mCPU) pool %s\",\n\t\t\t\tc.PrettyName(), kind, a.exclusive.String(), a.shared, p.shared.String())\n\t\t} else {\n\t\t\tc.SetCpusetCpus(a.exclusive.String())\n\t\t\tc.SetCPUShares(int64(MilliCPUToShares(1000 * a.exclusive.Size())))\n\t\t\tp.Info(\"container %s allocated to %s CPUs %s\", c.PrettyName(),\n\t\t\t\tkind, a.exclusive.String())\n\t\t}\n\n\t\t// for sliced-off exclusive we might need to update other containers shared allocations\n\t\tif !a.exclusive.IsEmpty() && a.exclusive.Intersection(p.sys.Isolated()).IsEmpty() {\n\t\t\tif err := p.updateSharedAllocations(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tp.allocations[c.GetCacheID()] = a\n\n\tp.cache.SetPolicyEntry(keySharedPool, p.shared)\n\tp.cache.SetPolicyEntry(keyAllocations,\n\t\tcache.Cachable(&cachedAllocations{a: p.allocations}))\n\n\treturn nil\n}\n\n// delAssignment updates container allocations for a deleted container assignment.\nfunc (p *staticplus) delAssignment(a *Assignment, id string) error {\n\tdelete(p.allocations, id)\n\n\tswitch {\n\t// for shared-only allocations there is not much to do...\n\tcase a.exclusive.IsEmpty():\n\t\tp.Info(\"freed shared-only (%d mCPU) allocations of container %s\",\n\t\t\ta.shared, id)\n\n\t\t// for isolated exclusive cpus, return them to the pool\n\tcase !a.exclusive.Intersection(p.sys.Isolated()).IsEmpty():\n\t\tp.isolated = p.isolated.Union(a.exclusive)\n\n\t\tp.Info(\"freed isolated allocations (%s) of container %s\",\n\t\t\ta.exclusive.String(), id)\n\n\t\t// for cpus sliced off the shared pool, return then and update others\n\tdefault:\n\t\tp.shared = p.shared.Union(a.exclusive)\n\n\t\tp.Info(\"freed exclusive allocations (%s) of container %s\",\n\t\t\ta.exclusive.String(), id)\n\n\t\tif err := p.updateSharedAllocations(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tp.cache.SetPolicyEntry(keySharedPool, p.shared)\n\tp.cache.SetPolicyEntry(keyAllocations,\n\t\tcache.Cachable(&cachedAllocations{a: p.allocations}))\n\n\treturn nil\n}\n\n// updateSharedAllocations updates containers with shared allocations.\nfunc (p *staticplus) updateSharedAllocations() error {\n\tavail := 1000 * p.shared.Size()\n\n\tfor id, ca := range p.allocations {\n\t\tcac, ok := p.cache.LookupContainer(id)\n\t\tif !ok {\n\t\t\tp.Warn(\"can't find allocated container %s\", id)\n\t\t\t// remove and recalculate shared CPUs\n\t\t\tp.delAssignment(ca, id)\n\t\t\treturn p.updateSharedAllocations()\n\t\t}\n\n\t\tif !ca.exclusive.Intersection(p.sys.Isolated()).IsEmpty() && ca.shared == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tcset := p.shared.Union(ca.exclusive)\n\n\t\tif avail <= 0 {\n\t\t\tcset = cset.Union(p.reserved)\n\t\t\tp.Warn(\"out of free shared (%s) capacity, using reserved pool (%s) as well\",\n\t\t\t\tp.shared.String(), p.reserved.String())\n\t\t}\n\n\t\tif cac.GetCpusetCpus() != cset.String() {\n\t\t\tcac.SetCpusetCpus(cset.String())\n\n\t\t\tp.Info(\"container %s reallocated to exclusive (%s) and shared (%d mCPU) pool %s\",\n\t\t\t\tcac.PrettyName(), ca.exclusive.String(), ca.shared, cset.String())\n\t\t}\n\n\t\tavail -= ca.shared\n\t}\n\n\tif avail < 0 {\n\t\tp.Warn(\"not enough free capacity in shared pool (%s): lacking %d mCPU\",\n\t\t\tp.shared.String(), -avail)\n\t} else {\n\t\tp.Info(\"free shared (%s) capacity left: %d mCPU\", p.shared.String(), avail)\n\t}\n\n\treturn nil\n}\n\n// updatePools updates the pools according to the current asignments.\nfunc (p *staticplus) updatePools() error {\n\tfor id, ca := range p.allocations {\n\t\tif ca.exclusive.IsEmpty() {\n\t\t\tcontinue\n\t\t}\n\n\t\tisolated := ca.exclusive.Intersection(p.sys.Isolated())\n\t\texcshare := ca.exclusive.Difference(isolated)\n\n\t\tif !isolated.IsEmpty() && !excshare.IsEmpty() {\n\t\t\treturn policyError(\"container %s has exclusive isolated (%s) and shareable (%s) cpus\",\n\t\t\t\tid, isolated.String(), excshare.String())\n\t\t}\n\n\t\tp.isolated = p.isolated.Difference(isolated)\n\t\tp.shared = p.shared.Difference(excshare)\n\t}\n\n\tif err := p.updateSharedAllocations(); err != nil {\n\t\treturn err\n\t}\n\n\tp.cache.SetPolicyEntry(keySharedPool, p.shared)\n\tp.cache.SetPolicyEntry(keyAllocations,\n\t\tcache.Cachable(&cachedAllocations{a: p.allocations}))\n\n\treturn nil\n}\n\n// dumpPools dumps the current state of pools.\nfunc (p *staticplus) dumpPools() {\n\tp.Info(\"current CPU pools:\")\n\toffline := p.offline.String()\n\tif offline == \"\" {\n\t\toffline = \"<none>\"\n\t}\n\tisolated := p.isolated.String()\n\tif isolated == \"\" {\n\t\tisolated = \"<none>\"\n\t}\n\n\tp.Info(\"  offline:  %s\", offline)\n\tp.Info(\"  reserved: %s\", p.reserved.String())\n\tp.Info(\"  shared:   %s\", p.shared.String())\n\tp.Info(\"  isolated: %s\", isolated)\n\n}\n\n// dumpAllocations dumps the current allocations.\nfunc (p *staticplus) dumpAllocations() {\n\tp.Info(\"container CPU allocations:\")\n\tswitch {\n\tcase p.allocations == nil:\n\t\tp.Info(\"  <nil>\")\n\tcase len(p.allocations) == 0:\n\t\tp.Info(\"  <none>\")\n\tdefault:\n\t\tfor id, ca := range p.allocations {\n\t\t\te := ca.exclusive.String()\n\t\t\tif e == \"\" {\n\t\t\t\te = \"<none>\"\n\t\t\t}\n\t\t\tp.Info(\"  %s: exclusive: %s, shared: %d milli-cpu\", id, e, ca.shared)\n\t\t}\n\t}\n}\n\n// Take up to cnt CPUs from a given CPU set to another.\nfunc (p *staticplus) takeCPUs(from, to *cpuset.CPUSet, cnt int, preferredPrio cpuallocator.CPUPriority) (cpuset.CPUSet, error) {\n\tcset, err := p.cpuAllocator.AllocateCpus(from, cnt, preferredPrio)\n\tif err != nil {\n\t\treturn cset, err\n\t}\n\n\tif to != nil {\n\t\t*to = to.Union(cset)\n\t}\n\n\treturn cset, err\n}\n\n//\n// Cachable data types for storing private static-plus policy data in the cache.\n//\n\n// CachedAllocations implements Cache.Cachable boilerplate for Allocations.\ntype CachedAllocations interface {\n\tcache.Cachable\n}\n\ntype cachedAllocations struct {\n\ta Allocations\n}\n\nvar _ cache.Cachable = &cachedAllocations{}\n\nvar _ json.Marshaler = &cachedAllocations{}\nvar _ json.Unmarshaler = &cachedAllocations{}\n\nfunc (ca *cachedAllocations) Get() interface{} {\n\treturn *ca\n}\n\nfunc (ca *cachedAllocations) Set(value interface{}) {\n\tswitch value.(type) {\n\tcase cachedAllocations:\n\t\tca.a = value.(cachedAllocations).a\n\tcase *cachedAllocations:\n\t\tca.a = value.(*cachedAllocations).a\n\t}\n}\n\ntype marshallableAssignment struct {\n\tExclusive string\n\tShared    int\n}\n\nfunc (ca *cachedAllocations) MarshalJSON() ([]byte, error) {\n\tdst := make(map[string]*marshallableAssignment)\n\tfor id, r := range ca.a {\n\t\tdst[id] = &marshallableAssignment{\n\t\t\tExclusive: r.exclusive.String(),\n\t\t\tShared:    r.shared,\n\t\t}\n\t}\n\n\treturn json.Marshal(dst)\n}\n\nfunc (ca *cachedAllocations) UnmarshalJSON(data []byte) error {\n\tvar err error\n\n\tdst := make(map[string]*marshallableAssignment)\n\tif err = json.Unmarshal(data, &dst); err != nil {\n\t\treturn err\n\t}\n\n\tca.a = make(map[string]*Assignment)\n\tfor id, r := range dst {\n\t\tif r == nil {\n\t\t\tcontinue\n\t\t}\n\t\tcset, err := cpuset.Parse(r.Exclusive)\n\t\tif err != nil {\n\t\t\treturn policyError(\"failed to unmarshal cpuset '%s': %v\",\n\t\t\t\tr.Exclusive, err)\n\t\t}\n\t\tca.a[id] = &Assignment{\n\t\t\texclusive: cset,\n\t\t\tshared:    r.Shared,\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Functions for calculating CFS cpu.shares and cpu.cfs_quota_us.\n//\n//\tNotes: These functions are almost verbatim taken from the kubelet\n//\tcode (from k8s.io/kubernetes/pkg/kubelet/cm/helpers_linux.go).\n//\tSince these are exported there, we could try to import them, set the\n//\trelated feature gates (kubefeatures.CPUCFSQuotaPeriod) for ourselves\n//\tinto the desired positions (disabled most probably for now) and use\n//\tthe imported code.\nconst (\n\tMinShares     = 2\n\tSharesPerCPU  = 1024\n\tMilliCPUToCPU = 1000\n\n\t// 100000 is equivalent to 100ms\n\tQuotaPeriod    = 100000\n\tMinQuotaPeriod = 1000\n)\n\n// MilliCPUToQuota converts milliCPU to CFS quota and period values.\nfunc MilliCPUToQuota(milliCPU int64, period int64) (quota int64) {\n\t// CFS quota is measured in two values:\n\t//  - cfs_period_us=100ms (the amount of time to measure usage across given by period)\n\t//  - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)\n\t// so in the above example, you are limited to 20% of a single CPU\n\t// for multi-cpu environments, you just scale equivalent amounts\n\t// see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details\n\n\tif milliCPU == 0 {\n\t\treturn\n\t}\n\n\tif true /*!utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod)*/ {\n\t\tperiod = QuotaPeriod\n\t}\n\n\t// we then convert your milliCPU to a value normalized over a period\n\tquota = (milliCPU * period) / MilliCPUToCPU\n\n\t// quota needs to be a minimum of 1ms.\n\tif quota < MinQuotaPeriod {\n\t\tquota = MinQuotaPeriod\n\t}\n\treturn\n}\n\n// MilliCPUToShares converts the milliCPU to CFS shares.\nfunc MilliCPUToShares(milliCPU int) int64 {\n\tif milliCPU == 0 {\n\t\t// Docker converts zero milliCPU to unset, which maps to kernel default\n\t\t// for unset: 1024. Return 2 here to really match kernel default for\n\t\t// zero milliCPU.\n\t\treturn MinShares\n\t}\n\t// Conceptually (milliCPU / milliCPUToCPU) * sharesPerCPU, but factored to improve rounding.\n\tshares := (milliCPU * SharesPerCPU) / MilliCPUToCPU\n\tif shares < MinShares {\n\t\treturn MinShares\n\t}\n\treturn int64(shares)\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, CreateStaticPlusPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static-pools/config.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage stp\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"sigs.k8s.io/yaml\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// config captures our runtime configurable parameters.\ntype config struct {\n\t// Pools defines our set of pools in use.\n\tPools pools `json:\"pools,omitempty\"`\n\t// ConfDirPath is the filesystem path to the legacy configuration directry structure.\n\tConfDirPath string\n\t// ConfFilePath is the filesystem path to the legacy configuration file.\n\tConfFilePath string\n\t// LabelNode controls whether backwards-compatible CMK node label is created.\n\tLabelNode bool\n\t// TaintNode controls whether backwards-compatible CMK node taint is created.\n\tTaintNode bool\n}\n\ntype pools map[string]poolConfig\n\ntype cpuList struct {\n\tSocket     uint64\n\tCpuset     string // TODO: might want to use cpuset from kubelet\n\tcontainers map[string]struct{}\n}\n\n// STP policy runtime configuration with their defaults.\nvar conf = defaultConfig().(*config)\n\n// defaultConfig returns a new conf instance, all initialized to defaults.\nfunc defaultConfig() interface{} {\n\treturn &config{\n\t\tPools:       make(pools),\n\t\tConfDirPath: \"/etc/cmk\",\n\t}\n}\n\nfunc (c *cpuList) addContainer(id string) {\n\tif c.containers == nil {\n\t\tc.containers = make(map[string]struct{})\n\t}\n\tc.containers[id] = struct{}{}\n}\n\nfunc (c *cpuList) removeContainer(id string) {\n\tif c.containers == nil {\n\t\treturn\n\t}\n\tdelete(c.containers, id)\n}\n\nfunc (c *cpuList) getContainers() []string {\n\tif c.containers == nil {\n\t\treturn []string{}\n\t}\n\n\tret := make([]string, len(c.containers))\n\ti := 0\n\tfor k := range c.containers {\n\t\tret[i] = k\n\t\ti++\n\t}\n\treturn ret\n}\n\ntype poolConfig struct {\n\tExclusive bool `json:\"exclusive\"`\n\t// Per-socket cpu lists\n\tCPULists []*cpuList `json:\"cpuLists\"`\n}\n\nfunc (p *poolConfig) cpuSet() string {\n\tcpuset := \"\"\n\tdelim := \"\"\n\tfor _, cl := range p.CPULists {\n\t\tcpuset += delim + cl.Cpuset\n\t\tdelim = \",\"\n\t}\n\treturn cpuset\n}\n\nvar (\n\tcpusetValidationRe = regexp.MustCompile(`^(([\\d]+)|([\\d]+-[\\d]+))(,(([\\d]+)|([\\d]+-[\\d]+)))*$`)\n)\n\nfunc parseConfData(raw []byte) (pools, error) {\n\tconf := &struct {\n\t\tPools pools\n\t}{}\n\n\terr := yaml.Unmarshal(raw, &conf)\n\tif err != nil {\n\t\treturn nil, stpError(\"Failed to parse config file: %v\", err)\n\t}\n\treturn conf.Pools, nil\n}\n\nfunc readConfFile(filepath string) (pools, error) {\n\t// Read config data\n\tdata, err := os.ReadFile(filepath)\n\tif err != nil {\n\t\treturn nil, stpError(\"Failed to read config file: %v\", err)\n\t}\n\n\treturn parseConfData(data)\n}\n\nfunc readConfDir(confDir string) (pools, error) {\n\tconf := pools{}\n\n\t// List pools in the pools configuration directory\n\tpoolsDir := path.Join(confDir, \"pools\")\n\tpools, err := os.ReadDir(poolsDir)\n\tif err != nil {\n\t\treturn nil, stpError(\"Failed to list pools config directory %s: %v\", poolsDir, err)\n\t}\n\n\t// Read pool configurations\n\tfor _, pool := range pools {\n\t\tpoolConf, err := readPoolConfDir(path.Join(poolsDir, pool.Name()))\n\t\tif err != nil {\n\t\t\treturn nil, stpError(\"Failed to read pool çonfiguration: %v\", err)\n\t\t}\n\t\tconf[pool.Name()] = poolConf\n\t}\n\n\treturn conf, nil\n}\n\n// Read configuration of one pool from original CMK configuration directory tree\nfunc readPoolConfDir(poolDir string) (poolConfig, error) {\n\tconf := poolConfig{Exclusive: false, CPULists: []*cpuList{}}\n\n\t// Read pool's exclusivity flag\n\texclusive, err := os.ReadFile(path.Join(poolDir, \"exclusive\"))\n\tif err != nil {\n\t\treturn conf, fmt.Errorf(\"Failed to read pool exclusive setting in %s: %v\", poolDir, err)\n\t}\n\tif len(exclusive) == 1 && exclusive[0] == '1' {\n\t\tconf.Exclusive = true\n\t}\n\n\t// Read socket configurations (per-socket cpu lists)\n\tfiles, err := os.ReadDir(poolDir)\n\tif err != nil {\n\t\treturn conf, fmt.Errorf(\"Failed to list pool config directory %s: %v\", poolDir, err)\n\t}\n\tfor _, file := range files {\n\t\tif !file.IsDir() {\n\t\t\t// Skip non-directory files (e.g. 'exclusive' file)\n\t\t\tcontinue\n\t\t}\n\n\t\tsocketPath := path.Join(poolDir, file.Name())\n\t\tsocketCPULists, err := readSocketConfDir(socketPath)\n\t\tif err != nil {\n\t\t\treturn conf, fmt.Errorf(\"Failed to list pool socket config: %s\", err)\n\t\t}\n\n\t\tconf.CPULists = append(conf.CPULists, socketCPULists...)\n\t}\n\treturn conf, nil\n}\n\n// Read configuration (cpu lists) of a socket of one pool in original CMK\n// configuration directory tree\nfunc readSocketConfDir(socketDir string) ([]*cpuList, error) {\n\t// Get socket number from the name of the directory\n\tsocketNum, err := strconv.ParseUint(path.Base(socketDir), 10, 32)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Invalid socket id %s: %v\", socketDir, err)\n\t}\n\n\t// Socket directory contains a set of subdirectories, one per cpu list\n\tcpuListDirs, err := os.ReadDir(socketDir)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"Failed to list socket directory %s: %v\", socketDir, err)\n\t}\n\n\tconf := make([]*cpuList, len(cpuListDirs))\n\n\tfor i, cpuListDir := range cpuListDirs {\n\t\t// Validate that the cpulist conforms to cpuset formatting\n\t\tif err := validateCPUList(cpuListDir.Name()); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"Invalid cpu list in %s: %v\", socketDir, err)\n\t\t}\n\t\tconf[i] = &cpuList{Socket: socketNum,\n\t\t\tCpuset:     cpuListDir.Name(),\n\t\t\tcontainers: map[string]struct{}{}}\n\t}\n\treturn conf, nil\n}\n\nfunc validateCPUList(name string) error {\n\t// NOTE: Naive implementation, we only check that it \"looks right\", we don't\n\t// check that the actual numbers make sense, i.e. that numbers are in\n\t// ascending order\n\tif !cpusetValidationRe.MatchString(name) {\n\t\treturn fmt.Errorf(\"%q does not look like a cpuset\", name)\n\t}\n\treturn nil\n}\n\n// Convert cpu list configuration directory name into a cpuList\nfunc parseCPUListName(name string) ([]uint, error) {\n\t// The name should be a list of cpu ids (positive integers) separated by commas\n\tcpuListMembers := strings.Split(name, \",\")\n\n\tcpus := make([]uint, len(cpuListMembers))\n\n\t// Convert cpu ids to a list of integers\n\tfor i, cpuStr := range cpuListMembers {\n\t\tcpu, err := strconv.ParseUint(cpuStr, 10, 32)\n\t\tif err != nil {\n\t\t\treturn cpus, fmt.Errorf(\"Invalid cpu id in %s: %v\", name, err)\n\t\t}\n\t\tcpus[i] = uint(cpu)\n\t}\n\treturn cpus, nil\n}\n\n// Register us for command line option processing and configuration management.\nfunc init() {\n\tpkgcfg.Register(PolicyPath, PolicyDescription, conf, defaultConfig)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static-pools/node.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage stp\n\nimport (\n\t\"strconv\"\n\t\"time\"\n\n\tcore_v1 \"k8s.io/api/core/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/agent\"\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\texclusiveCoreResourceName = \"cmk.intel.com/exclusive-cores\"\n\tcmkLegacyNodeLabelName    = \"cmk.intel.com/cmk-node\"\n)\n\ntype nodeUpdater struct {\n\tlog.Logger\n\tagent agent.Interface\n\tconf  chan config\n}\n\nfunc newNodeUpdater(agent agent.Interface) *nodeUpdater {\n\treturn &nodeUpdater{\n\t\tLogger: log.NewLogger(\"static-pools-nu\"),\n\t\tagent:  agent,\n\t\tconf:   make(chan config, 1),\n\t}\n}\n\nfunc (u *nodeUpdater) start() error {\n\tu.Info(\"starting node updater\")\n\n\tif u.agent == nil || u.agent.IsDisabled() {\n\t\treturn stpError(\"cri-resmgr-agent connection required\")\n\t}\n\n\tgo func() {\n\t\tvar pending *config\n\t\tvar retry <-chan time.Time\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase c := <-u.conf:\n\t\t\t\tpending = &c\n\t\t\t\tretry = time.After(0)\n\t\t\tcase _ = <-retry:\n\t\t\t\tif pending != nil {\n\t\t\t\t\terr := u.updateNode(pending, -1)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tu.Info(\"node update failed: %v\", err)\n\t\t\t\t\t\tretry = time.After(5 * time.Second)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tu.Info(\"node successfully updated\")\n\t\t\t\t\t\tpending = nil\n\t\t\t\t\t\tretry = nil\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tu.Panic(\"BUG: node update with nil config requested\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn nil\n}\n\nfunc (u *nodeUpdater) update(c config) {\n\t// Pop possibly pending value from the channel\n\tselect {\n\tcase <-u.conf:\n\tdefault:\n\t}\n\tu.conf <- c\n}\n\n// Update Node object with STP/CMK-specific things\nfunc (u *nodeUpdater) updateNode(conf *config, opTimeout time.Duration) error {\n\t// Count total number of cpu lists of all exclusive pools\n\tnumExclusiveCPULists := 0\n\tfor _, pool := range conf.Pools {\n\t\tif pool.Exclusive {\n\t\t\tnumExclusiveCPULists += len(pool.CPULists)\n\t\t}\n\t}\n\n\t// Update extended resources\n\tresources := map[string]string{\n\t\texclusiveCoreResourceName: strconv.Itoa(numExclusiveCPULists)}\n\tu.Info(\"updating node capacity (extended resources)\")\n\tif err := u.agent.UpdateNodeCapacity(resources, opTimeout); err != nil {\n\t\treturn err\n\t}\n\n\t// Manage legacy node label\n\tif conf.LabelNode {\n\t\tu.Info(\"creating CMK node label\")\n\t\terr := u.agent.SetLabels(map[string]string{cmkLegacyNodeLabelName: \"true\"}, opTimeout)\n\t\tif err != nil {\n\t\t\treturn stpError(\"failed to update legacy node label: %v\", err)\n\t\t}\n\t} else {\n\t\tu.Info(\"removing CMK node label\")\n\t\terr := u.agent.RemoveLabels([]string{cmkLegacyNodeLabelName}, opTimeout)\n\t\tif err != nil {\n\t\t\treturn stpError(\"failed to update legacy node label: %v\", err)\n\t\t}\n\t}\n\n\t// Manage legacy node taint\n\tnodeTaints, err := u.agent.GetTaints(opTimeout)\n\tif err != nil {\n\t\treturn stpError(\"failed to fetch node taints: %v\", err)\n\t}\n\n\tlegacyTaint := core_v1.Taint{\n\t\tKey:    \"cmk\",\n\t\tValue:  \"true\",\n\t\tEffect: core_v1.TaintEffectNoSchedule,\n\t}\n\tcmkTaints := []core_v1.Taint{legacyTaint}\n\t_, tainted := u.agent.FindTaintIndex(nodeTaints, &legacyTaint)\n\n\tif !tainted && conf.TaintNode {\n\t\tu.Info(\"creating CMK node taint\")\n\t\tif err := u.agent.SetTaints(cmkTaints, opTimeout); err != nil {\n\t\t\treturn stpError(\"failed to set legacy node taint: %v\", err)\n\t\t}\n\t}\n\tif tainted && !conf.TaintNode {\n\t\tu.Debug(\"removing CMK node taint\")\n\t\tif err := u.agent.RemoveTaints(cmkTaints, opTimeout); err != nil {\n\t\t\treturn stpError(\"failed to clear legacy node taint: %v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static-pools/stp-policy.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage stp\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"strings\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy implementation.\n\tPolicyName = \"static-pools\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"A reimplementation of CMK (CPU Manager for Kubernetes).\"\n\t// PolicyPath is the path of this policy in the configuration hierarchy.\n\tPolicyPath = \"policy.\" + PolicyName\n\t// StpEnvPool is the name of the env variable for selecting STP pool of a container\n\tStpEnvPool = \"STP_POOL\"\n\t// StpEnvSocketID is the name of the env variable for selecting cpu socket of a container\n\tStpEnvSocketID = \"STP_SOCKET_ID\"\n\t// StpEnvNoAffinity is the name of the env variable for switching off cpuset enforcement\n\tStpEnvNoAffinity = \"STP_NO_AFFINITY\"\n\t// CmkEnvAssigned is the name of the env variable that the original CMK\n\t// sets to communicate the selected cpuset to the workload. We use the same\n\t// environment variable for compatibility.\n\tCmkEnvAssigned = \"CMK_CPUS_ASSIGNED\"\n\t// CmkEnvInfra is the name of the env variable that the original CMK sets\n\t// to communicate all CPUs of the infra pool to the workload. We use the\n\t// same environment variable for compatibility.\n\tCmkEnvInfra = \"CMK_CPUS_INFRA\"\n\t// CmkEnvShared is the name of the env variable that the original CMK sets\n\t// to communicate all CPUs of the shared pool to the workload. We use the\n\t// same environment variable for compatibility.\n\tCmkEnvShared = \"CMK_CPUS_SHARED\"\n\t// CmkEnvNumCores is the name of the env used in the original CMK to select\n\t// the number of exclusive CPUs, deprecated here\n\tCmkEnvNumCores = \"CMK_NUM_CORES\"\n\t// PoolInfra is the hardcoded name of the 'infra' pool\n\tCmkPoolInfra = \"infra\"\n\t// PoolInfra is the hardcoded name of the 'infra' pool\n\tCmkPoolShared = \"shared\"\n)\n\ntype stp struct {\n\tlogger.Logger\n\n\tconf        *config      // STP policy configuration\n\tnodeUpdater *nodeUpdater // node updater thread\n\tstate       cache.Cache  // state cache\n}\n\nvar _ policy.Backend = &stp{}\n\n//\n// Policy backend implementation\n//\n\n// CreateStpPolicy creates a new policy instance.\nfunc CreateStpPolicy(opts *policy.BackendOptions) policy.Backend {\n\tstp := &stp{\n\t\tLogger:      logger.NewLogger(PolicyName),\n\t\tstate:       opts.Cache,\n\t\tnodeUpdater: newNodeUpdater(opts.AgentCli),\n\t}\n\n\tstp.Info(\"creating policy...\")\n\n\tpkgcfg.GetModule(PolicyPath).AddNotify(stp.configNotify)\n\n\treturn stp\n}\n\n// Name returns the name of this policy.\nfunc (stp *stp) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (stp *stp) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (stp *stp) Start(add []cache.Container, del []cache.Container) error {\n\tif err := stp.nodeUpdater.start(); err != nil {\n\t\treturn err\n\t}\n\n\tif stp.conf == nil {\n\t\tif err := stp.setConfig(conf); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif err := stp.initializeState(); err != nil {\n\t\treturn err\n\t}\n\tstp.Debug(\"retrieved stp container states from cache:\\n%s\", utils.DumpJSON(*stp.getContainerRegistry()))\n\n\tif err := stp.Sync(add, del); err != nil {\n\t\treturn err\n\t}\n\n\tstp.Debug(\"preparing for making decisions...\")\n\n\treturn nil\n}\n\n// Sync synchronizes the state of this policy.\nfunc (stp *stp) Sync(add []cache.Container, del []cache.Container) error {\n\tstp.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\tstp.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\tstp.AllocateResources(c)\n\t}\n\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (stp *stp) AllocateResources(c cache.Container) error {\n\tcontainerID := c.GetCacheID()\n\tstp.Debug(\"allocating resources for container %s...\", containerID)\n\n\tcs := stpContainerStatus{Socket: -1}\n\n\t// Default pool name\n\tpoolName := CmkPoolShared\n\n\t// Get resource requests\n\tstp.Debug(\"RESOURCE REQUESTS: %s\", c.GetResourceRequirements().Requests)\n\trequestedCPUs, ok := c.GetResourceRequirements().Requests[exclusiveCoreResourceName]\n\tif ok {\n\t\tnCPUs, _ := requestedCPUs.AsInt64()\n\t\tcs.NExclusiveCPUs = nCPUs\n\t}\n\n\t// Parse container command line. Backwards compatibility for old CMK\n\t// workloads\n\tcmkArgs := stp.parseContainerCmdline(c.GetCommand(), c.GetArgs())\n\tif cmkArgs != nil {\n\t\tpoolName = cmkArgs.Pool\n\t\tcs.Socket = cmkArgs.SocketID\n\t\tcs.NoAffinity = cmkArgs.NoAffinity\n\n\t\t// Overwrite container commandline\n\t\tc.SetCommand(cmkArgs.Command)\n\t\tc.SetArgs([]string{})\n\n\t\tstp.Debug(\"parsed options from container command line: %v\", cmkArgs)\n\t}\n\n\t// Get STP options from container env\n\tenvVal, ok := c.GetEnv(StpEnvSocketID)\n\tif ok {\n\t\tsocketID, err := strconv.ParseInt(envVal, 10, 32)\n\t\tif err != nil {\n\t\t\tstp.Warn(\"unable to parse socket id from %q: %v\", StpEnvSocketID, err)\n\t\t} else {\n\t\t\tcs.Socket = socketID\n\t\t}\n\t}\n\tenvVal, ok = c.GetEnv(StpEnvPool)\n\tif ok {\n\t\tpoolName = envVal\n\t}\n\t_, ok = c.GetEnv(StpEnvNoAffinity)\n\tif ok {\n\t\t// We do not care about the value of the env variable here\n\t\tcs.NoAffinity = true\n\t}\n\n\t// Force socket to -1 if pool is not \"socket aware\"\n\tif poolName == CmkPoolInfra {\n\t\tcs.Socket = -1\n\t}\n\n\t// Get pool configuration\n\tif _, ok := stp.conf.Pools[poolName]; !ok {\n\t\treturn stpError(\"non-existent pool %q\", poolName)\n\t}\n\tcs.Pool = poolName\n\n\t// Allocate (CPU) resources for the container\n\terr := stp.allocateStpResources(c, cs)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (stp *stp) ReleaseResources(c cache.Container) error {\n\tstp.Debug(\"releasing resources of container %s...\", c.PrettyName())\n\tstp.releaseStpResources(c.GetCacheID())\n\treturn nil\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (stp *stp) UpdateResources(c cache.Container) error {\n\tstp.Debug(\"updating resource allocations of container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (stp *stp) Rebalance() (bool, error) {\n\tstp.Debug(\"(not) rebalancing containers...\")\n\treturn false, nil\n}\n\n// HandleEvent handles policy-specific events.\nfunc (stp *stp) HandleEvent(*events.Policy) (bool, error) {\n\tstp.Debug(\"(not) handling event...\")\n\treturn false, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (stp *stp) ExportResourceData(c cache.Container) map[string]string {\n\treturn nil\n}\n\n// Introspect provides data for external introspection.\nfunc (stp *stp) Introspect(*introspect.State) {\n\treturn\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *stp) DescribeMetrics() []*prometheus.Desc {\n\treturn nil\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *stp) PollMetrics() policy.Metrics {\n\treturn nil\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *stp) CollectMetrics(policy.Metrics) ([]prometheus.Metric, error) {\n\treturn nil, nil\n}\n\nfunc (stp *stp) configNotify(event pkgcfg.Event, source pkgcfg.Source) error {\n\tstp.Info(\"configuration %s\", event)\n\n\tif err := stp.setConfig(conf); err != nil {\n\t\treturn err\n\t}\n\n\tstp.Info(\"config updated successfully\")\n\n\treturn nil\n}\n\nfunc (stp *stp) setConfig(cfg *config) error {\n\t// Read legacy pools configuration if the given config has no pools configured\n\tif cfg.Pools == nil || len(cfg.Pools) == 0 {\n\t\tif len(cfg.ConfDirPath) > 0 {\n\t\t\tstp.Debug(\"Reading legacy configuration directory tree %q\", cfg.ConfDirPath)\n\t\t\tp, err := readConfDir(cfg.ConfDirPath)\n\t\t\tif err != nil {\n\t\t\t\tstp.Warn(\"failed to read configuration directory: %v\", err)\n\t\t\t} else {\n\t\t\t\tcfg.Pools = p\n\t\t\t}\n\t\t}\n\t\tif len(cfg.ConfFilePath) > 0 {\n\t\t\tstp.Debug(\"Reading legacy configuration file %q\", cfg.ConfFilePath)\n\t\t\tp, err := readConfFile(cfg.ConfFilePath)\n\t\t\tif err != nil {\n\t\t\t\tstp.Warn(\"failed to read configuration file: %v\", err)\n\t\t\t} else {\n\t\t\t\tif cfg.Pools != nil || len(cfg.Pools) > 0 {\n\t\t\t\t\tstp.Info(\"Overriding pool configuration from %q with configuration from %q\",\n\t\t\t\t\t\tcfg.ConfDirPath, cfg.ConfFilePath)\n\t\t\t\t}\n\t\t\t\tcfg.Pools = p\n\t\t\t}\n\t\t}\n\t}\n\n\tif err := stp.verifyConfig(cfg); err != nil {\n\t\treturn err\n\t}\n\n\tstp.conf = cfg\n\tstp.Debug(\"policy configuration:\\n%s\", utils.DumpJSON(stp.conf))\n\n\tstp.nodeUpdater.update(*stp.conf)\n\n\treturn nil\n}\n\n//\n// Helper functions for STP policy backend\n//\n\nfunc stpError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n\nfunc (stp *stp) initializeState() error {\n\tccr := stp.getContainerRegistry()\n\n\tfor id := range *ccr {\n\t\t// Remove orphaned containers\n\t\tif _, ok := stp.state.LookupContainer(id); !ok {\n\t\t\tstp.Info(\"removing orphaned container %s from policy cache\", id)\n\t\t\tstp.releaseStpResources(id)\n\t\t}\n\t}\n\n\treturn stp.verifyConfig(stp.conf)\n}\n\n// Verify configuration against the existing set of containers\nfunc (stp *stp) verifyConfig(cfg *config) error {\n\t//  Sanity check for config\n\tif cfg == nil || cfg.Pools == nil || len(cfg.Pools) == 0 {\n\t\treturn stpError(\"invalid config, no pools configured\")\n\t}\n\n\t// Loop through all existing containers\n\tccr := stp.getContainerRegistry()\n\tfor id, cs := range *ccr {\n\t\t// Check that pool for container exists\n\t\tpool, ok := cfg.Pools[cs.Pool]\n\t\tif !ok {\n\t\t\treturn stpError(\"invalid stp configuration: pool %q for container %q not found\", cs.Pool, id)\n\t\t}\n\n\t\t// Check that pool exclusivity is compatible with container configuration\n\t\tif pool.Exclusive && cs.NExclusiveCPUs < 1 {\n\t\t\treturn stpError(\"invalid stp configuration: container %q with no exclusive CPUs set to run in exclusive pool %q\", id, cs.Pool)\n\t\t} else if !pool.Exclusive && cs.NExclusiveCPUs > 0 {\n\t\t\treturn stpError(\"invalid stp configuration: container %q with exclusive CPUs set to run in non-exclusive pool %q\", id, cs.Pool)\n\t\t}\n\n\t\t// Check that cpu lists (cpuset) of container can be satisfied by the pool\n\t\t// NOTE: we do not try to do any migration to possibly free cpu lists\n\t\t// if the originally allocated cpu lists are not available\n\t\t// TODO: for non-exclusive pools it might be feasible just to alter the\n\t\t// cpuset (i.e. reconcile new cpu list using the existing pool/socket\n\t\t// spec for container) in case cpu lists do not match exactly\n\t\tfor _, cCpuset := range cs.Cpusets {\n\t\t\tfor i, pClist := range pool.CPULists {\n\t\t\t\tif cCpuset == pClist.Cpuset {\n\t\t\t\t\tpool.CPULists[i].addContainer(id)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif i == len(pool.CPULists)-1 {\n\t\t\t\t\treturn stpError(\"invalid stp configuration: cpu list %q configured for container %q not found in pool %q\", cCpuset, id, cs.Pool)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\ntype cmkLegacyArgs struct {\n\tPool       string\n\tSocketID   int64\n\tCommand    []string\n\tNoAffinity bool\n}\n\n// parseContainerCmdline tries to parse the pool name and socket id parameters\n// from container command line\nfunc (stp *stp) parseContainerCmdline(cmd, args []string) *cmkLegacyArgs {\n\t// NOTE: This is naive implementation and not foolproof. E.g. args could be\n\t// defined throught env variables\n\tcmdLine := append(cmd, args...)\n\tstp.Debug(\"Parsing container command line %v\\n\", cmdLine)\n\n\tcmkArgs := parseCmkCmdline(cmdLine)\n\n\t// If we didn't find cmk arguments, try to parse each argument separately\n\t// in case cmk was invoked like 'bash -c \"cmk isolate ...\"\n\t// NOTE: We do somewhat naive strings.Fields() here, there is room for\n\t// improvement by usage go-shellquote or similar\n\tif cmkArgs == nil {\n\t\tfor _, arg := range cmdLine {\n\t\t\tcmkArgs = parseCmkCmdline(strings.Fields(arg))\n\t\t\tif cmkArgs != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn cmkArgs\n}\n\nfunc parseCmkCmdline(args []string) *cmkLegacyArgs {\n\tparsedArgs := cmkLegacyArgs{}\n\n\t// Create parser\n\tcmkCmd := flag.NewFlagSet(\"cmk-legacy\", flag.ContinueOnError)\n\tcmkCmd.SetOutput(io.Discard)\n\tcmkCmd.StringVar(&parsedArgs.Pool, \"pool\", \"\", \"pool to use\")\n\tcmkCmd.Int64Var(&parsedArgs.SocketID, \"socket-id\", -1, \"socket id to use\")\n\tcmkCmd.BoolVar(&parsedArgs.NoAffinity, \"no-affinity\", false, \"Do not set cpu affinity before forking the child command\")\n\t// Args that we're not really interested in\n\t_ = cmkCmd.String(\"conf-dir\", \"\", \"CMK configuration directory\")\n\n\tif len(args) > 1 && args[0] == \"cmk\" && args[1] == \"isolate\" {\n\t\terr := cmkCmd.Parse(args[2:])\n\t\t// Parse out (i.e. ignore) all unknown args\n\t\tfor err != nil {\n\t\t\terr = cmkCmd.Parse(cmkCmd.Args())\n\t\t}\n\t\t// Pool needs to be defined\n\t\tif parsedArgs.Pool != \"\" {\n\t\t\tparsedArgs.Command = cmkCmd.Args()\n\t\t\treturn &parsedArgs\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (stp *stp) allocateStpResources(c cache.Container, cs stpContainerStatus) error {\n\tvar CPULists [](*cpuList)\n\n\t// Get pool configuration for this container\n\tpool, ok := stp.conf.Pools[cs.Pool]\n\tif !ok {\n\t\treturn stpError(\"BUG: pool %q not found\", cs.Pool)\n\t}\n\n\tavailableCPULists := getAvailableCPULists(cs.Socket, &pool)\n\n\tif pool.Exclusive {\n\t\tif cs.NExclusiveCPUs < 1 {\n\t\t\treturn stpError(\"exclusive pool specified but the number of exclusive CPUs requested is 0\")\n\t\t}\n\n\t\t// Check the possible deprecated CMK_NUM_CORES setting. Print a warning\n\t\t// if this does not match what was requested through extended resources\n\t\tenvNumCores, ok := c.GetEnv(CmkEnvNumCores)\n\t\tif ok {\n\t\t\tiNumCores, err := strconv.ParseInt(envNumCores, 10, 64)\n\t\t\tif err != nil || iNumCores != cs.NExclusiveCPUs {\n\t\t\t\tstp.Warn(\"Ignoring deprecated env variable setting, %s=%q does \"+\n\t\t\t\t\t\"not match the number of cores (%d) from resource request\",\n\t\t\t\t\tCmkEnvNumCores, envNumCores, cs.NExclusiveCPUs)\n\t\t\t}\n\t\t}\n\n\t\tif int64(len(availableCPULists)) < cs.NExclusiveCPUs {\n\t\t\tif cs.Socket < 0 {\n\t\t\t\treturn stpError(\"not enough free cpu lists in pool %q\", cs.Pool)\n\t\t\t}\n\t\t\treturn stpError(\"not enough free cpu lists in pool %q with socket id %d\", cs.Pool, cs.Socket)\n\t\t}\n\n\t\tCPULists = availableCPULists[0:cs.NExclusiveCPUs]\n\n\t} else {\n\t\t/* NOTE (from CMK): This allocation algorithm is probably an\n\t\toversimplification, however for known use cases the non-exclusive\n\t\tpools should never have more than one cpu list anyhow.\n\t\tIf that ceases to hold in the future, we could explore population\n\t\tor load-based spreading. Keeping it simple for now. */\n\t\tif len(availableCPULists) == 0 {\n\t\t\treturn stpError(\"no available cpu lists in pool %q with socket id %d\", cs.Pool, cs.Socket)\n\t\t}\n\n\t\ti := rand.Int31n(int32((len(availableCPULists))))\n\t\tCPULists = availableCPULists[i : i+1]\n\t}\n\n\tcontainerID := c.GetCacheID()\n\tcpuset := \"\"\n\tsep := \"\"\n\tfor _, cl := range CPULists {\n\t\tcl.addContainer(containerID)\n\t\tcpuset += sep + cl.Cpuset\n\t\tsep = \",\"\n\t\tcs.Cpusets = append(cs.Cpusets, cpuset)\n\t}\n\n\t// Commit our changes\n\tcontainers := stp.getContainerRegistry()\n\t(*containers)[containerID] = cs\n\tstp.setContainerRegistry(containers)\n\n\tif cs.NoAffinity {\n\t\tstp.Info(\"not setting cpuset for container  %q as --no-affinity was specified\", containerID)\n\t} else {\n\t\tstp.Info(\"setting cpuset of container %q to %q\", containerID, cpuset)\n\t\tc.SetCpusetCpus(cpuset)\n\t}\n\n\tc.SetEnv(CmkEnvAssigned, cpuset)\n\n\t// Advertise CPUs belonging to the infa pool\n\tpool, ok = stp.conf.Pools[CmkPoolInfra]\n\tif ok {\n\t\tc.SetEnv(CmkEnvInfra, pool.cpuSet())\n\t}\n\n\t// Advertise CPUs belonging to the shared pool\n\tpool, ok = stp.conf.Pools[CmkPoolShared]\n\tif ok {\n\t\tc.SetEnv(CmkEnvShared, pool.cpuSet())\n\t}\n\n\treturn nil\n}\n\n// getAvailableCPULists Constructa a list of available cpu lists that satisfy\n// the possible socket constraint\nfunc getAvailableCPULists(socket int64, pool *poolConfig) [](*cpuList) {\n\tavailableCPULists := make([](*cpuList), 0, len(pool.CPULists))\n\tfor _, c := range pool.CPULists {\n\t\tif socket < 0 || socket == int64(c.Socket) {\n\t\t\tif pool.Exclusive && len(c.getContainers()) > 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tavailableCPULists = append(availableCPULists, c)\n\t\t}\n\t}\n\treturn availableCPULists\n}\n\nfunc (stp *stp) releaseStpResources(containerID string) error {\n\tccr := *stp.getContainerRegistry()\n\tif cs, ok := ccr[containerID]; ok {\n\t\tpool, ok := stp.conf.Pools[cs.Pool]\n\t\tif !ok {\n\t\t\treturn stpError(\"BUG: pool %q for container %q not found\", cs.Pool, containerID)\n\t\t}\n\t\tfor _, clist := range pool.CPULists {\n\t\t\tclist.removeContainer(containerID)\n\t\t}\n\t\tdelete(ccr, containerID)\n\n\t\t// Commit our changes to stp cache\n\t\tstp.setContainerRegistry(&ccr)\n\t}\n\n\treturn nil\n}\n\n//\n// Handling of cached data\n//\n\nconst (\n\tcacheKeyContainerRegistry = \"ContainerRegistry\"\n)\n\ntype stpContainerStatus struct {\n\tPool           string   // pool configuration\n\tSocket         int64    // physical socket id\n\tNExclusiveCPUs int64    // number of exclusive cpus\n\tCpusets        []string // cpusets (cpu lists) assigned to this container\n\tNoAffinity     bool     // disable cpuset enforcing\n}\n\n// stpContainerCache contains STP-specific data of containers\ntype stpContainerCache map[string]stpContainerStatus\n\n// Set the value of cached cachableContainerRegistry object\nfunc (c *stpContainerCache) Set(value interface{}) {\n\tswitch value.(type) {\n\tcase stpContainerCache:\n\t\t*c = value.(stpContainerCache)\n\tcase *stpContainerCache:\n\t\tcp := value.(*stpContainerCache)\n\t\t*c = *cp\n\t}\n}\n\n// Get the cached cachableContainerRegistry object\nfunc (c *stpContainerCache) Get() interface{} {\n\treturn *c\n}\n\n// getContainerRegistry gets the current state of our container registry\nfunc (stp *stp) getContainerRegistry() *stpContainerCache {\n\tccr := &stpContainerCache{}\n\n\tif !stp.state.GetPolicyEntry(cacheKeyContainerRegistry, ccr) {\n\t\tstp.Error(\"no cached container registry found\")\n\t}\n\n\treturn ccr\n}\n\n// setContainerRegistry caches the state of our container registry\nfunc (stp *stp) setContainerRegistry(ccr *stpContainerCache) {\n\tstp.state.SetPolicyEntry(cacheKeyContainerRegistry, cache.Cachable(ccr))\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicy.Register(PolicyName, PolicyDescription, CreateStpPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/static-pools/stp-policy_test.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage stp\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/google/go-cmp/cmp\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nfunc TestParseContainerCmdline(t *testing.T) {\n\tstp := &stp{Logger: logger.NewLogger(PolicyName + \"-test\")}\n\n\t// 1. empty command line should return a nil pointer\n\targs := stp.parseContainerCmdline([]string{}, []string{})\n\tif args != nil {\n\t\tt.Errorf(\"Exptected <nil> but got %v\", *args)\n\t}\n\n\t// 2. case where cmk isolate command is in container \"Command\"\n\targs = stp.parseContainerCmdline([]string{\"cmk\", \"isolate\", \"--pool\", \"foo\", \"--socket-id=2\", \"--conf-dir=/etc\", \"cmd\", \"-arg\"}, []string{})\n\texpected := cmkLegacyArgs{Pool: \"foo\", SocketID: 2, Command: []string{\"cmd\", \"-arg\"}}\n\tif args == nil || !cmp.Equal(expected, *args) {\n\t\tt.Errorf(\"Exptected %v but got %v\", expected, *args)\n\t}\n\n\t// 3. we should ignore unknown cmk options\n\targs = stp.parseContainerCmdline([]string{\"cmk\", \"isolate\", \"--invalid-1=inv1\", \"--pool\", \"foo\", \"--invalid-2=inv2\", \"cmd\", \"--arg\"}, []string{})\n\texpected = cmkLegacyArgs{Pool: \"foo\", SocketID: -1, Command: []string{\"cmd\", \"--arg\"}}\n\tif args == nil || !cmp.Equal(expected, *args) {\n\t\tt.Errorf(\"Exptected %v but got %v\", expected, *args)\n\t}\n\n\t// 4. --pool should be defined in cmk options\n\targs = stp.parseContainerCmdline([]string{\"cmk\", \"isolate\", \"--socket-id=2\", \"cmd\", \"--arg\"}, []string{})\n\tif args != nil {\n\t\tt.Errorf(\"Exptected <nil> but got %v\", *args)\n\t}\n\n\t// 5. parsing from container \"Args\"\n\targs = stp.parseContainerCmdline([]string{\"bash\"}, []string{\"-c\", \"cmk isolate --pool=foo --socket-id=2 cmd --arg\"})\n\texpected = cmkLegacyArgs{Pool: \"foo\", SocketID: 2, Command: []string{\"cmd\", \"--arg\"}}\n\tif args == nil || !cmp.Equal(expected, *args) {\n\t\tt.Errorf(\"Exptected %v but got %v\", expected, *args)\n\t}\n\n\t// 6. Only _cmk_ isolate should be accepted\n\targs = stp.parseContainerCmdline([]string{\"bash\"}, []string{\"-c\", \"dmk isolate --pool=foo cmd --arg\"})\n\tif args != nil {\n\t\tt.Errorf(\"Exptected <nil> but got %v\", *args)\n\t}\n}\n\nfunc TestCachableData(t *testing.T) {\n\tccr := &stpContainerCache{\"id1\": stpContainerStatus{Pool: \"p\", Socket: 1}}\n\n\t// Test JSON marshalling of cached data\n\tdata, err := json.Marshal(ccr)\n\tif err != nil {\n\t\tt.Errorf(\"JSON marshal failed: %v\", err)\n\t}\n\texpected := []byte(`{\"id1\":{\"Pool\":\"p\",\"Socket\":1,\"NExclusiveCPUs\":0,\"Cpusets\":null,\"NoAffinity\":false}}`)\n\tif !cmp.Equal(expected, data) {\n\t\tt.Errorf(\"Exptected %s but got %s\", expected, data)\n\t}\n\n\t// Test JSON unmarshalling of cached data\n\tccr2 := &stpContainerCache{}\n\terr = json.Unmarshal(data, ccr2)\n\tif err != nil {\n\t\tt.Errorf(\"JSON unmarshal failed: %v\", err)\n\t}\n\tif !cmp.Equal(*ccr, *ccr2) {\n\t\tt.Errorf(\"Exptected %v but got %v\", *ccr, *ccr2)\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/affinity.go",
    "content": "// Copyright Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n)\n\n// Calculate pool affinities for the given container.\nfunc (p *policy) calculatePoolAffinities(container cache.Container) (map[int]int32, error) {\n\tlog.Debug(\"=> calculating pool affinities...\")\n\n\taffinities, err := p.calculateContainerAffinity(container)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresult := make(map[int]int32, len(p.nodes))\n\tfor id, w := range affinities {\n\t\tgrant, ok := p.allocations.grants[id]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tnode := grant.GetCPUNode()\n\t\tresult[node.NodeID()] += w\n\n\t\t// TODO: calculate affinity for memory here too?\n\t}\n\n\treturn result, nil\n}\n\n// Calculate affinity of this container (against all other containers).\nfunc (p *policy) calculateContainerAffinity(container cache.Container) (map[string]int32, error) {\n\tlog.Debug(\"* calculating affinity for container %s...\", container.PrettyName())\n\n\tca, err := container.GetAffinity()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresult := make(map[string]int32)\n\tfor _, a := range ca {\n\t\tfor id, w := range p.cache.EvaluateAffinity(a) {\n\t\t\tresult[id] += w\n\t\t}\n\t}\n\n\t// self-affinity does not make sense, so remove any\n\tdelete(result, container.GetCacheID())\n\n\tlog.Debug(\"  => affinity: %v\", result)\n\n\treturn result, nil\n}\n\n// Register our policy-specific implicit affinities with the Cache.\nfunc (p *policy) registerImplicitAffinities() error {\n\taffinities := []struct {\n\t\tname     string\n\t\tdisabled bool\n\t\taffinity cache.ImplicitAffinity\n\t}{\n\t\t{\n\t\t\tname: \"AVX512-pull/push\",\n\t\t\taffinity: func(c cache.Container, hasExplicit bool) *cache.Affinity {\n\t\t\t\t_, tagged := c.GetTag(cache.TagAVX512)\n\t\t\t\tif tagged {\n\t\t\t\t\treturn cache.GlobalAffinity(\"tags/\"+cache.TagAVX512, 5)\n\t\t\t\t}\n\t\t\t\treturn cache.GlobalAntiAffinity(\"tags/\"+cache.TagAVX512, 5)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:     \"colocate-pods\",\n\t\t\tdisabled: !opt.ColocatePods,\n\t\t\taffinity: func(c cache.Container, hasExplicit bool) *cache.Affinity {\n\t\t\t\tif hasExplicit {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tpod, ok := c.GetPod()\n\t\t\t\tif !ok {\n\t\t\t\t\tlog.Error(\"failed to inject pod-colocation affinity, can't find pod\")\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn &cache.Affinity{\n\t\t\t\t\tScope: pod.ScopeExpression(),\n\t\t\t\t\tMatch: &resmgr.Expression{\n\t\t\t\t\t\tOp: resmgr.AlwaysTrue,\n\t\t\t\t\t},\n\t\t\t\t\tWeight: 10,\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:     \"colocate-namespaces\",\n\t\t\tdisabled: !opt.ColocateNamespaces,\n\t\t\taffinity: func(c cache.Container, hasExplicit bool) *cache.Affinity {\n\t\t\t\tif hasExplicit {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn &cache.Affinity{\n\t\t\t\t\tScope: &resmgr.Expression{\n\t\t\t\t\t\tOp: resmgr.AlwaysTrue,\n\t\t\t\t\t},\n\t\t\t\t\tMatch: &resmgr.Expression{\n\t\t\t\t\t\tKey: resmgr.KeyNamespace,\n\t\t\t\t\t\tOp:  resmgr.Equals,\n\t\t\t\t\t\tValues: []string{\n\t\t\t\t\t\t\tc.GetNamespace(),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tWeight: 10,\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t}\n\n\tenabled := map[string]cache.ImplicitAffinity{}\n\tfor _, a := range affinities {\n\t\tif a.disabled {\n\t\t\tlog.Info(\"implicit affinity %s is disabled\", a.name)\n\t\t\tcontinue\n\t\t}\n\t\tenabled[PolicyName+\":\"+a.name] = a.affinity\n\t}\n\n\tif err := p.cache.AddImplicitAffinities(enabled); err != nil {\n\t\treturn policyError(\"failed to register implicit affinities: %v\", err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/cache.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\tkeyAllocations = \"allocations\"\n\tkeyConfig      = \"config\"\n)\n\nfunc (p *policy) saveAllocations() {\n\tp.cache.SetPolicyEntry(keyAllocations, cache.Cachable(&p.allocations))\n\tp.cache.Save()\n}\n\nfunc (p *policy) restoreAllocations(allocations *allocations) error {\n\tsavedAllocations := allocations.clone()\n\tp.allocations = p.newAllocations()\n\n\t//\n\t// Try to reinstate all grants with the exact same resource assignments\n\t// as saved. If that fails, release and try to reallocate all corresponding\n\t// containers with pool hints pointing to the currently assigned pools. If\n\t// this fails too, save the original allocations unchanged to the cache and\n\t// return an error.\n\t//\n\n\tif err := p.reinstateGrants(allocations.grants); err != nil {\n\t\tlog.Error(\"failed to reinstate grants verbatim: %v\", err)\n\t\tcontainers, poolHints := allocations.getContainerPoolHints()\n\t\tif err := p.reallocateResources(containers, poolHints); err != nil {\n\t\t\tp.allocations = savedAllocations\n\t\t\tp.saveAllocations() // undo any potential changes in saved cache\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// reinstateGrants tries to restore the given grants exactly as such.\nfunc (p *policy) reinstateGrants(grants map[string]Grant) error {\n\tfor id, grant := range grants {\n\t\tc := grant.GetContainer()\n\n\t\tpool := grant.GetCPUNode()\n\t\tsupply := pool.FreeSupply()\n\n\t\tif err := supply.Reserve(grant); err != nil {\n\t\t\treturn policyError(\"failed to update pool %q with CPU grant of %q: %v\",\n\t\t\t\tpool.Name(), c.PrettyName(), err)\n\t\t}\n\n\t\tlog.Info(\"updated pool %q with reinstated CPU grant of %q\",\n\t\t\tpool.Name(), c.PrettyName())\n\n\t\tpool = grant.GetMemoryNode()\n\t\tif err := supply.ReserveMemory(grant); err != nil {\n\t\t\tgrant.GetCPUNode().FreeSupply().ReleaseCPU(grant)\n\t\t\treturn policyError(\"failed to update pool %q with extra memory of %q: %v\",\n\t\t\t\tpool.Name(), c.PrettyName(), err)\n\t\t}\n\n\t\tlog.Info(\"updated pool %q with reinstanted memory reservation of %q\",\n\t\t\tpool.Name(), c.PrettyName())\n\n\t\tp.allocations.grants[id] = grant\n\t\tp.applyGrant(grant)\n\t}\n\n\tp.updateSharedAllocations(nil)\n\n\treturn nil\n}\n\ntype cachedGrant struct {\n\tExclusive   string\n\tPart        int\n\tCPUType     cpuClass\n\tContainer   string\n\tPool        string\n\tMemoryPool  string\n\tMemType     memoryType\n\tMemset      idset.IDSet\n\tMemoryLimit memoryMap\n\tColdStart   time.Duration\n}\n\nfunc newCachedGrant(cg Grant) *cachedGrant {\n\tccg := &cachedGrant{}\n\tccg.Exclusive = cg.ExclusiveCPUs().String()\n\tccg.Part = cg.CPUPortion()\n\tccg.CPUType = cg.CPUType()\n\tccg.Container = cg.GetContainer().GetCacheID()\n\tccg.Pool = cg.GetCPUNode().Name()\n\tccg.MemoryPool = cg.GetMemoryNode().Name()\n\tccg.MemType = cg.MemoryType()\n\tccg.Memset = cg.Memset().Clone()\n\n\tccg.MemoryLimit = make(memoryMap)\n\tfor key, value := range cg.MemLimit() {\n\t\tccg.MemoryLimit[key] = value\n\t}\n\n\tccg.ColdStart = cg.ColdStart()\n\n\treturn ccg\n}\n\nfunc (ccg *cachedGrant) ToGrant(policy *policy) (Grant, error) {\n\tnode, ok := policy.nodes[ccg.Pool]\n\tif !ok {\n\t\treturn nil, policyError(\"cache error: failed to restore %v, unknown pool/node\", *ccg)\n\t}\n\tcontainer, ok := policy.cache.LookupContainer(ccg.Container)\n\tif !ok {\n\t\treturn nil, policyError(\"cache error: failed to restore %v, unknown container\", *ccg)\n\t}\n\n\tg := newGrant(\n\t\tnode,\n\t\tcontainer,\n\t\tccg.CPUType,\n\t\tcpuset.MustParse(ccg.Exclusive),\n\t\tccg.Part,\n\t\tccg.MemType,\n\t\tccg.MemoryLimit,\n\t\tccg.ColdStart,\n\t)\n\n\tif g.Memset().String() != ccg.Memset.String() {\n\t\tlog.Error(\"cache error: mismatch in stored/recalculated memset: %s != %s\",\n\t\t\tccg.Memset, g.Memset())\n\t}\n\n\treturn g, nil\n}\n\nfunc (cg *grant) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(newCachedGrant(cg))\n}\n\nfunc (cg *grant) UnmarshalJSON(data []byte) error {\n\tccg := cachedGrant{}\n\n\tif err := json.Unmarshal(data, &ccg); err != nil {\n\t\treturn policyError(\"failed to restore grant: %v\", err)\n\t}\n\n\tcg.exclusive = cpuset.MustParse(ccg.Exclusive)\n\n\treturn nil\n}\n\nfunc (a *allocations) MarshalJSON() ([]byte, error) {\n\tcgrants := make(map[string]*cachedGrant)\n\tfor id, cg := range a.grants {\n\t\tcgrants[id] = newCachedGrant(cg)\n\t}\n\n\treturn json.Marshal(cgrants)\n}\n\nfunc (a *allocations) UnmarshalJSON(data []byte) error {\n\tvar err error\n\n\tcgrants := make(map[string]*cachedGrant)\n\tif err := json.Unmarshal(data, &cgrants); err != nil {\n\t\treturn policyError(\"failed to restore allocations: %v\", err)\n\t}\n\n\ta.grants = make(map[string]Grant, 32)\n\tfor id, ccg := range cgrants {\n\t\ta.grants[id], err = ccg.ToGrant(a.policy)\n\t\tif err != nil {\n\t\t\tlog.Error(\"removing unresolvable cached grant %v: %v\", *ccg, err)\n\t\t\tdelete(a.grants, id)\n\t\t} else {\n\t\t\tlog.Debug(\"resolved cache grant: %v\", a.grants[id].String())\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (a *allocations) Get() interface{} {\n\treturn a\n}\n\nfunc (a *allocations) Set(value interface{}) {\n\tvar from *allocations\n\n\tswitch value.(type) {\n\tcase allocations:\n\t\tv := value.(allocations)\n\t\tfrom = &v\n\tcase *allocations:\n\t\tfrom = value.(*allocations)\n\t}\n\n\ta.grants = make(map[string]Grant, 32)\n\tfor id, cg := range from.grants {\n\t\ta.grants[id] = cg\n\t}\n}\n\nfunc (a *allocations) Dump(logfn func(format string, args ...interface{}), prefix string) {\n\tfor _, cg := range a.grants {\n\t\tlogfn(prefix+\"%s\", cg)\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/cache_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nfunc TestToGrant(t *testing.T) {\n\ttcases := []struct {\n\t\tname          string\n\t\tpolicy        *policy\n\t\tcgrant        *cachedGrant\n\t\texpectedError bool\n\t}{\n\t\t{\n\t\t\tname:   \"unknown node\",\n\t\t\tcgrant: &cachedGrant{},\n\t\t\tpolicy: &policy{\n\t\t\t\tnodes: map[string]Node{\n\t\t\t\t\t\"node1\": &node{},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"known node but failed lookup\",\n\t\t\tcgrant: &cachedGrant{\n\t\t\t\tPool: \"node1\",\n\t\t\t},\n\t\t\tpolicy: &policy{\n\t\t\t\tnodes: map[string]Node{\n\t\t\t\t\t\"node1\": &node{},\n\t\t\t\t},\n\t\t\t\tcache: &mockCache{},\n\t\t\t},\n\t\t\texpectedError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"known node\",\n\t\t\tcgrant: &cachedGrant{\n\t\t\t\tPool: \"node1\",\n\t\t\t},\n\t\t\tpolicy: &policy{\n\t\t\t\tnodes: map[string]Node{\n\t\t\t\t\t\"node1\": &node{},\n\t\t\t\t},\n\t\t\t\tcache: &mockCache{\n\t\t\t\t\treturnValue2ForLookupContainer: true,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t_, err := tc.cgrant.ToGrant(tc.policy)\n\t\t\tif tc.expectedError && err == nil {\n\t\t\t\tt.Errorf(\"Expected error, but got success\")\n\t\t\t}\n\t\t\tif !tc.expectedError && err != nil {\n\t\t\t\tt.Errorf(\"Unxpected error: %+v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAllocationMarshalling(t *testing.T) {\n\ttcases := []struct {\n\t\tname                       string\n\t\tdata                       []byte\n\t\texpectedUnmarshallingError bool\n\t\texpectedMarshallingError   bool\n\t}{\n\t\t{\n\t\t\tname: \"non-zero Exclusive\",\n\t\t\tdata: []byte(`{\"key1\":{\"Exclusive\":\"1\",\"Part\":1,\"CPUType\":0,\"Container\":\"1\",\"Pool\":\"testnode\",\"MemoryPool\":\"testnode\",\"MemType\":\"DRAM,PMEM,HBM\",\"Memset\":\"\",\"MemoryLimit\":{},\"ColdStart\":0}}`),\n\t\t},\n\t\t{\n\t\t\tname: \"zero Exclusive\",\n\t\t\tdata: []byte(`{\"key1\":{\"Exclusive\":\"\",\"Part\":1,\"CPUType\":0,\"Container\":\"1\",\"Pool\":\"testnode\",\"MemoryPool\":\"testnode\",\"MemType\":\"DRAM,PMEM,HBM\",\"Memset\":\"\",\"MemoryLimit\":{},\"ColdStart\":0}}`),\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\talloc := &allocations{\n\t\t\t\tpolicy: &policy{\n\t\t\t\t\tnodes: map[string]Node{\n\t\t\t\t\t\t\"testnode\": &virtualnode{\n\t\t\t\t\t\t\tnode: node{\n\t\t\t\t\t\t\t\tname:    \"testnode\",\n\t\t\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(0, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(0, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tcache: &mockCache{\n\t\t\t\t\t\treturnValue1ForLookupContainer: &mockContainer{\n\t\t\t\t\t\t\treturnValueForGetCacheID: \"1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\treturnValue2ForLookupContainer: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t\tunmarshallingErr := alloc.UnmarshalJSON(tc.data)\n\t\t\tif tc.expectedUnmarshallingError && unmarshallingErr == nil {\n\t\t\t\tt.Errorf(\"Expected unmarshalling error, but got success\")\n\t\t\t}\n\t\t\tif !tc.expectedUnmarshallingError && unmarshallingErr != nil {\n\t\t\t\tt.Errorf(\"Unxpected unmarshalling error: %+v\", unmarshallingErr)\n\t\t\t}\n\n\t\t\tout, marshallingErr := alloc.MarshalJSON()\n\t\t\tif !bytes.Equal(out, tc.data) {\n\t\t\t\tt.Errorf(\"Expected\\n%q\\nBut got\\n%q\", tc.data, out)\n\t\t\t}\n\t\t\tif tc.expectedMarshallingError && marshallingErr == nil {\n\t\t\t\tt.Errorf(\"Expected marshalling error, but got success\")\n\t\t\t}\n\t\t\tif !tc.expectedMarshallingError && marshallingErr != nil {\n\t\t\t\tt.Errorf(\"Unxpected marshalling error: %+v\", marshallingErr)\n\t\t\t}\n\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/coldstart.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n)\n\n// trigger cold start for the container if necessary.\nfunc (p *policy) triggerColdStart(c cache.Container) error {\n\tlog.Info(\"coldstart: triggering coldstart for %s...\", c.PrettyName())\n\tg, ok := p.allocations.grants[c.GetCacheID()]\n\tif !ok {\n\t\tlog.Warn(\"coldstart: no grant found, nothing to do...\")\n\t\treturn nil\n\t}\n\n\tcoldStart := g.ColdStart()\n\tif coldStart <= 0 {\n\t\tlog.Info(\"coldstart: no coldstart, nothing to do...\")\n\t\treturn nil\n\t}\n\n\t// Start a timer to restore the grant memset to full. Store the\n\t// timer so that we can release it if the grant is destroyed before\n\t// the timer elapses.\n\tduration := coldStart\n\ttimer := time.AfterFunc(duration, func() {\n\t\te := &events.Policy{\n\t\t\tType:   ColdStartDone,\n\t\t\tSource: PolicyName,\n\t\t\tData:   c.GetID(),\n\t\t}\n\t\tif err := p.options.SendEvent(e); err != nil {\n\t\t\t// we should retry this later, the channel is probably full...\n\t\t\tlog.Error(\"Ouch... we'should retry this later.\")\n\t\t}\n\t})\n\tg.AddTimer(timer)\n\treturn nil\n}\n\n// finish an ongoing coldstart for the container.\nfunc (p *policy) finishColdStart(c cache.Container) (bool, error) {\n\tg, ok := p.allocations.grants[c.GetCacheID()]\n\tif !ok {\n\t\tlog.Warn(\"coldstart: no grant found, nothing to do...\")\n\t\treturn false, policyError(\"coldstart: no grant found for %s\", c.PrettyName())\n\t}\n\n\tlog.Info(\"restoring memset to grant %v\", g)\n\tg.RestoreMemset()\n\tg.ClearTimer()\n\n\treturn true, nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/coldstart_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\tpolicyapi \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nvar globalPolicy *policy\nvar mutex sync.Mutex\n\nfunc sendEvent(param interface{}) error {\n\t// Simulate event synchronization in the upper levels.\n\tmutex.Lock()\n\tdefer mutex.Unlock()\n\n\tfmt.Printf(\"Event received: %v\", param)\n\tevent := param.(*events.Policy)\n\tglobalPolicy.HandleEvent(event)\n\treturn nil\n}\n\nfunc TestColdStart(t *testing.T) {\n\n\t// Idea with cold start is that the workload is first allocated only PMEM node. Only when timer expires\n\t// (or some other event is triggered) is the DRAM node added to the memset. This causes the initial\n\t// memory allocations to be made from PMEM only.\n\n\ttcases := []struct {\n\t\tname                     string\n\t\tnumaNodes                []system.Node\n\t\treq                      Request\n\t\taffinities               map[int]int32\n\t\tcontainer                cache.Container\n\t\texpectedColdStartTimeout time.Duration\n\t\texpectedDRAMNodeID       int\n\t\texpectedPMEMNodeID       int\n\t\texpectedDRAMSystemNodeID idset.ID\n\t\texpectedPMEMSystemNodeID idset.ID\n\t}{\n\t\t{\n\t\t\tname: \"three node cold start\",\n\t\t\tnumaNodes: []system.Node{\n\t\t\t\t&mockSystemNode{id: 1, memFree: 10000, memTotal: 10000, memType: system.MemoryTypeDRAM, distance: []int{5, 5, 1}},\n\t\t\t\t&mockSystemNode{id: 2, memFree: 50000, memTotal: 50000, memType: system.MemoryTypePMEM, distance: []int{5, 1, 5}},\n\t\t\t},\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname:                     \"demo-coldstart-container\",\n\t\t\t\treturnValueForGetCacheID: \"1234\",\n\t\t\t\tpod: &mockPod{\n\t\t\t\t\tcoldStartTimeout:                   1000 * time.Millisecond,\n\t\t\t\t\treturnValue1FotGetResmgrAnnotation: \"demo-coldstart-container: pmem,dram\",\n\t\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t\t\tcoldStartContainerName:             \"demo-coldstart-container\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedColdStartTimeout: 1000 * time.Millisecond,\n\t\t\texpectedDRAMNodeID:       101,\n\t\t\texpectedDRAMSystemNodeID: idset.ID(1),\n\t\t\texpectedPMEMSystemNodeID: idset.ID(2),\n\t\t\texpectedPMEMNodeID:       102,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tpolicy := &policy{\n\t\t\t\tsys: &mockSystem{\n\t\t\t\t\tnodes: tc.numaNodes,\n\t\t\t\t},\n\t\t\t\tcache: &mockCache{\n\t\t\t\t\treturnValue1ForLookupContainer: tc.container,\n\t\t\t\t\treturnValue2ForLookupContainer: true,\n\t\t\t\t},\n\t\t\t\tallocations: allocations{\n\t\t\t\t\tgrants: make(map[string]Grant, 0),\n\t\t\t\t},\n\t\t\t\toptions: &policyapi.BackendOptions{},\n\t\t\t}\n\t\t\tpolicy.allocations.policy = policy\n\t\t\tpolicy.options.SendEvent = sendEvent\n\n\t\t\tif err := policy.buildPoolsByTopology(); err != nil {\n\t\t\t\tt.Errorf(\"failed to build topology pool\")\n\t\t\t}\n\n\t\t\tgrant, err := policy.allocatePool(tc.container, \"\")\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t\tif grant.ColdStart() != tc.expectedColdStartTimeout {\n\t\t\t\tt.Errorf(\"Expected coldstart value '%v', but got '%v'\", tc.expectedColdStartTimeout, grant.ColdStart())\n\t\t\t}\n\n\t\t\tpolicy.allocations.grants[tc.container.GetCacheID()] = grant\n\n\t\t\tmems := grant.Memset()\n\t\t\tif len(mems) != 1 || mems.Members()[0] != tc.expectedPMEMSystemNodeID {\n\t\t\t\tt.Errorf(\"Expected one memory controller %v, got: %v\", tc.expectedPMEMSystemNodeID, mems)\n\t\t\t}\n\n\t\t\tif grant.MemoryType()&memoryDRAM != 0 {\n\t\t\t\t// FIXME: should we report only the limited memory types or the granted types\n\t\t\t\t// while the cold start is going on?\n\t\t\t\t// t.Errorf(\"No DRAM was expected before coldstart timer: %v\", grant.MemoryType())\n\t\t\t}\n\n\t\t\tglobalPolicy = policy\n\n\t\t\tpolicy.options.SendEvent(&events.Policy{\n\t\t\t\tType: events.ContainerStarted,\n\t\t\t\tData: tc.container,\n\t\t\t})\n\n\t\t\ttime.Sleep(tc.expectedColdStartTimeout * 2)\n\n\t\t\tnewMems := grant.Memset()\n\t\t\tif len(newMems) != 2 {\n\t\t\t\tt.Errorf(\"Expected two memory controllers, got %d: %v\", len(newMems), newMems)\n\t\t\t}\n\t\t\tif !newMems.Has(tc.expectedPMEMSystemNodeID) || !newMems.Has(tc.expectedDRAMSystemNodeID) {\n\t\t\t\tt.Errorf(\"Didn't get all expected system nodes in mems, got: %v\", newMems)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/error.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"fmt\"\n)\n\n// policyError creates a formatted policy-specific error.\nfunc policyError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(PolicyName+\": \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/flags.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\tconfig \"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// Options captures our configurable policy parameters.\ntype options struct {\n\t// PinCPU controls CPU pinning in this policy.\n\tPinCPU bool\n\t// PinMemory controls memory pinning in this policy.\n\tPinMemory bool\n\t// PreferIsolated controls whether isolated CPUs are preferred for isolated allocations.\n\tPreferIsolated bool `json:\"PreferIsolatedCPUs\"`\n\t// PreferShared controls whether shared CPU allocation is always preferred by default.\n\tPreferShared bool `json:\"PreferSharedCPUs\"`\n\t// ReservedPoolNamespaces is a list of namespace globs that will be allocated to reserved CPUs\n\tReservedPoolNamespaces []string `json:\"ReservedPoolNamespaces,omitempty\"`\n\t// ColocatePods causes all containers in a pod to have affinity for each other.\n\tColocatePods bool `json:\"ColocatePods\"`\n\t// ColocateNamespaces causes all containers in a namespace to have affinity for each other.\n\tColocateNamespaces bool `json:\"ColocateNamespaces\"`\n}\n\n// Our runtime configuration.\nvar opt = defaultOptions().(*options)\nvar aliasOpt = defaultOptions().(*options)\n\n// defaultOptions returns a new options instance, all initialized to defaults.\nfunc defaultOptions() interface{} {\n\treturn &options{\n\t\tPinCPU:                 true,\n\t\tPinMemory:              true,\n\t\tPreferIsolated:         true,\n\t\tPreferShared:           false,\n\t\tReservedPoolNamespaces: []string{\"kube-system\"},\n\t}\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tconfig.Register(PolicyPath, PolicyDescription, opt, defaultOptions)\n\tconfig.Register(AliasPath, PolicyDescription, aliasOpt, defaultOptions)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/hint.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"strconv\"\n\t\"strings\"\n\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// Calculate the hint score of the given hint and CPUSet.\nfunc cpuHintScore(hint topology.Hint, CPUs cpuset.CPUSet) float64 {\n\thCPUs, err := cpuset.Parse(hint.CPUs)\n\tif err != nil {\n\t\tlog.Warn(\"invalid hint CPUs '%s' from %s\", hint.CPUs, hint.Provider)\n\t\treturn 0.0\n\t}\n\tcommon := hCPUs.Intersection(CPUs)\n\treturn float64(common.Size()) / float64(hCPUs.Size())\n}\n\n// Calculate the NUMA node score of the given hint and NUMA node.\nfunc numaHintScore(hint topology.Hint, sysIDs ...idset.ID) float64 {\n\tfor _, idstr := range strings.Split(hint.NUMAs, \",\") {\n\t\thID, err := strconv.ParseInt(idstr, 0, 0)\n\t\tif err != nil {\n\t\t\tlog.Warn(\"invalid hint NUMA node %s from %s\", idstr, hint.Provider)\n\t\t\treturn 0.0\n\t\t}\n\n\t\tfor _, id := range sysIDs {\n\t\t\tif hID == int64(id) {\n\t\t\t\treturn 1.0\n\t\t\t}\n\t\t}\n\t}\n\n\treturn 0.0\n}\n\n// Calculate the die node score of the given hint and die.\nfunc dieHintScore(hint topology.Hint, sysID idset.ID, socket system.CPUPackage) float64 {\n\tnumaNodes := idset.NewIDSet(socket.DieNodeIDs(sysID)...)\n\n\tfor _, idstr := range strings.Split(hint.NUMAs, \",\") {\n\t\thID, err := strconv.ParseInt(idstr, 0, 0)\n\t\tif err != nil {\n\t\t\tlog.Warn(\"invalid hint NUMA node %s from %s\", idstr, hint.Provider)\n\t\t\treturn 0.0\n\t\t}\n\n\t\tif numaNodes.Has(idset.ID(hID)) {\n\t\t\treturn 1.0\n\t\t}\n\t}\n\n\treturn 0.0\n}\n\n// Calculate the socket node score of the given hint and NUMA node.\nfunc socketHintScore(hint topology.Hint, sysID idset.ID) float64 {\n\tfor _, idstr := range strings.Split(hint.Sockets, \",\") {\n\t\tid, err := strconv.ParseInt(idstr, 0, 0)\n\t\tif err != nil {\n\t\t\tlog.Warn(\"invalid hint socket '%s' from %s\", idstr, hint.Provider)\n\t\t\treturn 0.0\n\t\t}\n\t\tif id == int64(sysID) {\n\t\t\treturn 1.0\n\t\t}\n\t}\n\n\treturn 0.0\n}\n\n// return the cpuset for the CPU, NUMA or socket hints, preferred in this particular order.\nfunc (cs *supply) hintCpus(h topology.Hint) cpuset.CPUSet {\n\tvar cpus cpuset.CPUSet\n\n\tswitch {\n\tcase h.CPUs != \"\":\n\t\tcpus = cpuset.MustParse(h.CPUs)\n\n\tcase h.NUMAs != \"\":\n\t\tfor _, idstr := range strings.Split(h.NUMAs, \",\") {\n\t\t\tif id, err := strconv.ParseInt(idstr, 0, 0); err == nil {\n\t\t\t\tif node := cs.node.System().Node(idset.ID(id)); node != nil {\n\t\t\t\t\tcpus = cpus.Union(node.CPUSet())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\tcase h.Sockets != \"\":\n\t\tfor _, idstr := range strings.Split(h.Sockets, \",\") {\n\t\t\tif id, err := strconv.ParseInt(idstr, 0, 0); err == nil {\n\t\t\t\tif pkg := cs.node.System().Package(idset.ID(id)); pkg != nil {\n\t\t\t\t\tcpus = cpus.Union(pkg.CPUSet())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn cpus\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/hint_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nfunc TestCpuHintScore(t *testing.T) {\n\ttcases := []struct {\n\t\tname     string\n\t\texpected float64\n\t\thint     topology.Hint\n\t\tcpus     cpuset.CPUSet\n\t\tdisabled bool // TODO(rojkov): remove this field when the code is fixed.\n\t}{\n\t\t{\n\t\t\tname:     \"handle zero cpu size gracefully\",\n\t\t\tdisabled: true,\n\t\t},\n\t\t{\n\t\t\tname: \"handle unparsable cpu size gracefully\",\n\t\t\thint: topology.Hint{\n\t\t\t\tCPUs: \"unparsable\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"non-zero cpu size hint and empty CPUs\",\n\t\t\thint: topology.Hint{\n\t\t\t\tCPUs: \"1\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"hint corresponding to given CPU\",\n\t\t\thint: topology.Hint{\n\t\t\t\tCPUs: \"1,2\",\n\t\t\t},\n\t\t\tcpus:     cpuset.New(1),\n\t\t\texpected: 0.5,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tif tc.disabled {\n\t\t\t\tt.Skipf(\"The case '%s' is skipped\", tc.name)\n\t\t\t}\n\t\t\tactual := cpuHintScore(tc.hint, tc.cpus)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Errorf(\"Expected %f, but got %f\", tc.expected, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNumaHintScore(t *testing.T) {\n\ttcases := []struct {\n\t\tname     string\n\t\texpected float64\n\t\thint     topology.Hint\n\t\tids      []idset.ID\n\t}{\n\t\t{\n\t\t\tname: \"handle unparsable NUMAs gracefully\",\n\t\t\thint: topology.Hint{\n\t\t\t\tNUMAs: \"unparsable\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"non-zero NUMA hint and empty NUMAs\",\n\t\t\thint: topology.Hint{\n\t\t\t\tNUMAs: \"1\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"hint corresponding to a given ID\",\n\t\t\tids:  []idset.ID{1},\n\t\t\thint: topology.Hint{\n\t\t\t\tNUMAs: \"1,2\",\n\t\t\t},\n\t\t\texpected: 1.0,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tactual := numaHintScore(tc.hint, tc.ids...)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Errorf(\"Expected %f, but got %f\", tc.expected, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestSocketHintScore(t *testing.T) {\n\ttcases := []struct {\n\t\tname     string\n\t\texpected float64\n\t\thint     topology.Hint\n\t\tid       idset.ID\n\t}{\n\t\t{\n\t\t\tname: \"handle unparsable Sockets gracefully\",\n\t\t\thint: topology.Hint{\n\t\t\t\tSockets: \"unparsable\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"non-zero Sockets hint and empty Sockets\",\n\t\t\thint: topology.Hint{\n\t\t\t\tSockets: \"1\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"hint corresponding to a given ID\",\n\t\t\tid:   1,\n\t\t\thint: topology.Hint{\n\t\t\t\tSockets: \"1,2\",\n\t\t\t},\n\t\t\texpected: 1.0,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tactual := socketHintScore(tc.hint, tc.id)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Errorf(\"Expected %f, but got %f\", tc.expected, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestHintCpus(t *testing.T) {\n\ttcases := []struct {\n\t\tname     string\n\t\tsupply   *supply\n\t\thint     topology.Hint\n\t\texpected cpuset.CPUSet\n\t}{\n\t\t{\n\t\t\tname:   \"handle unparsable Sockets gracefully\",\n\t\t\tsupply: &supply{},\n\t\t\thint: topology.Hint{\n\t\t\t\tSockets: \"unparsable\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"non-zero Sockets hint and empty system.Package\",\n\t\t\tsupply: &supply{\n\t\t\t\tnode: &node{\n\t\t\t\t\tpolicy: &policy{\n\t\t\t\t\t\tsys: &mockSystem{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\thint: topology.Hint{\n\t\t\t\tSockets: \"1\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname:   \"handle unparsable NUMAs gracefully\",\n\t\t\tsupply: &supply{},\n\t\t\thint: topology.Hint{\n\t\t\t\tNUMAs: \"unparsable\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"non-zero NUMAs hint and empty system.Node\",\n\t\t\tsupply: &supply{\n\t\t\t\tnode: &node{\n\t\t\t\t\tpolicy: &policy{\n\t\t\t\t\t\tsys: &mockSystem{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\thint: topology.Hint{\n\t\t\t\tNUMAs: \"1\",\n\t\t\t},\n\t\t},\n\t\t// TODO(rojkov): add tests for non-empty system.Package's (can't be done while system.Package is closed struct)\n\t\t{\n\t\t\tname:   \"non-zero CPUs hint\",\n\t\t\tsupply: &supply{},\n\t\t\thint: topology.Hint{\n\t\t\t\tCPUs: \"1\",\n\t\t\t},\n\t\t\texpected: cpuset.New(1),\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tactual := tc.supply.hintCpus(tc.hint)\n\t\t\tif tc.expected.IsEmpty() && actual.IsEmpty() {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif !tc.expected.Equals(actual) {\n\t\t\t\tt.Errorf(\"Expected %+v, but got %+v\", tc.expected, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/logging.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"fmt\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\n// Create our logger instance.\nvar log logger.Logger = logger.NewLogger(\"policy\")\n\n// indent produces an indentation string for the given level.\nconst (\n\tIndentDepth = 4\n)\n\nfunc indent(prefix string, level ...int) string {\n\tif len(level) < 1 {\n\t\treturn prefix\n\t}\n\n\tdepth := level[0] * IndentDepth\n\treturn prefix + fmt.Sprintf(\"%*.*s\", depth, depth, \"\")\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/mocks_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/apis/resmgr\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/config\"\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/intel/goresctrl/pkg/sst\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n\tv1 \"k8s.io/api/core/v1\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n)\n\ntype mockSystemNode struct {\n\tid       idset.ID // node id\n\tmemFree  uint64\n\tmemTotal uint64\n\tmemType  system.MemoryType\n\tdistance []int\n}\n\nfunc (fake *mockSystemNode) MemoryInfo() (*system.MemInfo, error) {\n\treturn &system.MemInfo{MemFree: fake.memFree, MemTotal: fake.memTotal}, nil\n}\n\nfunc (fake *mockSystemNode) PackageID() idset.ID {\n\treturn 0\n}\n\nfunc (fake *mockSystemNode) DieID() idset.ID {\n\treturn 0\n}\n\nfunc (fake *mockSystemNode) ID() idset.ID {\n\treturn fake.id\n}\n\nfunc (fake *mockSystemNode) GetMemoryType() system.MemoryType {\n\treturn fake.memType\n}\n\nfunc (fake *mockSystemNode) HasNormalMemory() bool {\n\treturn true\n}\n\nfunc (fake *mockSystemNode) CPUSet() cpuset.CPUSet {\n\treturn cpuset.New()\n}\n\nfunc (fake *mockSystemNode) Distance() []int {\n\tif len(fake.distance) == 0 {\n\t\treturn []int{0}\n\t}\n\treturn fake.distance\n}\n\nfunc (fake *mockSystemNode) DistanceFrom(id idset.ID) int {\n\treturn 0\n}\n\ntype mockCPUPackage struct {\n}\n\nfunc (p *mockCPUPackage) ID() idset.ID {\n\treturn idset.ID(0)\n}\n\nfunc (p *mockCPUPackage) CPUSet() cpuset.CPUSet {\n\treturn cpuset.New()\n}\n\nfunc (p *mockCPUPackage) NodeIDs() []idset.ID {\n\treturn []idset.ID{}\n}\n\nfunc (p *mockCPUPackage) DieIDs() []idset.ID {\n\treturn []idset.ID{0}\n}\n\nfunc (p *mockCPUPackage) DieCPUSet(idset.ID) cpuset.CPUSet {\n\treturn cpuset.New()\n}\n\nfunc (p *mockCPUPackage) DieNodeIDs(idset.ID) []idset.ID {\n\treturn []idset.ID{}\n}\n\nfunc (p *mockCPUPackage) SstInfo() *sst.SstPackageInfo {\n\treturn &sst.SstPackageInfo{}\n}\n\ntype mockCPU struct {\n\tisolated cpuset.CPUSet\n\tonline   cpuset.CPUSet\n\tid       idset.ID\n\tnode     mockSystemNode\n\tpkg      mockCPUPackage\n}\n\nfunc (c *mockCPU) BaseFrequency() uint64 {\n\treturn 0\n}\nfunc (c *mockCPU) EPP() system.EPP {\n\treturn system.EPPUnknown\n}\nfunc (c *mockCPU) ID() idset.ID {\n\treturn idset.ID(0)\n}\nfunc (c *mockCPU) PackageID() idset.ID {\n\treturn c.pkg.ID()\n}\nfunc (c *mockCPU) DieID() idset.ID {\n\treturn idset.ID(0)\n}\nfunc (c *mockCPU) NodeID() idset.ID {\n\treturn c.node.ID()\n}\nfunc (c *mockCPU) CoreID() idset.ID {\n\treturn c.id\n}\nfunc (c *mockCPU) ThreadCPUSet() cpuset.CPUSet {\n\treturn cpuset.New()\n}\nfunc (c *mockCPU) FrequencyRange() system.CPUFreq {\n\treturn system.CPUFreq{}\n}\nfunc (c *mockCPU) Online() bool {\n\treturn true\n}\nfunc (c *mockCPU) Isolated() bool {\n\treturn false\n}\nfunc (c *mockCPU) SetFrequencyLimits(min, max uint64) error {\n\treturn nil\n}\n\nfunc (c *mockCPU) SstClos() int {\n\treturn -1\n}\n\ntype mockSystem struct {\n\tisolatedCPU  int\n\tnodes        []system.Node\n\tcpuCount     int\n\tpackageCount int\n\tsocketCount  int\n}\n\nfunc (fake *mockSystem) Node(id idset.ID) system.Node {\n\tfor _, node := range fake.nodes {\n\t\tif node.ID() == id {\n\t\t\treturn node\n\t\t}\n\t}\n\treturn &mockSystemNode{}\n}\n\nfunc (fake *mockSystem) CPU(idset.ID) system.CPU {\n\treturn &mockCPU{}\n}\nfunc (fake *mockSystem) CPUCount() int {\n\tif fake.cpuCount == 0 {\n\t\treturn 1\n\t}\n\treturn fake.cpuCount\n}\nfunc (fake *mockSystem) Discover() error {\n\treturn nil\n}\nfunc (fake *mockSystem) Package(idset.ID) system.CPUPackage {\n\treturn &mockCPUPackage{}\n}\nfunc (fake *mockSystem) Offlined() cpuset.CPUSet {\n\treturn cpuset.New()\n}\nfunc (fake *mockSystem) Isolated() cpuset.CPUSet {\n\tif fake.isolatedCPU > 0 {\n\t\treturn cpuset.New(fake.isolatedCPU)\n\t}\n\n\treturn cpuset.New()\n}\nfunc (fake *mockSystem) CPUSet() cpuset.CPUSet {\n\treturn cpuset.New()\n}\nfunc (fake *mockSystem) CPUIDs() []idset.ID {\n\treturn []idset.ID{}\n}\nfunc (fake *mockSystem) PackageCount() int {\n\tif fake.packageCount == 0 {\n\t\treturn 1\n\t}\n\treturn fake.packageCount\n}\nfunc (fake *mockSystem) SocketCount() int {\n\tif fake.socketCount == 0 {\n\t\treturn 1\n\t}\n\treturn fake.socketCount\n}\nfunc (fake *mockSystem) NUMANodeCount() int {\n\treturn len(fake.nodes)\n}\nfunc (fake *mockSystem) ThreadCount() int {\n\tif fake.cpuCount == 0 {\n\t\treturn 1\n\t}\n\treturn fake.cpuCount\n}\nfunc (fake *mockSystem) PackageIDs() []idset.ID {\n\tids := make([]idset.ID, len(fake.nodes))\n\tfor i, node := range fake.nodes {\n\t\tids[i] = node.PackageID()\n\t}\n\treturn ids\n}\nfunc (fake *mockSystem) NodeIDs() []idset.ID {\n\tids := make([]idset.ID, len(fake.nodes))\n\tfor i, node := range fake.nodes {\n\t\tids[i] = node.ID()\n\t}\n\treturn ids\n}\nfunc (fake *mockSystem) SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error {\n\treturn nil\n}\nfunc (fake *mockSystem) SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error) {\n\treturn idset.NewIDSet(), nil\n}\nfunc (fake *mockSystem) NodeDistance(idset.ID, idset.ID) int {\n\treturn 10\n}\n\ntype mockContainer struct {\n\tname                                  string\n\tnamespace                             string\n\treturnValueForGetResourceRequirements v1.ResourceRequirements\n\treturnValueForGetCacheID              string\n\treturnValueForGetID                   string\n\tmemoryLimit                           int64\n\tcpuset                                cpuset.CPUSet\n\treturnValueForQOSClass                v1.PodQOSClass\n\tpod                                   cache.Pod\n}\n\nfunc (m *mockContainer) PrettyName() string {\n\treturn m.name\n}\nfunc (m *mockContainer) GetPod() (cache.Pod, bool) {\n\tif m.pod == nil {\n\t\treturn &mockPod{}, false\n\t}\n\treturn m.pod, true\n}\nfunc (m *mockContainer) GetID() string {\n\treturn m.returnValueForGetID\n}\nfunc (m *mockContainer) GetPodID() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCacheID() string {\n\tif len(m.returnValueForGetCacheID) == 0 {\n\t\treturn \"0\"\n\t}\n\n\treturn m.returnValueForGetCacheID\n}\nfunc (m *mockContainer) GetName() string {\n\treturn m.name\n}\nfunc (m *mockContainer) GetNamespace() string {\n\treturn m.namespace\n}\nfunc (m *mockContainer) UpdateState(cache.ContainerState) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetState() cache.ContainerState {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetQOSClass() v1.PodQOSClass {\n\tif len(m.returnValueForQOSClass) == 0 {\n\t\treturn v1.PodQOSGuaranteed\n\t}\n\n\treturn m.returnValueForQOSClass\n}\nfunc (m *mockContainer) GetImage() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCommand() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetArgs() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetLabelKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetLabel(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetLabels() map[string]string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetResmgrLabelKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetResmgrLabel(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetAnnotationKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetAnnotation(string, interface{}) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetResmgrAnnotationKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetResmgrAnnotation(string, interface{}) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetEffectiveAnnotation(key string) (string, bool) {\n\tpod, ok := m.GetPod()\n\tif !ok {\n\t\treturn \"\", false\n\t}\n\treturn pod.GetEffectiveAnnotation(key, m.name)\n}\nfunc (m *mockContainer) GetAnnotations() map[string]string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetEnvKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetEnv(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetMounts() []cache.Mount {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetMountByHost(string) *cache.Mount {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetMountByContainer(string) *cache.Mount {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetDevices() []cache.Device {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetDeviceByHost(string) *cache.Device {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetDeviceByContainer(string) *cache.Device {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetResourceRequirements() v1.ResourceRequirements {\n\treturn m.returnValueForGetResourceRequirements\n}\nfunc (m *mockContainer) GetLinuxResources() *criv1.LinuxContainerResources {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetCommand([]string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetArgs([]string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetLabel(string, string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) DeleteLabel(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetAnnotation(string, string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) DeleteAnnotation(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetEnv(string, string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) UnsetEnv(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) InsertMount(*cache.Mount) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) DeleteMount(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) InsertDevice(*cache.Device) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) DeleteDevice(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetTopologyHints() topology.Hints {\n\treturn topology.Hints{}\n}\nfunc (m *mockContainer) GetCPUPeriod() int64 {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCPUQuota() int64 {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCPUShares() int64 {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetMemoryLimit() int64 {\n\treturn m.memoryLimit\n}\nfunc (m *mockContainer) GetOomScoreAdj() int64 {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCpusetCpus() string {\n\treturn m.cpuset.String()\n}\nfunc (m *mockContainer) GetCpusetMems() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetLinuxResources(*criv1.LinuxContainerResources) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetCPUPeriod(int64) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetCPUQuota(int64) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetCPUShares(int64) {\n}\nfunc (m *mockContainer) SetMemoryLimit(int64) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetOomScoreAdj(int64) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetCpusetCpus(string) {\n}\nfunc (m *mockContainer) SetCpusetMems(string) {\n}\nfunc (m *mockContainer) UpdateCriCreateRequest(*criv1.CreateContainerRequest) error {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) CriUpdateRequest() (*criv1.UpdateContainerResourcesRequest, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetAffinity() ([]*cache.Affinity, error) {\n\treturn nil, nil\n}\nfunc (m *mockContainer) SetRDTClass(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetRDTClass() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetBlockIOClass(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetBlockIOClass() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetToptierLimit(int64) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetToptierLimit() int64 {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetPageMigration(*cache.PageMigrate) {\n\treturn\n}\nfunc (m *mockContainer) GetPageMigration() *cache.PageMigrate {\n\treturn nil\n}\nfunc (m *mockContainer) SetCRIRequest(req interface{}) error {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCRIRequest() (interface{}, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) ClearCRIRequest() (interface{}, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCRIEnvs() []*criv1.KeyValue {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCRIMounts() []*criv1.Mount {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCRIDevices() []*criv1.Device {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetPending() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) HasPending(string) bool {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) ClearPending(string) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetTag(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) SetTag(string, string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) DeleteTag(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) String() string {\n\treturn \"mockContainer\"\n}\nfunc (m *mockContainer) Eval(string) interface{} {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetProcesses() ([]string, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetTasks() ([]string, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockContainer) GetCgroupDir() string {\n\tpanic(\"unimplemented\")\n}\n\ntype mockPod struct {\n\tname                               string\n\treturnValueFotGetQOSClass          v1.PodQOSClass\n\treturnValue1FotGetResmgrAnnotation string\n\treturnValue2FotGetResmgrAnnotation bool\n\tcoldStartTimeout                   time.Duration\n\tcoldStartContainerName             string\n\tannotations                        map[string]string\n}\n\nfunc (m *mockPod) GetInitContainers() []cache.Container {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetContainers() []cache.Container {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetContainer(string) (cache.Container, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetID() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetUID() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetName() string {\n\treturn m.name\n}\nfunc (m *mockPod) GetNamespace() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetState() cache.PodState {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetQOSClass() v1.PodQOSClass {\n\treturn m.returnValueFotGetQOSClass\n}\nfunc (m *mockPod) GetLabelKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetLabel(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetResmgrLabelKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetResmgrLabel(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetAnnotationKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetAnnotation(string) (string, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetAnnotationObject(string, interface{}, func([]byte, interface{}) error) (bool, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetResmgrAnnotationKeys() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetResmgrAnnotation(key string) (string, bool) {\n\tif key == keyColdStartPreference && len(m.coldStartContainerName) > 0 {\n\t\treturn m.coldStartContainerName + \": { duration: \" + m.coldStartTimeout.String() + \" }\", true\n\t}\n\treturn m.returnValue1FotGetResmgrAnnotation, m.returnValue2FotGetResmgrAnnotation\n}\nfunc (m *mockPod) GetResmgrAnnotationObject(string, interface{}, func([]byte, interface{}) error) (bool, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetEffectiveAnnotation(key, container string) (string, bool) {\n\tif v, ok := m.annotations[key+\"/container.\"+container]; ok {\n\t\treturn v, true\n\t}\n\tif v, ok := m.annotations[key+\"/pod\"]; ok {\n\t\treturn v, true\n\t}\n\tv, ok := m.annotations[key]\n\treturn v, ok\n}\nfunc (m *mockPod) GetCgroupParentDir() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetPodResourceRequirements() cache.PodResourceRequirements {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetContainerAffinity(string) ([]*cache.Affinity, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) ScopeExpression() *resmgr.Expression {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) String() string {\n\treturn \"mockPod\"\n}\nfunc (m *mockPod) Eval(string) interface{} {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetProcesses(bool) ([]string, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockPod) GetTasks(bool) ([]string, error) {\n\tpanic(\"unimplemented\")\n}\n\ntype mockCache struct {\n\treturnValueForGetPolicyEntry   bool\n\treturnValue1ForLookupContainer cache.Container\n\treturnValue2ForLookupContainer bool\n}\n\nfunc (m *mockCache) InsertPod(string, interface{}, *cache.PodStatus) (cache.Pod, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) DeletePod(string) cache.Pod {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) LookupPod(string) (cache.Pod, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) InsertContainer(interface{}) (cache.Container, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) UpdateContainerID(string, interface{}) (cache.Container, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) DeleteContainer(string) cache.Container {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) LookupContainer(string) (cache.Container, bool) {\n\treturn m.returnValue1ForLookupContainer, m.returnValue2ForLookupContainer\n}\nfunc (m *mockCache) LookupContainerByCgroup(path string) (cache.Container, bool) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) GetPendingContainers() []cache.Container {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) GetPods() []cache.Pod {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) GetContainers() []cache.Container {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) GetContainerCacheIds() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) GetContainerIds() []string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) FilterScope(*resmgr.Expression) []cache.Container {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) EvaluateAffinity(*cache.Affinity) map[string]int32 {\n\treturn map[string]int32{\n\t\t\"fake key\": 1,\n\t}\n}\nfunc (m *mockCache) AddImplicitAffinities(map[string]cache.ImplicitAffinity) error {\n\treturn nil\n}\nfunc (m *mockCache) GetActivePolicy() string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) SetActivePolicy(string) error {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) ResetActivePolicy() error {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) SetPolicyEntry(string, interface{}) {\n}\nfunc (m *mockCache) GetPolicyEntry(string, interface{}) bool {\n\treturn m.returnValueForGetPolicyEntry\n}\nfunc (m *mockCache) SetConfig(*config.RawConfig) error {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) GetConfig() *config.RawConfig {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) ResetConfig() error {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) SetAdjustment(*config.Adjustment) (bool, map[string]error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) Save() error {\n\treturn nil\n}\nfunc (m *mockCache) RefreshPods(*criv1.ListPodSandboxResponse, map[string]*cache.PodStatus) ([]cache.Pod, []cache.Pod, []cache.Container) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) RefreshContainers(*criv1.ListContainersResponse) ([]cache.Container, []cache.Container) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) ContainerDirectory(string) string {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) OpenFile(string, string, os.FileMode) (*os.File, error) {\n\tpanic(\"unimplemented\")\n}\nfunc (m *mockCache) WriteFile(string, string, os.FileMode, []byte) error {\n\tpanic(\"unimplemented\")\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/node.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"fmt\"\n\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n//\n// Nodes (currently) correspond to some tangible entity in the hardware topology\n// hierarchy: full machine (virtual root in multi-socket systems), an individual\n// sockets a NUMA node. These nodes are linked into a tree resembling the topology\n// tree, with the full machine at the top, and CPU cores at the bottom. In a single\n// socket system, the virtual root is replaced with the single socket. In a single\n// NUMA node case, the single node is omitted. Also, CPU cores are not modelled as\n// nodes, instead they are properties of the nodes (as capacity and free CPU).\n//\n\n// NodeKind represents a unique node type.\ntype NodeKind string\n\nconst (\n\t// NilNode is the type of a nil node.\n\tNilNode NodeKind = \"\"\n\t// UnknownNode is the type of unknown node type.\n\tUnknownNode NodeKind = \"unknown\"\n\t// SocketNode represents a physical CPU package/socket in the system.\n\tSocketNode NodeKind = \"socket\"\n\t// DieNode represents a die within a physical CPU package/socket in the system.\n\tDieNode NodeKind = \"die\"\n\t// NumaNode represents a NUMA node in the system.\n\tNumaNode NodeKind = \"numa node\"\n\t// VirtualNode represents a virtual node, currently the root multi-socket setups.\n\tVirtualNode NodeKind = \"virtual node\"\n)\n\nconst (\n\t// OverfitPenalty is the per layer penalty for overfitting in the node tree.\n\tOverfitPenalty = 0.9\n)\n\n// Node is the abstract interface our partition tree nodes implement.\ntype Node interface {\n\t// IsNil tests if this node is nil.\n\tIsNil() bool\n\t// Name returns the name of this node.\n\tName() string\n\t// Kind returns the type of this node.\n\tKind() NodeKind\n\t// NodeID returns the (enumerated) node id of this node.\n\tNodeID() int\n\t// Parent returns the parent node of this node.\n\tParent() Node\n\t// Children returns the child nodes of this node.\n\tChildren() []Node\n\t// LinkParent sets the given node as the parent node, and appends this node as a its child.\n\tLinkParent(Node)\n\t// AddChildren appends the nodes to the children, *WITHOUT* updating their parents.\n\tAddChildren([]Node)\n\t// IsSameNode returns true if the given node is the same as this one.\n\tIsSameNode(Node) bool\n\t// IsRootNode returns true if this node has no parent.\n\tIsRootNode() bool\n\t// IsLeafNode returns true if this node has no children.\n\tIsLeafNode() bool\n\t// Get the distance of this node from the root node.\n\tRootDistance() int\n\t// Get the height of this node (inverse of depth: tree depth - node depth).\n\tNodeHeight() int\n\t// System returns the policy sysfs instance.\n\tSystem() system.System\n\t// Policy returns the policy back pointer.\n\tPolicy() *policy\n\t// DiscoverSupply\n\tDiscoverSupply(assignedNUMANodes []idset.ID) Supply\n\t// GetSupply returns the full CPU at this node.\n\tGetSupply() Supply\n\t// FreeSupply returns the available CPU supply of this node.\n\tFreeSupply() Supply\n\t// GrantedReservedCPU returns the amount of granted reserved CPU of this node and its children.\n\tGrantedReservedCPU() int\n\t// GrantedSharedCPU returns the amount of granted shared CPU of this node and its children.\n\tGrantedSharedCPU() int\n\t// GetMemset\n\tGetMemset(mtype memoryType) idset.IDSet\n\t// AssignNUMANodes assigns the given set of NUMA nodes to this one.\n\tAssignNUMANodes(ids []idset.ID)\n\t// DepthFirst traverse the tree@node calling the function at each node.\n\tDepthFirst(func(Node) error) error\n\t// BreadthFirst traverse the tree@node calling the function at each node.\n\tBreadthFirst(func(Node) error) error\n\t// Dump state of the node.\n\tDump(string, ...int)\n\t// Dump type-specific state of the node.\n\tdump(string, ...int)\n\n\tGetMemoryType() memoryType\n\tHasMemoryType(memoryType) bool\n\tGetPhysicalNodeIDs() []idset.ID\n\n\tGetScore(Request) Score\n\tHintScore(topology.Hint) float64\n}\n\n// node represents data common to all node types.\ntype node struct {\n\tpolicy   *policy     // policy back pointer\n\tself     nodeself    // upcasted/type-specific interface\n\tname     string      // node name\n\tid       int         // node id\n\tkind     NodeKind    // node type\n\tdepth    int         // node depth in the tree\n\tparent   Node        // parent node\n\tchildren []Node      // child nodes\n\tnoderes  Supply      // CPU and memory available at this node\n\tfreeres  Supply      // CPU and memory allocatable at this node\n\tmem      idset.IDSet // controllers with normal DRAM attached\n\tpMem     idset.IDSet // controllers with PMEM attached\n\thbm      idset.IDSet // controllers with HBM attached\n}\n\n// nodeself is used to 'upcast' a generic Node interface to a type-specific one.\ntype nodeself struct {\n\tnode Node\n}\n\n// socketnode represents a physical CPU package/socket in the system.\ntype socketnode struct {\n\tnode                     // common node data\n\tid     idset.ID          // NUMA node socket id\n\tsyspkg system.CPUPackage // corresponding system.Package\n}\n\n// dienode represents a die within a physical CPU package/socket in the system.\ntype dienode struct {\n\tnode                     // common node data\n\tid     idset.ID          // die id within socket\n\tsyspkg system.CPUPackage // corresponding system.Package\n}\n\n// numanode represents a NUMA node in the system.\ntype numanode struct {\n\tnode                // common node data\n\tid      idset.ID    // NUMA node system id\n\tsysnode system.Node // corresponding system.Node\n}\n\n// virtualnode represents a virtual node (ATM only the root in a multi-socket system).\ntype virtualnode struct {\n\tnode // common node data\n}\n\n// special node instance to represent a nonexistent node\nvar nilnode Node = &node{\n\tname:     \"<nil node>\",\n\tid:       -1,\n\tkind:     NilNode,\n\tdepth:    -1,\n\tchildren: nil,\n}\n\n// Init initializes the resource with common node data.\nfunc (n *node) init(p *policy, name string, kind NodeKind, parent Node) {\n\tn.policy = p\n\tn.name = name\n\tn.kind = kind\n\tn.parent = parent\n\tn.id = -1\n\n\tn.LinkParent(parent)\n\n\tn.mem = idset.NewIDSet()\n\tn.pMem = idset.NewIDSet()\n\tn.hbm = idset.NewIDSet()\n}\n\n// IsNil tests if a node\nfunc (n *node) IsNil() bool {\n\treturn n.kind == NilNode\n}\n\n// Name returns the name of this node.\nfunc (n *node) Name() string {\n\tif n.IsNil() {\n\t\treturn \"<nil node>\"\n\t}\n\treturn n.name\n}\n\n// Kind returns the kind of this node.\nfunc (n *node) Kind() NodeKind {\n\treturn n.kind\n}\n\n// NodeID returns the node id of this node.\nfunc (n *node) NodeID() int {\n\tif n.IsNil() {\n\t\treturn -1\n\t}\n\treturn n.id\n}\n\n// IsSameNode checks if the given node is that same as this one.\nfunc (n *node) IsSameNode(other Node) bool {\n\treturn n.NodeID() == other.NodeID()\n}\n\n// IsRootNode returns true if this node has no parent.\nfunc (n *node) IsRootNode() bool {\n\treturn n.parent.IsNil()\n}\n\n// IsLeafNode returns true if this node has no children.\nfunc (n *node) IsLeafNode() bool {\n\treturn len(n.children) == 0\n}\n\n// RootDistance returns the distance of this node from the root node.\nfunc (n *node) RootDistance() int {\n\tif n.IsNil() {\n\t\treturn -1\n\t}\n\treturn n.depth\n}\n\n// NodeHeight returns the hight of this node (tree depth - node depth).\nfunc (n *node) NodeHeight() int {\n\tif n.IsNil() {\n\t\treturn -1\n\t}\n\treturn n.policy.depth - n.depth\n}\n\n// Parent returns the parent of this node.\nfunc (n *node) Parent() Node {\n\tif n.IsNil() {\n\t\treturn nil\n\t}\n\n\treturn n.parent\n}\n\n// Children returns the children of this node.\nfunc (n *node) Children() []Node {\n\tif n.IsNil() {\n\t\treturn nil\n\t}\n\n\treturn n.children\n}\n\n// LinkParent sets the given node as the node parent and appends this node to the parents children.\nfunc (n *node) LinkParent(parent Node) {\n\tn.parent = parent\n\tif !parent.IsNil() {\n\t\tparent.AddChildren([]Node{n})\n\t}\n\n\tn.depth = parent.RootDistance() + 1\n}\n\n// AddChildren appends the nodes to the childres, *WITHOUT* setting their parent.\nfunc (n *node) AddChildren(nodes []Node) {\n\tn.children = append(n.children, nodes...)\n}\n\n// Dump information/state of the node.\nfunc (n *node) Dump(prefix string, level ...int) {\n\tif !log.DebugEnabled() {\n\t\treturn\n\t}\n\n\tlvl := 0\n\tif len(level) > 0 {\n\t\tlvl = level[0]\n\t}\n\tidt := indent(prefix, lvl)\n\n\tn.self.node.dump(prefix, lvl)\n\tlog.Debug(\"%s  - %s\", idt, n.noderes.DumpCapacity())\n\tlog.Debug(\"%s  - %s\", idt, n.freeres.DumpAllocatable())\n\tn.freeres.DumpMemoryState(idt + \"  \")\n\tif n.mem.Size() > 0 {\n\t\tlog.Debug(\"%s  - normal memory: %v\", idt, n.mem)\n\t}\n\tif n.hbm.Size() > 0 {\n\t\tlog.Debug(\"%s  - HBM memory: %v\", idt, n.hbm)\n\t}\n\tif n.pMem.Size() > 0 {\n\t\tlog.Debug(\"%s  - PMEM memory: %v\", idt, n.pMem)\n\t}\n\tfor _, grant := range n.policy.allocations.grants {\n\t\tcpuNodeID := grant.GetCPUNode().NodeID()\n\t\tmemNodeID := grant.GetMemoryNode().NodeID()\n\t\tswitch {\n\t\tcase cpuNodeID == n.id && memNodeID == n.id:\n\t\t\tlog.Debug(\"%s    + cpu+mem %s\", idt, grant)\n\t\tcase cpuNodeID == n.id:\n\t\t\tlog.Debug(\"%s    + cpuonly %s\", idt, grant)\n\t\tcase memNodeID == n.id:\n\t\t\tlog.Debug(\"%s    + memonly %s\", idt, grant)\n\t\t}\n\t}\n\tif !n.Parent().IsNil() {\n\t\tlog.Debug(\"%s  - parent: <%s>\", idt, n.Parent().Name())\n\t}\n\tif len(n.children) > 0 {\n\t\tlog.Debug(\"%s  - children:\", idt)\n\t\tfor _, c := range n.children {\n\t\t\tc.Dump(prefix, lvl+1)\n\t\t}\n\t}\n}\n\n// Dump type-specific information about the node.\nfunc (n *node) dump(prefix string, level ...int) {\n\tn.self.node.dump(prefix, level...)\n}\n\n// Do a depth-first traversal starting at node calling the given function at each node.\nfunc (n *node) DepthFirst(fn func(Node) error) error {\n\tfor _, c := range n.children {\n\t\tif err := c.DepthFirst(fn); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn fn(n)\n}\n\n// Do a breadth-first traversal starting at node calling the given function at each node.\nfunc (n *node) BreadthFirst(fn func(Node) error) error {\n\tif err := fn(n); err != nil {\n\t\treturn err\n\t}\n\n\tfor _, c := range n.children {\n\t\tif err := c.BreadthFirst(fn); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// System returns the policy System instance.\nfunc (n *node) System() system.System {\n\treturn n.policy.sys\n}\n\n// Policy returns the policy back pointer.\nfunc (n *node) Policy() *policy {\n\treturn n.policy\n}\n\n// GetSupply returns the full CPU supply of this node.\nfunc (n *node) GetSupply() Supply {\n\treturn n.self.node.GetSupply()\n}\n\n// Discover CPU available at this node.\nfunc (n *node) DiscoverSupply(assignedNUMANodes []idset.ID) Supply {\n\treturn n.self.node.DiscoverSupply(assignedNUMANodes)\n}\n\n// discoverSupply discovers the resource supply assigned to this pool node.\nfunc (n *node) discoverSupply(assignedNUMANodes []idset.ID) Supply {\n\tif n.noderes != nil {\n\t\treturn n.noderes.Clone()\n\t}\n\n\tif !n.IsLeafNode() {\n\t\tlog.Debug(\"%s: cumulating child resources...\", n.Name())\n\n\t\tif len(assignedNUMANodes) > 0 {\n\t\t\tlog.Fatal(\"invalid pool setup: trying to attach NUMA nodes to non-leaf node %s\",\n\t\t\t\tn.Name())\n\t\t}\n\n\t\tn.noderes = newSupply(n, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, nil, nil)\n\t\tfor _, c := range n.children {\n\t\t\tsupply := c.GetSupply()\n\t\t\tn.noderes.Cumulate(supply)\n\t\t\tn.mem.Add(c.GetMemset(memoryDRAM).Members()...)\n\t\t\tn.hbm.Add(c.GetMemset(memoryHBM).Members()...)\n\t\t\tn.pMem.Add(c.GetMemset(memoryPMEM).Members()...)\n\t\t\tlog.Debug(\"  + %s\", supply.DumpCapacity())\n\t\t}\n\t\tlog.Debug(\"  = %s\", n.noderes.DumpCapacity())\n\t} else {\n\t\tlog.Debug(\"%s: discovering attached/assigned resources...\", n.Name())\n\n\t\tmmap := createMemoryMap(0, 0, 0)\n\t\tcpus := cpuset.New()\n\n\t\tfor _, nodeID := range assignedNUMANodes {\n\t\t\tnode := n.System().Node(nodeID)\n\t\t\tnodeCPUs := node.CPUSet()\n\n\t\t\tmeminfo, err := node.MemoryInfo()\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatal(\"%s: failed to get memory info for NUMA node #%d\", n.Name(), nodeID)\n\t\t\t}\n\n\t\t\tswitch node.GetMemoryType() {\n\t\t\tcase system.MemoryTypeDRAM:\n\t\t\t\tn.mem.Add(nodeID)\n\t\t\t\tmmap.AddDRAM(meminfo.MemTotal)\n\t\t\t\tshortCPUs := cpuset.ShortCPUSet(nodeCPUs)\n\t\t\t\tlog.Debug(\"  + assigned DRAM NUMA node #%d (cpuset: %s, DRAM %.2fM)\",\n\t\t\t\t\tnodeID, shortCPUs, float64(meminfo.MemTotal)/float64(1024*1024))\n\t\t\tcase system.MemoryTypePMEM:\n\t\t\t\tn.pMem.Add(nodeID)\n\t\t\t\tmmap.AddPMEM(meminfo.MemTotal)\n\t\t\t\tlog.Debug(\"  + assigned PMEM NUMA node #%d (DRAM %.2fM)\", nodeID,\n\t\t\t\t\tfloat64(meminfo.MemTotal)/float64(1024*1024))\n\t\t\tcase system.MemoryTypeHBM:\n\t\t\t\tn.hbm.Add(nodeID)\n\t\t\t\tmmap.AddHBM(meminfo.MemTotal)\n\t\t\t\tlog.Debug(\"  + assigned HBMEM NUMA node #%d (DRAM %.2fM)\",\n\t\t\t\t\tnodeID, float64(meminfo.MemTotal)/float64(1024*1024))\n\t\t\tdefault:\n\t\t\t\tlog.Fatal(\"NUMA node #%d with unknown memory type %v\", node.GetMemoryType())\n\t\t\t}\n\n\t\t\tallowed := nodeCPUs.Intersection(n.policy.allowed)\n\t\t\tisolated := allowed.Intersection(n.policy.isolated)\n\t\t\treserved := allowed.Intersection(n.policy.reserved).Difference(isolated)\n\t\t\tsharable := allowed.Difference(isolated).Difference(reserved)\n\n\t\t\tif !reserved.IsEmpty() {\n\t\t\t\tlog.Debug(\"    allowed reserved CPUs: %s\", cpuset.ShortCPUSet(reserved))\n\t\t\t}\n\t\t\tif !sharable.IsEmpty() {\n\t\t\t\tlog.Debug(\"    allowed sharable CPUs: %s\", cpuset.ShortCPUSet(sharable))\n\t\t\t}\n\t\t\tif !isolated.IsEmpty() {\n\t\t\t\tlog.Debug(\"    allowed isolated CPUs: %s\", cpuset.ShortCPUSet(isolated))\n\t\t\t}\n\n\t\t\tcpus = cpus.Union(allowed)\n\t\t}\n\n\t\tisolated := cpus.Intersection(n.policy.isolated)\n\t\treserved := cpus.Intersection(n.policy.reserved).Difference(isolated)\n\t\tsharable := cpus.Difference(isolated).Difference(reserved)\n\t\tn.noderes = newSupply(n, isolated, reserved, sharable, 0, 0, mmap, nil)\n\t\tlog.Debug(\"  = %s\", n.noderes.DumpCapacity())\n\t}\n\n\tn.freeres = n.noderes.Clone()\n\treturn n.noderes.Clone()\n}\n\n// FreeSupply returns the available CPU supply of this node.\nfunc (n *node) FreeSupply() Supply {\n\treturn n.freeres\n}\n\n// Get the set of memory attached to this node.\nfunc (n *node) GetMemset(mtype memoryType) idset.IDSet {\n\tif n.self.node == nil { // protect against &node{}-abuse by test cases...\n\t\treturn idset.NewIDSet()\n\t}\n\treturn n.self.node.GetMemset(mtype)\n}\n\n// AssignNUMANodes assigns the given set of NUMA nodes to this one.\nfunc (n *node) AssignNUMANodes(ids []idset.ID) {\n\tn.self.node.AssignNUMANodes(ids)\n}\n\n// assignNUMANodes assigns the given set of NUMA nodes to this one.\nfunc (n *node) assignNUMANodes(ids []idset.ID) {\n\tmem := createMemoryMap(0, 0, 0)\n\n\tfor _, numaNodeID := range ids {\n\t\tif n.mem.Has(numaNodeID) || n.pMem.Has(numaNodeID) || n.hbm.Has(numaNodeID) {\n\t\t\tlog.Warn(\"*** NUMA node #%d already discovered by or assigned to %s\",\n\t\t\t\tnumaNodeID, n.Name())\n\t\t\tcontinue\n\t\t}\n\t\tnumaNode := n.policy.sys.Node(numaNodeID)\n\t\tmemTotal := uint64(0)\n\t\tif meminfo, err := numaNode.MemoryInfo(); err != nil {\n\t\t\tlog.Error(\"%s: failed to get memory info for NUMA node #%d\",\n\t\t\t\tn.Name(), numaNodeID)\n\t\t} else {\n\t\t\tmemTotal = meminfo.MemTotal\n\t\t}\n\t\tswitch numaNode.GetMemoryType() {\n\t\tcase system.MemoryTypeDRAM:\n\t\t\tmem.Add(memTotal, 0, 0)\n\t\t\tn.mem.Add(numaNodeID)\n\t\t\tlog.Info(\"*** DRAM NUMA node #%d assigned to pool node %q\",\n\t\t\t\tnumaNodeID, n.Name())\n\t\tcase system.MemoryTypePMEM:\n\t\t\tn.pMem.Add(numaNodeID)\n\t\t\tmem.Add(0, memTotal, 0)\n\t\t\tlog.Info(\"*** PMEM NUMA node #%d assigned to pool node %q\",\n\t\t\t\tnumaNodeID, n.Name())\n\t\tcase system.MemoryTypeHBM:\n\t\t\tn.hbm.Add(numaNodeID)\n\t\t\tmem.Add(0, 0, memTotal)\n\t\t\tlog.Info(\"*** HBM NUMA node #%d assigned to pool node %q\",\n\t\t\t\tnumaNodeID, n.Name())\n\t\tdefault:\n\t\t\tlog.Fatal(\"can't assign NUMA node #%d of type %v to pool node %q\",\n\t\t\t\tnumaNodeID, numaNode.GetMemoryType())\n\t\t}\n\t}\n\n\tn.noderes.AssignMemory(mem)\n\tn.freeres.AssignMemory(mem)\n}\n\n// Discover the set of memory attached to this node.\nfunc (n *node) GetPhysicalNodeIDs() []idset.ID {\n\treturn n.self.node.GetPhysicalNodeIDs()\n}\n\n// GrantedReservedCPU returns the amount of granted reserved CPU of this node and its children.\nfunc (n *node) GrantedReservedCPU() int {\n\tgrantedReserved := n.freeres.GrantedReserved()\n\tfor _, c := range n.children {\n\t\tgrantedReserved += c.GrantedReservedCPU()\n\t}\n\treturn grantedReserved\n}\n\n// GrantedSharedCPU returns the amount of granted shared CPU of this node and its children.\nfunc (n *node) GrantedSharedCPU() int {\n\tgrantedShared := n.freeres.GrantedShared()\n\tfor _, c := range n.children {\n\t\tgrantedShared += c.GrantedSharedCPU()\n\t}\n\treturn grantedShared\n}\n\n// Get Score for a cpu request.\nfunc (n *node) GetScore(req Request) Score {\n\tf := n.FreeSupply()\n\treturn f.GetScore(req)\n}\n\n// HintScore calculates the (CPU) score of the node for the given topology hint.\nfunc (n *node) HintScore(hint topology.Hint) float64 {\n\treturn n.self.node.HintScore(hint)\n}\n\nfunc (n *node) GetMemoryType() memoryType {\n\tvar memoryMask memoryType = 0x0\n\tif n.pMem.Size() > 0 {\n\t\tmemoryMask |= memoryPMEM\n\t}\n\tif n.mem.Size() > 0 {\n\t\tmemoryMask |= memoryDRAM\n\t}\n\tif n.hbm.Size() > 0 {\n\t\tmemoryMask |= memoryHBM\n\t}\n\treturn memoryMask\n}\n\nfunc (n *node) HasMemoryType(reqType memoryType) bool {\n\tnodeType := n.GetMemoryType()\n\treturn (nodeType & reqType) == reqType\n}\n\n// NewNumaNode create a node for a CPU socket.\nfunc (p *policy) NewNumaNode(id idset.ID, parent Node) Node {\n\tn := &numanode{}\n\tn.self.node = n\n\tn.node.init(p, fmt.Sprintf(\"NUMA node #%v\", id), NumaNode, parent)\n\tn.id = id\n\tn.sysnode = p.sys.Node(id)\n\n\treturn n\n}\n\n// Dump (the NUMA-specific parts of) this node.\nfunc (n *numanode) dump(prefix string, level ...int) {\n\tlog.Debug(\"%s<NUMA node #%v>\", indent(prefix, level...), n.id)\n}\n\n// Get CPU supply available at this node.\nfunc (n *numanode) GetSupply() Supply {\n\treturn n.noderes.Clone()\n}\n\nfunc (n *numanode) GetPhysicalNodeIDs() []idset.ID {\n\treturn []idset.ID{n.id}\n}\n\n// DiscoverSupply discovers the CPU supply available at this node.\nfunc (n *numanode) DiscoverSupply(assignedNUMANodes []idset.ID) Supply {\n\treturn n.node.discoverSupply(assignedNUMANodes)\n}\n\n// GetMemset returns the set of memory attached to this node.\nfunc (n *numanode) GetMemset(mtype memoryType) idset.IDSet {\n\tmset := idset.NewIDSet()\n\n\tif mtype&memoryDRAM != 0 {\n\t\tmset.Add(n.mem.Members()...)\n\t}\n\tif mtype&memoryHBM != 0 {\n\t\tmset.Add(n.hbm.Members()...)\n\t}\n\tif mtype&memoryPMEM != 0 {\n\t\tmset.Add(n.pMem.Members()...)\n\t}\n\n\treturn mset\n}\n\n// AssignNUMANodes assigns the given NUMA nodes to this one.\nfunc (n *numanode) AssignNUMANodes(ids []idset.ID) {\n\tn.node.assignNUMANodes(ids)\n}\n\n// HintScore calculates the (CPU) score of the node for the given topology hint.\nfunc (n *numanode) HintScore(hint topology.Hint) float64 {\n\tswitch {\n\tcase hint.CPUs != \"\":\n\t\treturn cpuHintScore(hint, n.sysnode.CPUSet())\n\n\tcase hint.NUMAs != \"\":\n\t\treturn numaHintScore(hint, n.id)\n\n\tcase hint.Sockets != \"\":\n\t\tpkgID := n.sysnode.PackageID()\n\t\tscore := socketHintScore(hint, n.sysnode.PackageID())\n\t\tif score > 0.0 {\n\t\t\t// penalize underfit reciprocally (inverse-proportionally) to the socket size\n\t\t\tscore /= float64(len(n.System().Package(pkgID).NodeIDs()))\n\t\t}\n\t\treturn score\n\t}\n\n\treturn 0.0\n}\n\n// NewDieNode create a node for a CPU die.\nfunc (p *policy) NewDieNode(id idset.ID, parent Node) Node {\n\tpkg := parent.(*socketnode)\n\tn := &dienode{}\n\tn.self.node = n\n\tn.node.init(p, fmt.Sprintf(\"die #%v/%v\", pkg.id, id), DieNode, parent)\n\tn.id = id\n\tn.syspkg = p.sys.Package(pkg.id)\n\n\treturn n\n}\n\n// Dump (the die-specific parts of) this node.\nfunc (n *dienode) dump(prefix string, level ...int) {\n\tlog.Debug(\"%s<die #%v/%v>\", indent(prefix, level...), n.syspkg.ID(), n.id)\n}\n\n// Get CPU supply available at this node.\nfunc (n *dienode) GetSupply() Supply {\n\treturn n.noderes.Clone()\n}\n\nfunc (n *dienode) GetPhysicalNodeIDs() []idset.ID {\n\tids := make([]idset.ID, 0)\n\tids = append(ids, n.id)\n\tfor _, c := range n.children {\n\t\tcIds := c.GetPhysicalNodeIDs()\n\t\tids = append(ids, cIds...)\n\t}\n\treturn ids\n}\n\n// DiscoverSupply discovers the CPU supply available at this die.\nfunc (n *dienode) DiscoverSupply(assignedNUMANodes []idset.ID) Supply {\n\treturn n.node.discoverSupply(assignedNUMANodes)\n}\n\n// GetMemset returns the set of memory attached to this die.\nfunc (n *dienode) GetMemset(mtype memoryType) idset.IDSet {\n\tmset := idset.NewIDSet()\n\n\tif mtype&memoryDRAM != 0 {\n\t\tmset.Add(n.mem.Members()...)\n\t}\n\tif mtype&memoryHBM != 0 {\n\t\tmset.Add(n.hbm.Members()...)\n\t}\n\tif mtype&memoryPMEM != 0 {\n\t\tmset.Add(n.pMem.Members()...)\n\t}\n\n\treturn mset\n}\n\n// AssignNUMANodes assigns the given NUMA nodes to this one.\nfunc (n *dienode) AssignNUMANodes(ids []idset.ID) {\n\tn.node.assignNUMANodes(ids)\n}\n\n// HintScore calculates the (CPU) score of the node for the given topology hint.\nfunc (n *dienode) HintScore(hint topology.Hint) float64 {\n\tswitch {\n\tcase hint.CPUs != \"\":\n\t\treturn cpuHintScore(hint, n.syspkg.CPUSet())\n\n\tcase hint.NUMAs != \"\":\n\t\treturn OverfitPenalty * dieHintScore(hint, n.id, n.syspkg)\n\n\tcase hint.Sockets != \"\":\n\t\tscore := socketHintScore(hint, n.syspkg.ID())\n\t\tif score > 0.0 {\n\t\t\t// penalize underfit reciprocally (inverse-proportionally) to the socket size in dies\n\t\t\tscore /= float64(len(n.syspkg.DieNodeIDs(n.id)))\n\t\t}\n\t\treturn score\n\t}\n\n\treturn 0.0\n}\n\n// NewSocketNode create a node for a CPU socket.\nfunc (p *policy) NewSocketNode(id idset.ID, parent Node) Node {\n\tn := &socketnode{}\n\tn.self.node = n\n\tn.node.init(p, fmt.Sprintf(\"socket #%v\", id), SocketNode, parent)\n\tn.id = id\n\tn.syspkg = p.sys.Package(id)\n\n\treturn n\n}\n\n// Dump (the socket-specific parts of) this node.\nfunc (n *socketnode) dump(prefix string, level ...int) {\n\tlog.Debug(\"%s<socket #%v>\", indent(prefix, level...), n.id)\n}\n\n// Get CPU supply available at this node.\nfunc (n *socketnode) GetSupply() Supply {\n\treturn n.noderes.Clone()\n}\n\nfunc (n *socketnode) GetPhysicalNodeIDs() []idset.ID {\n\tids := make([]idset.ID, 0)\n\tids = append(ids, n.id)\n\tfor _, c := range n.children {\n\t\tcIds := c.GetPhysicalNodeIDs()\n\t\tids = append(ids, cIds...)\n\t}\n\treturn ids\n}\n\n// DiscoverSupply discovers the CPU supply available at this socket.\nfunc (n *socketnode) DiscoverSupply(assignedNUMANodes []idset.ID) Supply {\n\treturn n.node.discoverSupply(assignedNUMANodes)\n}\n\n// GetMemset returns the set of memory attached to this socket.\nfunc (n *socketnode) GetMemset(mtype memoryType) idset.IDSet {\n\tmset := idset.NewIDSet()\n\n\tif mtype&memoryDRAM != 0 {\n\t\tmset.Add(n.mem.Members()...)\n\t}\n\tif mtype&memoryHBM != 0 {\n\t\tmset.Add(n.hbm.Members()...)\n\t}\n\tif mtype&memoryPMEM != 0 {\n\t\tmset.Add(n.pMem.Members()...)\n\t}\n\n\treturn mset\n}\n\n// AssignNUMANodes assigns the given NUMA nodes to this one.\nfunc (n *socketnode) AssignNUMANodes(ids []idset.ID) {\n\tn.node.assignNUMANodes(ids)\n}\n\n// HintScore calculates the (CPU) score of the node for the given topology hint.\nfunc (n *socketnode) HintScore(hint topology.Hint) float64 {\n\tswitch {\n\tcase hint.CPUs != \"\":\n\t\treturn cpuHintScore(hint, n.syspkg.CPUSet())\n\n\tcase hint.NUMAs != \"\":\n\t\treturn OverfitPenalty * numaHintScore(hint, n.syspkg.NodeIDs()...)\n\n\tcase hint.Sockets != \"\":\n\t\treturn socketHintScore(hint, n.id)\n\t}\n\n\treturn 0.0\n}\n\n// NewVirtualNode creates a new virtual node.\nfunc (p *policy) NewVirtualNode(name string, parent Node) Node {\n\tn := &virtualnode{}\n\tn.self.node = n\n\tn.node.init(p, fmt.Sprintf(\"%s\", name), VirtualNode, parent)\n\n\treturn n\n}\n\n// Dump (the virtual-node specific parts of) this node.\nfunc (n *virtualnode) dump(prefix string, level ...int) {\n\tlog.Debug(\"%s<virtual %s>\", indent(prefix, level...), n.name)\n}\n\n// Get CPU supply available at this node.\nfunc (n *virtualnode) GetSupply() Supply {\n\treturn n.noderes.Clone()\n}\n\n// DiscoverSupply discovers the CPU supply available at this node.\nfunc (n *virtualnode) DiscoverSupply(assignedNUMANodes []idset.ID) Supply {\n\treturn n.node.discoverSupply(assignedNUMANodes)\n}\n\n// GetMemset returns the set of memory attached to this socket.\nfunc (n *virtualnode) GetMemset(mtype memoryType) idset.IDSet {\n\tmset := idset.NewIDSet()\n\n\tif mtype&memoryDRAM != 0 {\n\t\tmset.Add(n.mem.Members()...)\n\t}\n\tif mtype&memoryHBM != 0 {\n\t\tmset.Add(n.hbm.Members()...)\n\t}\n\tif mtype&memoryPMEM != 0 {\n\t\tmset.Add(n.pMem.Members()...)\n\t}\n\n\treturn mset\n}\n\n// AssignNUMANodes assigns the given NUMA nodes to this one.\nfunc (n *virtualnode) AssignNUMANodes(ids []idset.ID) {\n\tlog.Panic(\"cannot assign NUMA nodes #%s to %s\",\n\t\tidset.NewIDSet(ids...).String(), n.Name())\n}\n\n// HintScore calculates the (CPU) score of the node for the given topology hint.\nfunc (n *virtualnode) HintScore(hint topology.Hint) float64 {\n\t// don't bother calculating any scores, the root should always score 1.0\n\tswitch {\n\tcase hint.CPUs != \"\":\n\t\treturn cpuHintScore(hint, n.System().CPUSet())\n\n\tcase hint.NUMAs != \"\":\n\t\treturn OverfitPenalty * OverfitPenalty\n\n\tcase hint.Sockets != \"\":\n\t\treturn OverfitPenalty\n\t}\n\n\treturn 0.0\n}\n\nfunc (n *virtualnode) GetPhysicalNodeIDs() []idset.ID {\n\tids := make([]idset.ID, 0)\n\tfor _, c := range n.children {\n\t\tcIds := c.GetPhysicalNodeIDs()\n\t\tids = append(ids, cIds...)\n\t}\n\treturn ids\n}\n\n// Finalize the setup of nilnode.\nfunc init() {\n\tnilnode.(*node).self.node = nilnode\n\tnilnode.(*node).parent = nilnode.(*node).self.node\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/pod-preferences.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"sigs.k8s.io/yaml\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n)\n\nconst (\n\t// annotation key for opting in to multiple isolated exclusive CPUs per container.\n\tkeyIsolationPreference = \"prefer-isolated-cpus\"\n\t// annotation key for opting out of exclusive allocation and relaxed topology fitting.\n\tkeySharedCPUPreference = \"prefer-shared-cpus\"\n\t// annotation key for type of memory to allocate\n\tkeyMemoryTypePreference = \"memory-type\"\n\t// annotation key for type \"cold start\" of workloads\n\tkeyColdStartPreference = \"cold-start\"\n\t// annotation key for reserved pools\n\tkeyReservedCPUsPreference = \"prefer-reserved-cpus\"\n\n\t// effective annotation key for isolated CPU preference\n\tpreferIsolatedCPUsKey = keyIsolationPreference + \".\" + kubernetes.ResmgrKeyNamespace\n\t// effective annotation key for shared CPU preference\n\tpreferSharedCPUsKey = keySharedCPUPreference + \".\" + kubernetes.ResmgrKeyNamespace\n\t// effective annotation key for memory type preference\n\tpreferMemoryTypeKey = keyMemoryTypePreference + \".\" + kubernetes.ResmgrKeyNamespace\n\t// effective annotation key for \"cold start\" preference\n\tpreferColdStartKey = keyColdStartPreference + \".\" + kubernetes.ResmgrKeyNamespace\n\t// annotation key for reserved pools\n\tpreferReservedCPUsKey = keyReservedCPUsPreference + \".\" + kubernetes.ResmgrKeyNamespace\n)\n\n// cpuClass is a type of CPU to allocate\ntype cpuClass int\n\n// names by cpu class\nvar cpuClassNames = map[cpuClass]string{\n\tcpuNormal:   \"normal\",\n\tcpuReserved: \"reserved\",\n}\n\nconst (\n\tcpuNormal cpuClass = iota\n\tcpuReserved\n)\n\n// types by memory type name\nvar memoryNamedTypes = map[string]memoryType{\n\t\"dram\":  memoryDRAM,\n\t\"pmem\":  memoryPMEM,\n\t\"hbm\":   memoryHBM,\n\t\"mixed\": memoryAll,\n}\n\n// names by memory type\nvar memoryTypeNames = map[memoryType]string{\n\tmemoryDRAM: \"DRAM\",\n\tmemoryPMEM: \"PMEM\",\n\tmemoryHBM:  \"HBM\",\n}\n\n// memoryType is bitmask of types of memory to allocate\ntype memoryType int\n\n// memoryType bits\nconst (\n\tmemoryUnspec memoryType = (0x1 << iota) >> 1\n\tmemoryDRAM\n\tmemoryPMEM\n\tmemoryHBM\n\tmemoryFirstUnusedBit\n\tmemoryAll = memoryFirstUnusedBit - 1\n\n\t// type of memory to use if none specified\n\tdefaultMemoryType = memoryAll\n)\n\n// isolatedCPUsPreference returns whether isolated CPUs should be preferred for\n// containers that allocate multiple CPUs, and if the container was explicitly\n// annotated with this setting.\n//\n// If the effective annotations are not found, this function falls back to\n// looking for the deprecated syntax by calling podIsolationPreference.\nfunc isolatedCPUsPreference(pod cache.Pod, container cache.Container) (bool, bool) {\n\tkey := preferIsolatedCPUsKey\n\tvalue, ok := pod.GetEffectiveAnnotation(key, container.GetName())\n\tif !ok {\n\t\treturn podIsolationPreference(pod, container)\n\t}\n\n\tpreference, err := strconv.ParseBool(value)\n\tif err != nil {\n\t\tlog.Error(\"invalid CPU isolation preference annotation (%q, %q): %v\",\n\t\t\tkey, value, err)\n\t\treturn opt.PreferIsolated, false\n\t}\n\n\tlog.Debug(\"%s: effective CPU isolation preference %v\", container.PrettyName(), preference)\n\n\treturn preference, true\n}\n\n// sharedCPUsPreference returns whether shared CPUs should be preferred for\n// containers otherwise eligible for exclusive allocation, and whether the\n// container was explicitly annotated with this setting.\n//\n// If the effective annotations are not found, this function falls back to\n// looking for the deprecated syntax by calling podSharedCPUPreference.\nfunc sharedCPUsPreference(pod cache.Pod, container cache.Container) (bool, bool) {\n\tkey := preferSharedCPUsKey\n\tvalue, ok := pod.GetEffectiveAnnotation(key, container.GetName())\n\tif !ok {\n\t\treturn podSharedCPUPreference(pod, container)\n\t}\n\n\tpreference, err := strconv.ParseBool(value)\n\tif err != nil {\n\t\tlog.Error(\"invalid shared CPU preference annotation (%q, %q): %v\",\n\t\t\tkey, value, err)\n\t\treturn opt.PreferShared, false\n\t}\n\n\tlog.Debug(\"%s: effective shared CPU preference %v\", container.PrettyName(), preference)\n\n\treturn preference, true\n}\n\n// memoryTypePreference returns what type of memory should be allocated for the container.\n//\n// If the effective annotations are not found, this function falls back to\n// looking for the deprecated syntax by calling podMemoryTypePreference.\nfunc memoryTypePreference(pod cache.Pod, container cache.Container) memoryType {\n\tkey := preferMemoryTypeKey\n\tvalue, ok := pod.GetEffectiveAnnotation(key, container.GetName())\n\tif !ok {\n\t\treturn podMemoryTypePreference(pod, container)\n\t}\n\n\tmtype, err := parseMemoryType(value)\n\tif err != nil {\n\t\tlog.Error(\"invalid memory type preference (%q, %q): %v\", key, value, err)\n\t\treturn memoryUnspec\n\t}\n\n\tlog.Debug(\"%s: effective cold start preference %v\", container.PrettyName(), mtype)\n\n\treturn mtype\n}\n\n// coldStartPreference figures out 'cold start' preferences for the container, IOW\n// if the container memory should be allocated for an initial 'cold start' period\n// from PMEM, and how long this initial period should be.\n//\n// If the effective annotations are not found, this function falls back to\n// looking for the deprecated syntax by calling podColdStartPreference.\nfunc coldStartPreference(pod cache.Pod, container cache.Container) (ColdStartPreference, error) {\n\tkey := preferColdStartKey\n\tvalue, ok := pod.GetEffectiveAnnotation(key, container.GetName())\n\tif !ok {\n\t\treturn podColdStartPreference(pod, container)\n\t}\n\n\tpreference := ColdStartPreference{}\n\tif err := yaml.Unmarshal([]byte(value), &preference); err != nil {\n\t\tlog.Error(\"failed to parse cold start preference (%q, %q): %v\",\n\t\t\tkeyColdStartPreference, value, err)\n\t\treturn ColdStartPreference{}, policyError(\"invalid cold start preference %q: %v\",\n\t\t\tvalue, err)\n\t}\n\n\tif preference.Duration < 0 || time.Duration(preference.Duration) > time.Hour {\n\t\treturn ColdStartPreference{}, policyError(\"cold start duration %s out of range\",\n\t\t\tpreference.Duration.String())\n\t}\n\n\tlog.Debug(\"%s: effective cold start preference %v\",\n\t\tcontainer.PrettyName(), preference.Duration.String())\n\n\treturn preference, nil\n}\n\n// podIsolationPreference checks if containers explicitly prefers to run on multiple isolated CPUs.\n// The first return value indicates whether the container is isolated or not.\n// The second return value indicates whether that decision was explicit (true) or implicit (false).\nfunc podIsolationPreference(pod cache.Pod, container cache.Container) (bool, bool) {\n\tkey := keyIsolationPreference\n\tvalue, ok := pod.GetResmgrAnnotation(key)\n\tif !ok {\n\t\treturn opt.PreferIsolated, false\n\t}\n\n\tlog.Warn(\"WARNING: using deprecated annotation %q\", key)\n\tlog.Warn(\"WARNING: consider using instead\")\n\tlog.Warn(\"WARNING:     %q, or\", preferIsolatedCPUsKey+\"/container.\"+container.GetName())\n\tlog.Warn(\"WARNING:     %q\", preferIsolatedCPUsKey+\"/pod\")\n\n\tif value == \"false\" || value == \"true\" {\n\t\treturn (value[0] == 't'), true\n\t}\n\n\tpreferences := map[string]bool{}\n\tif err := yaml.Unmarshal([]byte(value), &preferences); err != nil {\n\t\tlog.Error(\"failed to parse isolation preference %s = '%s': %v\",\n\t\t\tkeyIsolationPreference, value, err)\n\t\treturn opt.PreferIsolated, false\n\t}\n\n\tname := container.GetName()\n\tif pref, ok := preferences[name]; ok {\n\t\tlog.Debug(\"%s per-container isolation preference '%v'\", name, pref)\n\t\treturn pref, true\n\t}\n\n\tlog.Debug(\"%s defaults to isolation preference '%v'\", name, opt.PreferIsolated)\n\treturn opt.PreferIsolated, false\n}\n\n// podSharedCPUPreference checks if a container wants to opt-out from exclusive allocation.\n// The first return value indicates if the container prefers to opt-out from\n// exclusive (sliced-off or isolated) CPU allocation even if it was otherwise\n// eligible for it.\nfunc podSharedCPUPreference(pod cache.Pod, container cache.Container) (bool, bool) {\n\tkey := keySharedCPUPreference\n\tvalue, ok := pod.GetResmgrAnnotation(key)\n\tif !ok {\n\t\treturn opt.PreferShared, false\n\t}\n\n\tlog.Warn(\"WARNING: using deprecated annotation %q\", key)\n\tlog.Warn(\"WARNING: consider using instead\")\n\tlog.Warn(\"WARNING:     %q, or\", preferSharedCPUsKey+\"/container.\"+container.GetName())\n\tlog.Warn(\"WARNING:     %q\", preferSharedCPUsKey+\"/pod\")\n\n\tif value == \"false\" || value == \"true\" {\n\t\treturn value[0] == 't', true\n\t}\n\n\tpreferences := map[string]string{}\n\tif err := yaml.Unmarshal([]byte(value), &preferences); err != nil {\n\t\tlog.Error(\"failed to parse shared CPU preference %s = '%s': %v\",\n\t\t\tkeySharedCPUPreference, value, err)\n\t\treturn opt.PreferShared, false\n\t}\n\n\tname := container.GetName()\n\tpref, ok := preferences[name]\n\tif !ok {\n\t\treturn opt.PreferShared, false\n\t}\n\tif pref == \"false\" || pref == \"true\" {\n\t\treturn pref[0] == 't', true\n\t}\n\n\tlog.Error(\"invalid shared CPU boolean preference for container %s: %s\", name, pref)\n\treturn opt.PreferShared, false\n}\n\n// ColdStartPreference lists the various ways the container can be configured to trigger\n// cold start. Currently, only timer is supported. If the \"duration\" is set to a duration\n// greater than 0, cold start is enabled and the DRAM controller is added to the container\n// after the duration has passed.\ntype ColdStartPreference struct {\n\tDuration config.Duration // `json:\"duration,omitempty\"`\n}\n\n// podColdStartPreference figures out if the container memory should be first allocated from PMEM.\n// It returns the time (in milliseconds) after which DRAM controller should be added to the mix.\nfunc podColdStartPreference(pod cache.Pod, container cache.Container) (ColdStartPreference, error) {\n\tkey := keyColdStartPreference\n\tvalue, ok := pod.GetResmgrAnnotation(key)\n\tif !ok {\n\t\treturn ColdStartPreference{}, nil\n\t}\n\n\tlog.Warn(\"WARNING: using deprecated annotation %q\", key)\n\tlog.Warn(\"WARNING: consider using instead\")\n\tlog.Warn(\"WARNING:     %q, or\", preferColdStartKey+\"/container.\"+container.GetName())\n\tlog.Warn(\"WARNING:     %q\", preferColdStartKey+\"/pod\")\n\n\tpreferences := map[string]ColdStartPreference{}\n\tif err := yaml.Unmarshal([]byte(value), &preferences); err != nil {\n\t\tlog.Error(\"failed to parse cold start preference %s = '%s': %v\",\n\t\t\tkey, value, err)\n\t\treturn ColdStartPreference{}, err\n\t}\n\tname := container.GetName()\n\tpreference, ok := preferences[name]\n\tif !ok {\n\t\tlog.Debug(\"container %s has no entry among cold start preferences\", container.PrettyName())\n\t\treturn ColdStartPreference{}, nil\n\t}\n\n\tif preference.Duration < 0 || time.Duration(preference.Duration) > time.Hour {\n\t\t// Duration can't be negative. We also reject durations which are longer than one hour.\n\t\treturn ColdStartPreference{}, fmt.Errorf(\"failed to validate cold start timeout %s: value out of scope\", preference.Duration.String())\n\t}\n\n\treturn preference, nil\n}\n\nfunc checkReservedPoolNamespaces(namespace string) bool {\n\tif namespace == metav1.NamespaceSystem {\n\t\treturn true\n\t}\n\n\tfor _, str := range opt.ReservedPoolNamespaces {\n\t\tret, err := filepath.Match(str, namespace)\n\t\tif err != nil {\n\t\t\treturn false\n\t\t}\n\n\t\tif ret {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc checkReservedCPUsAnnotations(c cache.Container) (bool, bool) {\n\thintSetting, ok := c.GetEffectiveAnnotation(preferReservedCPUsKey)\n\tif !ok {\n\t\treturn false, false\n\t}\n\n\tpreference, err := strconv.ParseBool(hintSetting)\n\tif err != nil {\n\t\tlog.Error(\"failed to parse reserved CPU preference %s = '%s': %v\",\n\t\t\tkeyReservedCPUsPreference, hintSetting, err)\n\t\treturn false, false\n\t}\n\n\treturn preference, true\n}\n\n// cpuAllocationPreferences figures out the amount and kind of CPU to allocate.\n// Returned values:\n// 1. full: number of full CPUs\n// 2. fraction: amount of fractional CPU in milli-CPU\n// 3. isolate: (bool) whether to prefer isolated full CPUs\n// 4. cpuType: (cpuClass) class of CPU to allocate (reserved vs. normal)\nfunc cpuAllocationPreferences(pod cache.Pod, container cache.Container) (int, int, bool, cpuClass) {\n\t//\n\t// CPU allocation preferences for a container consist of\n\t//\n\t//   - the number of exclusive cores to allocate\n\t//   - the amount of fractional cores to allocate (in milli-CPU)\n\t//   - whether kernel-isolated cores are preferred for exclusive allocation\n\t//   - cpu class IOW, whether reserved or normal cores should be allocated\n\t//\n\t// The rules for determining these preferences are:\n\t//\n\t//   - reserved cores are only and always preferred for kube-system namespace containers\n\t//   - kube-system namespace containers:\n\t//       => fractional/shared (reserved) cores\n\t//   - BestEffort QoS class containers:\n\t//       => fractional/shared cores\n\t//   - Burstable QoS class containers:\n\t//       => fractional/shared cores\n\t//   - Guaranteed QoS class containers:\n\t//      - 1 full core > CPU request\n\t//          => fractional/shared cores\n\t//      - 1 full core <= CPU request < 2 full cores:\n\t//          a. fractional allocation:\n\t//            - shared preference explicitly annotated/configured false:\n\t//              => mixed cores, prefer isolated, unless annotated/configured otherwise (*)\n\t//            - shared preference explicitly annotated/configured true:\n\t//              => shared cores\n\t//          b. non-fractional allocation:\n\t//            - shared preference explicitly annotated true:\n\t//              => shared cores\n\t//            - isolated default preference false or explicitly annotated false:\n\t//              => exclusive cores\n\t//            - isolated default preference true or explicitly annotated true:\n\t//              => exclusive cores, prefer isolated (*)\n\t//      - 2 full cores <= CPU request\n\t//          a. fractional allocation:\n\t//            - shared preference explicitly annotated false:\n\t//              => mixed cores, prefer isolated only if explicitly annotated (**)\n\t//            - otherwise (no shared annotation):\n\t//              => shared cores\n\t//          b. non-fractional allocation:\n\t//            - shared preference explicitly annotated true:\n\t//              => shared cores\n\t//            - otherwise (no shared annotation):\n\t//              => exclusive cores, prefer isolated only if explicitly annotated (**)\n\t//\n\t//   - Rationale for isolation defaults:\n\t//     *)\n\t//        In the single core case, a workload does not need to do anything extra to\n\t//        benefit from running on isolated vs. ordinary exclusive cores. Therefore,\n\t//        allocating isolated cores is a safe default choice.\n\t//     **)\n\t//        In the multiple cores case, a workload needs to be 'isolation-aware' to\n\t//        benefit (or actually to not even get hindered) by running on isolated vs.\n\t//        ordinary exclusive cores. If it gets isolated cores allocated, it needs\n\t//        to actively spread itself/its correct processes over the cores, because\n\t//        the scheduler is not going to do load-balancing for it. Therefore, the\n\t//        safe choice in this case is to not allocate isolated cores by default.\n\t//\n\n\tnamespace := container.GetNamespace()\n\trequest := container.GetResourceRequirements().Requests[corev1.ResourceCPU]\n\tqosClass := pod.GetQOSClass()\n\tfraction := int(request.MilliValue())\n\n\t// easy cases: kube-system namespace, Burstable or BestEffort QoS class containers\n\tpreferReserved, explicitReservation := checkReservedCPUsAnnotations(container)\n\tswitch {\n\tcase preferReserved == true:\n\t\treturn 0, fraction, false, cpuReserved\n\tcase checkReservedPoolNamespaces(namespace) && !explicitReservation:\n\t\treturn 0, fraction, false, cpuReserved\n\tcase qosClass == corev1.PodQOSBurstable:\n\t\treturn 0, fraction, false, cpuNormal\n\tcase qosClass == corev1.PodQOSBestEffort:\n\t\treturn 0, 0, false, cpuNormal\n\t}\n\n\t// complex case: Guaranteed QoS class containers\n\tcores := fraction / 1000\n\tfraction = fraction % 1000\n\tpreferIsolated, explicitIsolated := isolatedCPUsPreference(pod, container)\n\tpreferShared, explicitShared := sharedCPUsPreference(pod, container)\n\n\tswitch {\n\t// sub-core CPU request\n\tcase cores == 0:\n\t\treturn 0, fraction, false, cpuNormal\n\t\t// 1 <= CPU request < 2\n\tcase cores < 2:\n\t\t// fractional allocation, potentially mixed\n\t\tif fraction > 0 {\n\t\t\tif preferShared {\n\t\t\t\treturn 0, 1000*cores + fraction, false, cpuNormal\n\t\t\t}\n\t\t\treturn cores, fraction, preferIsolated, cpuNormal\n\t\t}\n\t\t// non-fractional allocation\n\t\tif preferShared && explicitShared {\n\t\t\treturn 0, 1000*cores + fraction, false, cpuNormal\n\t\t}\n\t\treturn cores, fraction, preferIsolated, cpuNormal\n\t\t// CPU request >= 2\n\tdefault:\n\t\t// fractional allocation, only mixed if explicitly annotated as unshared\n\t\tif fraction > 0 {\n\t\t\tif !preferShared && explicitShared {\n\t\t\t\treturn cores, fraction, preferIsolated && explicitIsolated, cpuNormal\n\t\t\t}\n\t\t\treturn 0, 1000*cores + fraction, false, cpuNormal\n\t\t}\n\t\t// non-fractional allocation\n\t\tif preferShared && explicitShared {\n\t\t\treturn 0, 1000 * cores, false, cpuNormal\n\t\t}\n\t\treturn cores, fraction, preferIsolated && explicitIsolated, cpuNormal\n\t}\n}\n\n// podMemoryTypePreference returns what type of memory should be allocated for the container.\nfunc podMemoryTypePreference(pod cache.Pod, c cache.Container) memoryType {\n\tkey := keyMemoryTypePreference\n\tvalue, ok := pod.GetResmgrAnnotation(key)\n\tif !ok {\n\t\tlog.Debug(\"pod %s has no memory preference annotations\", pod.GetName())\n\t\treturn memoryUnspec\n\t}\n\n\tlog.Warn(\"WARNING: using deprecated annotation %q\", key)\n\tlog.Warn(\"WARNING: consider using instead\")\n\tlog.Warn(\"WARNING:     %q, or\", keyMemoryTypePreference+\"/container.\"+c.GetName())\n\tlog.Warn(\"WARNING:     %q\", keyMemoryTypePreference+\"/pod\")\n\n\t// Try to parse as per-container preference. Assume common for all containers if fails.\n\tpref := \"\"\n\tpreferences := map[string]string{}\n\tif err := yaml.Unmarshal([]byte(value), &preferences); err == nil {\n\t\tname := c.GetName()\n\t\tp, ok := preferences[name]\n\t\tif !ok {\n\t\t\tlog.Debug(\"container %s has no entry among memory preferences\", c.PrettyName())\n\t\t\treturn memoryUnspec\n\t\t}\n\t\tpref = p\n\t} else {\n\t\tpref = value\n\t}\n\n\tmtype, err := parseMemoryType(pref)\n\tif err != nil {\n\t\tlog.Error(\"invalid memory type preference ('%s') in annotation %s: %v\",\n\t\t\tpref, keyMemoryTypePreference, err)\n\t\treturn memoryUnspec\n\t}\n\tlog.Debug(\"container %s has effective memory preference: %s\", c.PrettyName(), mtype)\n\treturn mtype\n}\n\n// memoryAllocationPreference returns the amount and kind of memory to allocate.\nfunc memoryAllocationPreference(pod cache.Pod, c cache.Container) (uint64, uint64, memoryType) {\n\tresources := c.GetResourceRequirements()\n\tmtype := memoryTypePreference(pod, c)\n\treq, lim := uint64(0), uint64(0)\n\n\tif memReq, ok := resources.Requests[corev1.ResourceMemory]; ok {\n\t\treq = uint64(memReq.Value())\n\t}\n\tif memLim, ok := resources.Limits[corev1.ResourceMemory]; ok {\n\t\tlim = uint64(memLim.Value())\n\t}\n\n\treturn req, lim, mtype\n}\n\n// String stringifies a cpuClass.\nfunc (t cpuClass) String() string {\n\tif cpuClassName, ok := cpuClassNames[t]; ok {\n\t\treturn cpuClassName\n\t}\n\treturn fmt.Sprintf(\"#UNNAMED-CPUCLASS(%d)\", int(t))\n}\n\n// String stringifies a memoryType.\nfunc (t memoryType) String() string {\n\tstr := \"\"\n\tsep := \"\"\n\tfor _, bit := range []memoryType{memoryDRAM, memoryPMEM, memoryHBM} {\n\t\tif int(t)&int(bit) != 0 {\n\t\t\tstr += sep + memoryTypeNames[bit]\n\t\t\tsep = \",\"\n\t\t}\n\t}\n\treturn str\n}\n\n// parseMemoryType parses a memory type string, ideally produced by String()\nfunc parseMemoryType(value string) (memoryType, error) {\n\tif value == \"\" {\n\t\treturn memoryUnspec, nil\n\t}\n\tmtype := 0\n\tfor _, typestr := range strings.Split(value, \",\") {\n\t\tt, ok := memoryNamedTypes[strings.ToLower(typestr)]\n\t\tif !ok {\n\t\t\treturn memoryUnspec, policyError(\"unknown memory type value '%s'\", typestr)\n\t\t}\n\t\tmtype |= int(t)\n\t}\n\treturn memoryType(mtype), nil\n}\n\n// MarshalJSON is the JSON marshaller for memoryType.\nfunc (t memoryType) MarshalJSON() ([]byte, error) {\n\tvalue := t.String()\n\treturn json.Marshal(value)\n}\n\n// UnmarshalJSON is the JSON unmarshaller for memoryType\nfunc (t *memoryType) UnmarshalJSON(data []byte) error {\n\tival := 0\n\tif err := json.Unmarshal(data, &ival); err == nil {\n\t\t*t = memoryType(ival)\n\t\treturn nil\n\t}\n\n\tvalue := \"\"\n\tif err := json.Unmarshal(data, &value); err != nil {\n\t\treturn policyError(\"failed to unmarshal memoryType '%s': %v\",\n\t\t\tstring(data), err)\n\t}\n\n\tmtype, err := parseMemoryType(value)\n\tif err != nil {\n\t\treturn policyError(\"failed parse memoryType '%s': %v\", value, err)\n\t}\n\n\t*t = mtype\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/pod-preferences_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"testing\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nfunc TestPodIsolationPreference(t *testing.T) {\n\ttcases := []struct {\n\t\tname             string\n\t\tpod              *mockPod\n\t\tcontainer        *mockContainer\n\t\texpectedIsolate  bool\n\t\texpectedExplicit bool\n\t\tdisabled         bool\n\t}{\n\t\t{\n\t\t\tname:     \"podIsolationPreference() should handle nil pod arg gracefully\",\n\t\t\tdisabled: true,\n\t\t},\n\t\t{\n\t\t\tname:            \"return defaults\",\n\t\t\tpod:             &mockPod{},\n\t\t\tcontainer:       &mockContainer{},\n\t\t\texpectedIsolate: opt.PreferIsolated,\n\t\t},\n\t\t{\n\t\t\tname: \"prefer resmgr's annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"true\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer:        &mockContainer{},\n\t\t\texpectedIsolate:  true,\n\t\t\texpectedExplicit: true,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for unparsable\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"UNPARSABLE\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer:       &mockContainer{},\n\t\t\texpectedIsolate: opt.PreferIsolated,\n\t\t},\n\t\t{\n\t\t\tname: \"podIsolationPreference() should handle nil container arg gracefully\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"key: true\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer: &mockContainer{},\n\t\t\tdisabled:  true,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for missing preferences\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"key: true\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer:       &mockContainer{},\n\t\t\texpectedIsolate: opt.PreferIsolated,\n\t\t},\n\t\t{\n\t\t\tname: \"return defined preferences\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"testcontainer: false\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t},\n\t\t\texpectedExplicit: true,\n\t\t},\n\t\t// effective annotation tests\n\t\t{\n\t\t\tname: \"prefer resmgr's annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.c0\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:        &mockContainer{name: \"c0\"},\n\t\t\texpectedIsolate:  true,\n\t\t\texpectedExplicit: true,\n\t\t},\n\t\t{\n\t\t\tname: \"prefer resmgr's annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.c0\": \"false\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:        &mockContainer{name: \"c0\"},\n\t\t\texpectedIsolate:  false,\n\t\t\texpectedExplicit: true,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for unparsable annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.c0\": \"blah\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:       &mockContainer{name: \"c0\"},\n\t\t\texpectedIsolate: opt.PreferIsolated,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for missing preferences\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.c0\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:       &mockContainer{name: \"c1\"},\n\t\t\texpectedIsolate: opt.PreferIsolated,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tif tc.disabled {\n\t\t\t\tt.Skipf(\"The case '%s' is skipped\", tc.name)\n\t\t\t}\n\t\t\tisolate, explicit := isolatedCPUsPreference(tc.pod, tc.container)\n\t\t\tif isolate != tc.expectedIsolate || explicit != tc.expectedExplicit {\n\t\t\t\tt.Errorf(\"Expected (%v, %v), but got (%v, %v)\", tc.expectedIsolate, tc.expectedExplicit, isolate, explicit)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestPodSharedCPUPreference(t *testing.T) {\n\ttcases := []struct {\n\t\tname           string\n\t\tpod            *mockPod\n\t\tcontainer      *mockContainer\n\t\texpectedShared bool\n\t\tdisabled       bool\n\t}{\n\t\t{\n\t\t\tname:     \"podSharedCPUPreference() should handle nil pod arg gracefully\",\n\t\t\tdisabled: true,\n\t\t},\n\t\t{\n\t\t\tname:           \"return defaults\",\n\t\t\tpod:            &mockPod{},\n\t\t\tcontainer:      &mockContainer{},\n\t\t\texpectedShared: opt.PreferShared,\n\t\t},\n\t\t{\n\t\t\tname: \"prefer resmgr's annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"true\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{},\n\t\t\texpectedShared: true,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for unparsable\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"UNPARSABLE\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{},\n\t\t\texpectedShared: opt.PreferShared,\n\t\t},\n\t\t{\n\t\t\tname: \"podSharedCPUPreference() should handle nil container arg gracefully\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"key: true\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer: &mockContainer{},\n\t\t\tdisabled:  true,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for missing preferences\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"key: true\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{},\n\t\t\texpectedShared: opt.PreferShared,\n\t\t},\n\t\t{\n\t\t\tname: \"return defined preferences\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"testcontainer: false\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for unparsable annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValue1FotGetResmgrAnnotation: \"testcontainer: UNPARSABLE\",\n\t\t\t\treturnValue2FotGetResmgrAnnotation: true,\n\t\t\t},\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t},\n\t\t\texpectedShared: opt.PreferShared,\n\t\t},\n\t\t// effective annotation tests\n\t\t{\n\t\t\tname: \"prefer resmgr's annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.c0\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{name: \"c0\"},\n\t\t\texpectedShared: true,\n\t\t},\n\t\t{\n\t\t\tname: \"prefer resmgr's annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.c0\": \"false\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{name: \"c0\"},\n\t\t\texpectedShared: false,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for unparsable annotation value\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.c0\": \"blah\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{name: \"c0\"},\n\t\t\texpectedShared: opt.PreferShared,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults for missing preferences\",\n\t\t\tpod: &mockPod{\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.c0\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tcontainer:      &mockContainer{name: \"c1\"},\n\t\t\texpectedShared: opt.PreferShared,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tif tc.disabled {\n\t\t\t\tt.Skipf(\"The case '%s' is skipped\", tc.name)\n\t\t\t}\n\t\t\tshared, _ := sharedCPUsPreference(tc.pod, tc.container)\n\t\t\tif shared != tc.expectedShared {\n\t\t\t\tt.Errorf(\"Expected %v, but got %v\", tc.expectedShared, shared)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestCpuAllocationPreferences(t *testing.T) {\n\ttcases := []struct {\n\t\tname                   string\n\t\tpod                    *mockPod\n\t\tcontainer              *mockContainer\n\t\tpreferIsolated         bool\n\t\tpreferShared           bool\n\t\texpectedFull           int\n\t\texpectedFraction       int\n\t\texpectedIsolate        bool\n\t\texpectedCpuType        cpuClass\n\t\tdisabled               bool\n\t\treservedPoolNamespaces []string\n\t}{\n\t\t{\n\t\t\tname:     \"cpuAllocationPreferences() should handle nil container arg gracefully\",\n\t\t\tdisabled: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"no resource requirements\",\n\t\t\tpod:       &mockPod{},\n\t\t\tcontainer: &mockContainer{},\n\t\t},\n\t\t{\n\t\t\tname: \"cpuAllocationPreferences() should handle nil pod arg gracefully\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tdisabled: true,\n\t\t},\n\t\t{\n\t\t\tname: \"return defaults\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction: 1000,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for system container\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: metav1.NamespaceSystem,\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction: 2000,\n\t\t\texpectedCpuType:  cpuReserved,\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for burstable QoS\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction: 2000,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with sub-core request\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"750m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 750,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with sub-core request, prefer isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"750m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 750,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with sub-core request, prefer shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"750m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 750,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with sub-core request, prefer isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"750m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 750,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\n\t\t{\n\t\t\tname: \"guaranteed QoS with single full core request, prefer isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:  true,\n\t\t\texpectedFull:    1,\n\t\t\texpectedIsolate: true,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with single full core request, prefer no isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:  false,\n\t\t\texpectedFull:    1,\n\t\t\texpectedIsolate: false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with single full core request, prefer shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     1,\n\t\t\texpectedFraction: 0,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with single full core request, prefer isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     1,\n\t\t\texpectedFraction: 0,\n\t\t\texpectedIsolate:  true,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with single full core request, annotated shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 1000,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with single full core request, annotated no isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.testcontainer\": \"false\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     1,\n\t\t\texpectedFraction: 0,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with potential mixed request\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:     1,\n\t\t\texpectedFraction: 500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with potential mixed request, prefer isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\texpectedFull:     1,\n\t\t\texpectedFraction: 500,\n\t\t\texpectedIsolate:  true,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with potential mixed request, prefer shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 1500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with potential mixed request, prefer isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"1500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:   true,\n\t\t\tpreferShared:     true,\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 1500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:    2,\n\t\t\texpectedIsolate: false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request, prefer isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:  true,\n\t\t\texpectedFull:    2,\n\t\t\texpectedIsolate: false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request, prefer shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferShared:    true,\n\t\t\texpectedFull:    2,\n\t\t\texpectedIsolate: false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request, prefer isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\tpreferIsolated:  true,\n\t\t\tpreferShared:    true,\n\t\t\texpectedFull:    2,\n\t\t\texpectedIsolate: false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request, annotate isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:    2,\n\t\t\texpectedIsolate: true,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request, annotate shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2000,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core full request, annotate isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\":   \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2000,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, prefer isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, prefer shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, prefer isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, annotate isolated\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, annotate shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, annotate isolated & shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\":   \"true\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     0,\n\t\t\texpectedFraction: 2500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, annotate no shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\": \"false\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     2,\n\t\t\texpectedFraction: 500,\n\t\t\texpectedIsolate:  false,\n\t\t},\n\t\t{\n\t\t\tname: \"guaranteed QoS with multi-core mixed request, annotate isolated, no shared\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2500m\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\tannotations: map[string]string{\n\t\t\t\t\tpreferIsolatedCPUsKey + \"/container.testcontainer\": \"true\",\n\t\t\t\t\tpreferSharedCPUsKey + \"/container.testcontainer\":   \"false\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFull:     2,\n\t\t\texpectedFraction: 500,\n\t\t\texpectedIsolate:  true,\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved pool namespace container\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: \"foobar\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction:       2000,\n\t\t\texpectedCpuType:        cpuReserved,\n\t\t\treservedPoolNamespaces: []string{\"foobar\"},\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved pool namespace container using a glob 1\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: \"foobar2\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction:       2000,\n\t\t\texpectedCpuType:        cpuReserved,\n\t\t\treservedPoolNamespaces: []string{\"foobar*\"},\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved pool namespace container using a glob 2\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: \"foobar-testing\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction:       2000,\n\t\t\texpectedCpuType:        cpuReserved,\n\t\t\treservedPoolNamespaces: []string{\"barfoo\", \"foobar*\"},\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved pool namespace container using a glob 3\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: \"testing\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction:       2000,\n\t\t\texpectedCpuType:        cpuNormal,\n\t\t\treservedPoolNamespaces: []string{\"barfoo\", \"foobar?\"},\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved pool namespace container using a glob 4\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: \"1foobar2\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction:       2000,\n\t\t\texpectedCpuType:        cpuNormal,\n\t\t\treservedPoolNamespaces: []string{\"barfoo\", \"foobar?\"},\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved pool namespace container using a glob 5\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tnamespace: \"foobar12\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tRequests: v1.ResourceList{\n\t\t\t\t\t\tcorev1.ResourceCPU: resapi.MustParse(\"2\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction:       2000,\n\t\t\texpectedCpuType:        cpuNormal,\n\t\t\treservedPoolNamespaces: []string{\"barfoo\", \"foobar?\", \"testing\"},\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved cpu annotation container\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tname: \"testcontainer\",\n\t\t\t\tpod: &mockPod{\n\t\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\t\tannotations: map[string]string{\n\t\t\t\t\t\tpreferReservedCPUsKey + \"/container.special\": \"false\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction: 0,\n\t\t\texpectedCpuType:  cpuNormal,\n\t\t},\n\t\t{\n\t\t\tname: \"return request's value for reserved cpu annotation container\",\n\t\t\tcontainer: &mockContainer{\n\t\t\t\tpod: &mockPod{\n\t\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSGuaranteed,\n\t\t\t\t\tannotations: map[string]string{\n\t\t\t\t\t\tpreferReservedCPUsKey + \"/pod\": \"true\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tpod: &mockPod{\n\t\t\t\treturnValueFotGetQOSClass: corev1.PodQOSBurstable,\n\t\t\t},\n\t\t\texpectedFraction: 0,\n\t\t\texpectedCpuType:  cpuReserved,\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tif tc.disabled {\n\t\t\t\tt.Skipf(\"The case '%s' is skipped\", tc.name)\n\t\t\t}\n\t\t\topt.PreferIsolated, opt.PreferShared = tc.preferIsolated, tc.preferShared\n\t\t\topt.ReservedPoolNamespaces = tc.reservedPoolNamespaces\n\t\t\tfull, fraction, isolate, cpuType := cpuAllocationPreferences(tc.pod, tc.container)\n\t\t\tif full != tc.expectedFull || fraction != tc.expectedFraction ||\n\t\t\t\tisolate != tc.expectedIsolate || cpuType != tc.expectedCpuType {\n\t\t\t\tt.Errorf(\"Expected (%v, %v, %v, %s), but got (%v, %v, %v, %s)\",\n\t\t\t\t\ttc.expectedFull, tc.expectedFraction, tc.expectedIsolate, tc.expectedCpuType,\n\t\t\t\t\tfull, fraction, isolate, cpuType)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/pools.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"math\"\n\t\"sort\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// buildPoolsByTopology builds a hierarchical tree of pools based on HW topology.\nfunc (p *policy) buildPoolsByTopology() error {\n\tif err := p.checkHWTopology(); err != nil {\n\t\treturn err\n\t}\n\n\t// Notes:\n\t//   we never create pool nodes for PMEM-only NUMA nodes (as these\n\t//   are always without any close/local set of CPUs). We instead\n\t//   assign the PMEM memory of such a node to one of the closest\n\t//   normal (DRAM) pool NUMA nodes.\n\t//\n\t//   Akin to omitting lone dies from their parent, we omit from the\n\t//   pool tree each NUMA node that would end up being the only child\n\t//   of its parent (a die or a socket pool node). Resources for each\n\t//   such node will get discovered by and assigned to the would be\n\t//   parent which is now a leaf (die or socket) node in the tree.\n\t//\n\t//   The PMEM memory of (omitted) PMEM-only nodes is assigned\n\t//   to one of the closest normal (DRAM) NUMA nodes. This right\n\t//   assignment has already been calculated by assignPMEMNodes().\n\t//   However, making the corresponding assignment in the pool\n\t//   tree is a bit more involved as the DRAM node where a PMEM\n\t//   node has been assigned to might have gotten omitted from the\n\t//   tree if it ended up being a lone child. We use the recorded\n\t//   per NUMA node surrogates to find both if and where resources\n\t//   of omitted DRAM NUMA nodes need to get assigned to, and also\n\t//   where PMEM NUMA node resources need to get assigned to.\n\n\tlog.Debug(\"building topology pool tree...\")\n\n\tp.nodes = make(map[string]Node)\n\n\t// create a virtual root node, if we have a multi-socket system\n\tif p.sys.SocketCount() > 1 {\n\t\tp.root = p.NewVirtualNode(\"root\", nilnode)\n\t\tp.nodes[p.root.Name()] = p.root\n\t\tlog.Debug(\"  + created pool (root) %q\", p.root.Name())\n\t} else {\n\t\tlog.Debug(\"  - omitted pool virtual root (single-socket system)\")\n\t}\n\n\t// create socket nodes, for a single-socket system set the only socket as the root\n\tsockets := map[idset.ID]Node{}\n\tfor _, socketID := range p.sys.PackageIDs() {\n\t\tvar socket Node\n\n\t\tif p.root != nil {\n\t\t\tsocket = p.NewSocketNode(socketID, p.root)\n\t\t\tlog.Debug(\"    + created pool %q\", socket.Parent().Name()+\"/\"+socket.Name())\n\t\t} else {\n\t\t\tsocket = p.NewSocketNode(socketID, nilnode)\n\t\t\tp.root = socket\n\t\t\tlog.Debug(\"    + created pool %q (as root)\", socket.Name())\n\t\t}\n\n\t\tp.nodes[socket.Name()] = socket\n\t\tsockets[socketID] = socket\n\t}\n\n\t// create dies for every socket, but only if we have more than one die in the socket\n\tnumaDies := map[idset.ID]Node{} // created die Nodes per NUMA node id\n\tfor socketID, socket := range sockets {\n\t\tdieIDs := p.sys.Package(socketID).DieIDs()\n\t\tif len(dieIDs) < 2 {\n\t\t\tlog.Debug(\"      - omitted pool %q (die count: %d)\", socket.Name()+\"/die #0\",\n\t\t\t\tlen(dieIDs))\n\t\t\tcontinue\n\t\t}\n\t\tfor _, dieID := range dieIDs {\n\t\t\tdie := p.NewDieNode(dieID, socket)\n\t\t\tp.nodes[die.Name()] = die\n\t\t\tfor _, numaNodeID := range p.sys.Package(socketID).DieNodeIDs(dieID) {\n\t\t\t\tnumaDies[numaNodeID] = die\n\t\t\t}\n\t\t\tlog.Debug(\"      + created pool %q\", die.Parent().Name()+\"/\"+die.Name())\n\t\t}\n\t}\n\n\t// create pool nodes for NUMA nodes\n\tpmemNodes := map[idset.ID]system.Node{} // collected PMEM-only nodes\n\tdramNodes := map[idset.ID]system.Node{} // collected DRAM-only nodes\n\tnumaSurrogates := map[idset.ID]Node{}   // surrogate leaf nodes for omitted NUMA nodes\n\tfor _, numaNodeID := range p.sys.NodeIDs() {\n\t\tvar numaNode Node\n\n\t\tnumaSysNode := p.sys.Node(numaNodeID)\n\t\tswitch numaSysNode.GetMemoryType() {\n\t\tcase system.MemoryTypeDRAM:\n\t\t\tdramNodes[numaNodeID] = numaSysNode\n\t\tcase system.MemoryTypePMEM:\n\t\t\tpmemNodes[numaNodeID] = numaSysNode\n\t\t\tlog.Debug(\"        - omitted pool \\\"NUMA node #%d\\\": PMEM node\", numaNodeID)\n\t\t\tcontinue // don't create pool, will assign to a closest DRAM node\n\t\tdefault:\n\t\t\tlog.Warn(\"        - ignored pool \\\"NUMA node #%d\\\": unhandled memory type %v\",\n\t\t\t\tnumaNodeID, numaSysNode.GetMemoryType())\n\t\t\tcontinue\n\t\t}\n\n\t\t//\n\t\t// Notes:\n\t\t//   We omit inserting NUMA nodes (as leaf nodes) in the tree, if that NUMA node\n\t\t//   would be the only child of its parent. In this case, we record the would-be\n\t\t//   parent as the surrogate for the NUMA node. This surrogate will get assigned\n\t\t//   any closest PMEM-only NUMA node that the original one would have received.\n\t\t//\n\n\t\tif die, ok := numaDies[numaNodeID]; ok {\n\t\t\tif p.parentNumaNodeCountWithCPUs(numaSysNode) < 2 {\n\t\t\t\tnumaSurrogates[numaNodeID] = die\n\t\t\t\tlog.Debug(\"        - omitted pool \\\"NUMA node #%d\\\": using surrogate %q\",\n\t\t\t\t\tnumaNodeID, numaSurrogates[numaNodeID].Name())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tnumaNode = p.NewNumaNode(numaNodeID, die)\n\t\t} else {\n\t\t\tsocket := sockets[p.sys.Node(numaNodeID).PackageID()]\n\t\t\tif p.parentNumaNodeCountWithCPUs(numaSysNode) < 2 {\n\t\t\t\tnumaSurrogates[numaNodeID] = socket\n\t\t\t\tlog.Debug(\"        - omitted pool \\\"NUMA node #%d\\\": using surrogate %q\",\n\t\t\t\t\tnumaNodeID, numaSurrogates[numaNodeID].Name())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tnumaNode = p.NewNumaNode(numaNodeID, socket)\n\t\t}\n\n\t\tp.nodes[numaNode.Name()] = numaNode\n\t\tnumaSurrogates[numaNodeID] = numaNode\n\t\tlog.Debug(\"        + created pool %q\", numaNode.Parent().Name()+\"/\"+numaNode.Name())\n\t}\n\n\t// set up assignment of PMEM and DRAM node resources to pool nodes and surrogates\n\tassigned := p.assignNUMANodes(numaSurrogates, pmemNodes, dramNodes)\n\tlog.Debug(\"NUMA node to pool assignment:\")\n\tfor n, numaNodeIDs := range assigned {\n\t\tlog.Debug(\"  pool %q: NUMA nodes #%s\", n.Name(), idset.NewIDSet(numaNodeIDs...))\n\t}\n\n\t// enumerate pools, calculate depth, discover resource capacity, assign NUMA nodes\n\tp.pools = make([]Node, 0)\n\tp.root.DepthFirst(func(n Node) error {\n\t\tp.pools = append(p.pools, n)\n\t\tn.(*node).id = p.nodeCnt\n\t\tp.nodeCnt++\n\n\t\tif p.depth < n.(*node).depth {\n\t\t\tp.depth = n.(*node).depth\n\t\t}\n\n\t\tn.DiscoverSupply(assigned[n.(*node).self.node])\n\t\tdelete(assigned, n.(*node).self.node)\n\n\t\treturn nil\n\t})\n\n\t// make sure all PMEM nodes got assigned\n\tif len(assigned) > 0 {\n\t\tfor node, pmem := range assigned {\n\t\t\tlog.Error(\"failed to assign PMEM NUMA nodes #%s (to NUMA node/surrogate %s %v)\",\n\t\t\t\tidset.NewIDSet(pmem...), node.Name(), node)\n\t\t}\n\t\tlog.Fatal(\"internal error: unassigned PMEM NUMA nodes remaining\")\n\t}\n\n\tp.root.Dump(\"<pool-setup>\")\n\n\treturn nil\n}\n\n// parentNumaNodeCountWithCPUs returns the number of CPU-ful NUMA nodes in the parent die/socket.\nfunc (p *policy) parentNumaNodeCountWithCPUs(numaNode system.Node) int {\n\tsocketID := numaNode.PackageID()\n\tsocket := p.sys.Package(socketID)\n\tcount := 0\n\tfor _, nodeID := range socket.DieNodeIDs(numaNode.DieID()) {\n\t\tnode := p.sys.Node(nodeID)\n\t\tif !node.CPUSet().IsEmpty() {\n\t\t\tcount++\n\t\t}\n\t}\n\treturn count\n}\n\n// assignNUMANodes assigns each PMEM node to one of the closest DRAM nodes\nfunc (p *policy) assignNUMANodes(surrogates map[idset.ID]Node, pmem, dram map[idset.ID]system.Node) map[Node][]idset.ID {\n\t// collect the closest DRAM NUMA nodes (sorted by idset.ID) for each PMEM NUMA node.\n\tclosest := map[idset.ID][]idset.ID{}\n\tfor pmemID := range pmem {\n\t\tvar min []idset.ID\n\t\tfor dramID := range dram {\n\t\t\tif len(min) < 1 {\n\t\t\t\tmin = []idset.ID{dramID}\n\t\t\t} else {\n\t\t\t\tminDist := p.sys.NodeDistance(pmemID, min[0])\n\t\t\t\tnewDist := p.sys.NodeDistance(pmemID, dramID)\n\t\t\t\tswitch {\n\t\t\t\tcase newDist == minDist:\n\t\t\t\t\tmin = append(min, dramID)\n\t\t\t\tcase newDist < minDist:\n\t\t\t\t\tmin = []idset.ID{dramID}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tsort.Slice(min, func(i, j int) bool { return min[i] < min[j] })\n\t\tclosest[pmemID] = min\n\t}\n\n\tassigned := map[Node][]idset.ID{}\n\n\t// assign each PMEM node to the closest DRAM surrogate with the least PMEM assigned\n\tfor pmemID, min := range closest {\n\t\tvar taker Node\n\t\tvar takerID idset.ID\n\n\t\tfor _, dramID := range min {\n\t\t\tif taker == nil {\n\t\t\t\ttaker = surrogates[dramID]\n\t\t\t\ttakerID = dramID\n\t\t\t} else {\n\t\t\t\tif len(assigned[taker]) > len(assigned[surrogates[dramID]]) {\n\t\t\t\t\ttaker = surrogates[dramID]\n\t\t\t\t\ttakerID = dramID\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif taker == nil {\n\t\t\tlog.Panic(\"failed to assign CPU-less PMEM node #%d to any surrogate\", pmemID)\n\t\t}\n\n\t\tassigned[taker] = append(assigned[taker], pmemID)\n\t\tlog.Debug(\"        + PMEM node #%d assigned to %s with distance %v\", pmemID, taker.Name(),\n\t\t\tp.sys.NodeDistance(pmemID, takerID))\n\t}\n\n\t// assign each DRAM node to its own surrogate (can be the DRAM node itself)\n\tfor dramID := range dram {\n\t\ttaker := surrogates[dramID]\n\t\tassigned[taker] = append([]idset.ID{dramID}, assigned[taker]...)\n\t\tlog.Debug(\"        + DRAM node #%d assigned to %s\", dramID, taker.Name())\n\t}\n\n\treturn assigned\n}\n\n// checkHWTopology verifies our otherwise implicit assumptions about the HW.\nfunc (p *policy) checkHWTopology() error {\n\t// NUMA nodes (memory controllers) should not be shared by multiple sockets.\n\tsocketNodes := map[idset.ID]cpuset.CPUSet{}\n\tfor _, socketID := range p.sys.PackageIDs() {\n\t\tpkg := p.sys.Package(socketID)\n\t\tsocketNodes[socketID] = system.CPUSetFromIDSet(idset.NewIDSet(pkg.NodeIDs()...))\n\t}\n\tfor id1, nodes1 := range socketNodes {\n\t\tfor id2, nodes2 := range socketNodes {\n\t\t\tif id1 == id2 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif shared := nodes1.Intersection(nodes2); !shared.IsEmpty() {\n\t\t\t\tlog.Error(\"can't handle HW topology: sockets #%v, #%v share NUMA node(s) #%s\",\n\t\t\t\t\tid1, id2, shared.String())\n\t\t\t\treturn policyError(\"unhandled HW topology: sockets #%v, #%v share NUMA node(s) #%s\",\n\t\t\t\t\tid1, id2, shared.String())\n\t\t\t}\n\t\t}\n\t}\n\n\t// NUMA nodes (memory controllers) should not be shared by multiple dies.\n\tfor _, socketID := range p.sys.PackageIDs() {\n\t\tpkg := p.sys.Package(socketID)\n\t\tfor _, id1 := range pkg.DieIDs() {\n\t\t\tnodes1 := idset.NewIDSet(pkg.DieNodeIDs(id1)...)\n\t\t\tfor _, id2 := range pkg.DieIDs() {\n\t\t\t\tif id1 == id2 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tnodes2 := idset.NewIDSet(pkg.DieNodeIDs(id2)...)\n\t\t\t\tif shared := system.CPUSetFromIDSet(nodes1).Intersection(system.CPUSetFromIDSet(nodes2)); !shared.IsEmpty() {\n\t\t\t\t\tlog.Error(\"can't handle HW topology: \"+\n\t\t\t\t\t\t\"socket #%v, dies #%v,%v share NUMA node(s) #%s\",\n\t\t\t\t\t\tsocketID, id1, id2, shared.String())\n\t\t\t\t\treturn policyError(\"unhandled HW topology: \"+\n\t\t\t\t\t\t\"socket #%v, dies #%v,#%v share NUMA node(s) #%s\",\n\t\t\t\t\t\tsocketID, id1, id2, shared.String())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// NUMA distance matrix should be symmetric.\n\tfor _, from := range p.sys.NodeIDs() {\n\t\tfor _, to := range p.sys.NodeIDs() {\n\t\t\td1 := p.sys.NodeDistance(from, to)\n\t\t\td2 := p.sys.NodeDistance(to, from)\n\t\t\tif d1 != d2 {\n\t\t\t\tlog.Error(\"asymmetric NUMA distance (#%d, #%d): %d != %d\",\n\t\t\t\t\tfrom, to, d1, d2)\n\t\t\t\treturn policyError(\"asymmetric NUMA distance (#%d, #%d): %d != %d\",\n\t\t\t\t\tfrom, to, d1, d2)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Pick a pool and allocate resource from it to the container.\nfunc (p *policy) allocatePool(container cache.Container, poolHint string) (Grant, error) {\n\tvar pool Node\n\n\trequest := newRequest(container)\n\n\tif p.root.FreeSupply().ReservedCPUs().IsEmpty() && request.CPUType() == cpuReserved {\n\t\t// Fallback to allocating reserved CPUs from the shared pool\n\t\t// if there are no reserved CPUs.\n\t\trequest.SetCPUType(cpuNormal)\n\t}\n\n\t// Assumption: in the beginning the CPUs and memory will be allocated from\n\t// the same pool. This assumption can be relaxed later, requires separate\n\t// (but connected) scoring of memory and CPU.\n\n\tif request.CPUType() == cpuReserved || container.GetNamespace() == kubernetes.NamespaceSystem {\n\t\tpool = p.root\n\t} else {\n\t\taffinity, err := p.calculatePoolAffinities(request.GetContainer())\n\n\t\tif err != nil {\n\t\t\treturn nil, policyError(\"failed to calculate affinity for container %s: %v\",\n\t\t\t\tcontainer.PrettyName(), err)\n\t\t}\n\n\t\tscores, pools := p.sortPoolsByScore(request, affinity)\n\n\t\tif log.DebugEnabled() {\n\t\t\tlog.Debug(\"* node fitting for %s\", request)\n\t\t\tfor idx, n := range pools {\n\t\t\t\tlog.Debug(\"    - #%d: node %s, score %s, affinity: %d\",\n\t\t\t\t\tidx, n.Name(), scores[n.NodeID()], affinity[n.NodeID()])\n\t\t\t}\n\t\t}\n\n\t\tif len(pools) == 0 {\n\t\t\treturn nil, policyError(\"no suitable pool found for container %s\",\n\t\t\t\tcontainer.PrettyName())\n\t\t}\n\n\t\tif poolHint != \"\" {\n\t\t\tfor idx, p := range pools {\n\t\t\t\tif p.Name() == poolHint {\n\t\t\t\t\tlog.Debug(\"* using hinted pool %q (#%d best fit)\", poolHint, idx+1)\n\t\t\t\t\tpool = p\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif pool == nil {\n\t\t\t\tlog.Debug(\"* cannot use hinted pool %q\", poolHint)\n\t\t\t}\n\t\t}\n\n\t\tif pool == nil {\n\t\t\tpool = pools[0]\n\t\t}\n\t}\n\n\tsupply := pool.FreeSupply()\n\tgrant, err := supply.Allocate(request)\n\tif err != nil {\n\t\treturn nil, policyError(\"failed to allocate %s from %s: %v\",\n\t\t\trequest, supply.DumpAllocatable(), err)\n\t}\n\n\tlog.Debug(\"allocated req '%s' to memory node '%s' (memset %s,%s,%s)\",\n\t\tcontainer.PrettyName(), grant.GetMemoryNode().Name(),\n\t\tgrant.GetMemoryNode().GetMemset(memoryDRAM),\n\t\tgrant.GetMemoryNode().GetMemset(memoryPMEM),\n\t\tgrant.GetMemoryNode().GetMemset(memoryHBM))\n\n\t// In case the workload is assigned to a memory node with multiple\n\t// child nodes, there is no guarantee that the workload will\n\t// allocate memory \"nicely\". Instead we'll have to make the\n\t// conservative assumption that the memory will all be allocated\n\t// from one single node, and that node can be any of the child\n\t// nodes in the system. Thus, we'll need to reserve the memory\n\t// from all child nodes, and move the containers already\n\t// assigned to the child nodes upwards in the topology tree, if\n\t// they no longer fit to the child node that they are in. In\n\t// other words, they'll need to have a wider range of memory\n\t// node options in order to fit to memory.\n\t//\n\t//\n\t// Example:\n\t//\n\t// Workload 1 and Workload 2 are running on the leaf nodes:\n\t//\n\t//                    +----------------+\n\t//                    |Total mem: 4G   |\n\t//                    |Total CPUs: 4   |            Workload 1:\n\t//                    |Reserved:       |\n\t//                    |  1.5G          |             1G mem\n\t//                    |                |\n\t//                    |                |            Workload 2:\n\t//                    |                |\n\t//                    +----------------+             0.5G mem\n\t//                       /          \\\n\t//                      /            \\\n\t//                     /              \\\n\t//                    /                \\\n\t//                   /                  \\\n\t//                  /                    \\\n\t//                 /                      \\\n\t//                /                        \\\n\t//  +----------------+                  +----------------+\n\t//  |Total mem: 2G   |                  |Total mem: 2G   |\n\t//  |Total CPUs: 2   |                  |Total CPUs: 2   |\n\t//  |Reserved:       |                  |Reserved:       |\n\t//  |  1G            |                  |  0.5G          |\n\t//  |                |                  |                |\n\t//  |                |                  |                |\n\t//  |     * WL 1     |                  |     * WL 2     |\n\t//  +----------------+                  +----------------+\n\t//\n\t//\n\t// Then Workload 3 comes in and is assigned to the root node. Memory\n\t// reservations are done on the leaf nodes:\n\t//\n\t//                    +----------------+\n\t//                    |Total mem: 4G   |\n\t//                    |Total CPUs: 4   |            Workload 1:\n\t//                    |Reserved:       |\n\t//                    |  3G            |             1G mem\n\t//                    |                |\n\t//                    |                |            Workload 2:\n\t//                    |  * WL 3        |\n\t//                    +----------------+             0.5G mem\n\t//                       /          \\\n\t//                      /            \\              Workload 3:\n\t//                     /              \\\n\t//                    /                \\             1.5G mem\n\t//                   /                  \\\n\t//                  /                    \\\n\t//                 /                      \\\n\t//                /                        \\\n\t//  +----------------+                  +----------------+\n\t//  |Total mem: 2G   |                  |Total mem: 2G   |\n\t//  |Total CPUs: 2   |                  |Total CPUs: 2   |\n\t//  |Reserved:       |                  |Reserved:       |\n\t//  |  2.5G          |                  |  2G            |\n\t//  |                |                  |                |\n\t//  |                |                  |                |\n\t//  |     * WL 1     |                  |     * WL 2     |\n\t//  +----------------+                  +----------------+\n\t//\n\t//\n\t// Workload 1 no longer fits to the leaf node, because the total\n\t// reservation from the leaf node is over the memory maximum.\n\t// Thus, it's moved upwards in the tree to the root node. Memory\n\t// resevations are again updated accordingly:\n\t//\n\t//                    +----------------+\n\t//                    |Total mem: 4G   |\n\t//                    |Total CPUs: 4   |            Workload 1:\n\t//                    |Reserved:       |\n\t//                    |  3G            |             1G mem\n\t//                    |                |\n\t//                    |  * WL 1        |            Workload 2:\n\t//                    |  * WL 3        |\n\t//                    +----------------+             0.5G mem\n\t//                       /          \\\n\t//                      /            \\              Workload 3:\n\t//                     /              \\\n\t//                    /                \\             1.5G mem\n\t//                   /                  \\\n\t//                  /                    \\\n\t//                 /                      \\\n\t//                /                        \\\n\t//  +----------------+                  +----------------+\n\t//  |Total mem: 2G   |                  |Total mem: 2G   |\n\t//  |Total CPUs: 2   |                  |Total CPUs: 2   |\n\t//  |Reserved:       |                  |Reserved:       |\n\t//  |  2.5G          |                  |  3G            |\n\t//  |                |                  |                |\n\t//  |                |                  |                |\n\t//  |                |                  |     * WL 2     |\n\t//  +----------------+                  +----------------+\n\t//\n\t//\n\t// Now Workload 2 doesn't fit to the leaf node either. It's also moved\n\t// to the root node:\n\t//\n\t//                    +----------------+\n\t//                    |Total mem: 4G   |\n\t//                    |Total CPUs: 4   |            Workload 1:\n\t//                    |Reserved:       |\n\t//                    |  3G            |             1G mem\n\t//                    |  * WL 2        |\n\t//                    |  * WL 1        |            Workload 2:\n\t//                    |  * WL 3        |\n\t//                    +----------------+             0.5G mem\n\t//                       /          \\\n\t//                      /            \\              Workload 3:\n\t//                     /              \\\n\t//                    /                \\             1.5G mem\n\t//                   /                  \\\n\t//                  /                    \\\n\t//                 /                      \\\n\t//                /                        \\\n\t//  +----------------+                  +----------------+\n\t//  |Total mem: 2G   |                  |Total mem: 2G   |\n\t//  |Total CPUs: 2   |                  |Total CPUs: 2   |\n\t//  |Reserved:       |                  |Reserved:       |\n\t//  |  3G            |                  |  3G            |\n\t//  |                |                  |                |\n\t//  |                |                  |                |\n\t//  |                |                  |                |\n\t//  +----------------+                  +----------------+\n\t//\n\n\t// We need to analyze all existing containers which are a subset of current grant.\n\tmemset := grant.GetMemoryNode().GetMemset(grant.MemoryType())\n\n\t// Add an extra memory reservation to all subnodes.\n\t// TODO: no need to do any of this if no memory request\n\tgrant.UpdateExtraMemoryReservation()\n\n\t// See how much memory reservations the workloads on the\n\t// nodes up from this one cause to the node. We only need to\n\t// analyze the workloads up until this node, because it's\n\t// guaranteed that the subtree can hold the workloads.\n\n\t// If it turns out that the current workloads no longer fit\n\t// to the node with the reservations from nodes from above\n\t// in the tree, move all nodes upward. Note that this\n\t// creates a reservation of the same size to the node, so in\n\t// effect the node has to be empty of its \"own\" workloads.\n\t// In this case move all the workloads one level up in the tree.\n\n\tchanged := true\n\tfor changed {\n\t\tchanged = false\n\t\tfor _, oldGrant := range p.allocations.grants {\n\t\t\toldMemset := oldGrant.GetMemoryNode().GetMemset(grant.MemoryType())\n\t\t\tif oldMemset.Size() < memset.Size() && memset.Has(oldMemset.Members()...) {\n\t\t\t\tchanged, err = oldGrant.ExpandMemset()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tif changed {\n\t\t\t\t\tlog.Debug(\"* moved container %s upward to node %s to guarantee memory\",\n\t\t\t\t\t\toldGrant.GetContainer().PrettyName(), oldGrant.GetMemoryNode().Name())\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tp.allocations.grants[container.GetCacheID()] = grant\n\n\tp.saveAllocations()\n\n\treturn grant, nil\n}\n\n// Apply the result of allocation to the requesting container.\nfunc (p *policy) applyGrant(grant Grant) {\n\tlog.Debug(\"* applying grant %s\", grant)\n\n\tcontainer := grant.GetContainer()\n\tcpuType := grant.CPUType()\n\texclusive := grant.ExclusiveCPUs()\n\treserved := grant.ReservedCPUs()\n\tshared := grant.SharedCPUs()\n\tcpuPortion := grant.SharedPortion()\n\n\tcpus := \"\"\n\tkind := \"\"\n\tif cpuType == cpuNormal {\n\t\tif exclusive.IsEmpty() {\n\t\t\tcpus = shared.String()\n\t\t\tkind = \"shared\"\n\t\t} else {\n\t\t\tkind = \"exclusive\"\n\t\t\tif cpuPortion > 0 {\n\t\t\t\tkind += \"+shared\"\n\t\t\t\tcpus = exclusive.Union(shared).String()\n\t\t\t} else {\n\t\t\t\tcpus = exclusive.String()\n\t\t\t}\n\t\t}\n\t} else if cpuType == cpuReserved {\n\t\tkind = \"reserved\"\n\t\tcpus = reserved.String()\n\t\tcpuPortion = grant.ReservedPortion()\n\t} else {\n\t\tlog.Debug(\"unsupported granted cpuType %s\", cpuType)\n\t\treturn\n\t}\n\n\tmems := \"\"\n\tif opt.PinMemory {\n\t\tmems = grant.Memset().String()\n\t}\n\n\tif opt.PinCPU {\n\t\tif cpus != \"\" {\n\t\t\tlog.Debug(\"  => pinning to (%s) cpuset %s\", kind, cpus)\n\t\t} else {\n\t\t\tlog.Debug(\"  => not pinning CPUs, allocated cpuset is empty...\")\n\t\t}\n\t\tcontainer.SetCpusetCpus(cpus)\n\n\t\t// Notes:\n\t\t//     It is extremely important to ensure that the exclusive subset of mixed\n\t\t//     CPU allocations are really exclusive at the level of the whole system\n\t\t//     and not just the orchestration. This is something we can't really do\n\t\t//     from here reliably ATM.\n\t\t//\n\t\t//     We set the CPU scheduling weight for the whole container (all processes\n\t\t//     within the container) according to container's partial allocation.\n\t\t//     This is typically a sub-CPU allocation (< 1000 mCPU) which is meant to be\n\t\t//     consumed by an 'infra/mgmt' process within the container from the shared subset\n\t\t//     of CPUs assigned to the container. The container entry point or the processes\n\t\t//     within the container are supposed to arrange so that the 'infra' process(es)\n\t\t//     are pinned to the shared CPUs and the 'data/performance critical' critical'\n\t\t//     process(es) to the exclusive CPU(s).\n\t\t//\n\t\t//     With this setup the kernel will slice out the correct amount of CPU from\n\t\t//     the shared pool for the 'infra' process as it competes with other workloads'\n\t\t//     processes in the same pool. Also the 'data' process should run fine, since\n\t\t//     it does not need to compete for CPU with any other processes in the system\n\t\t//     as long as that allocation is genuinely system-wide exclusive.\n\t\tcontainer.SetCPUShares(int64(cache.MilliCPUToShares(int64(cpuPortion))))\n\t}\n\n\tif mems != \"\" {\n\t\tlog.Debug(\"  => pinning to memory %s\", mems)\n\t\tcontainer.SetCpusetMems(mems)\n\t\tp.setDemotionPreferences(container, grant)\n\t} else {\n\t\tlog.Debug(\"  => not pinning memory, memory set is empty...\")\n\t}\n}\n\n// Release resources allocated by this grant.\nfunc (p *policy) releasePool(container cache.Container) (Grant, bool) {\n\tlog.Debug(\"* releasing resources allocated to %s\", container.PrettyName())\n\n\tgrant, ok := p.allocations.grants[container.GetCacheID()]\n\tif !ok {\n\t\tlog.Debug(\"  => no grant found, nothing to do...\")\n\t\treturn nil, false\n\t}\n\n\tlog.Debug(\"  => releasing grant %s...\", grant)\n\n\t// Remove the grant from all supplys it uses.\n\tgrant.Release()\n\n\tdelete(p.allocations.grants, container.GetCacheID())\n\tp.saveAllocations()\n\n\treturn grant, true\n}\n\n// Update shared allocations effected by agrant.\nfunc (p *policy) updateSharedAllocations(grant *Grant) {\n\tif grant != nil {\n\t\tlog.Debug(\"* updating shared allocations affected by %s\", (*grant).String())\n\t\tif (*grant).CPUType() == cpuReserved {\n\t\t\tlog.Debug(\"  this grant uses reserved CPUs, does not affect shared allocations\")\n\t\t\treturn\n\t\t}\n\t} else {\n\t\tlog.Debug(\"* updating shared allocations\")\n\t}\n\n\tfor _, other := range p.allocations.grants {\n\t\tif grant != nil {\n\t\t\tif other.GetContainer().GetCacheID() == (*grant).GetContainer().GetCacheID() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif other.CPUType() == cpuReserved {\n\t\t\tlog.Debug(\"  => %s not affected (only reserved CPUs)...\", other)\n\t\t\tcontinue\n\t\t}\n\n\t\tif other.SharedPortion() == 0 && !other.ExclusiveCPUs().IsEmpty() {\n\t\t\tlog.Debug(\"  => %s not affected (only exclusive CPUs)...\", other)\n\t\t\tcontinue\n\t\t}\n\n\t\tif opt.PinCPU {\n\t\t\tshared := other.GetCPUNode().FreeSupply().SharableCPUs()\n\t\t\texclusive := other.ExclusiveCPUs()\n\t\t\tif exclusive.IsEmpty() {\n\t\t\t\tlog.Debug(\"  => updating %s with shared CPUs of %s: %s...\",\n\t\t\t\t\tother, other.GetCPUNode().Name(), shared.String())\n\t\t\t\tother.GetContainer().SetCpusetCpus(shared.String())\n\t\t\t} else {\n\t\t\t\tlog.Debug(\"  => updating %s with exclusive+shared CPUs of %s: %s+%s...\",\n\t\t\t\t\tother, other.GetCPUNode().Name(), exclusive.String(), shared.String())\n\t\t\t\tother.GetContainer().SetCpusetCpus(exclusive.Union(shared).String())\n\t\t\t}\n\t\t}\n\t}\n}\n\n// setDemotionPreferences sets the dynamic demotion preferences a container.\nfunc (p *policy) setDemotionPreferences(c cache.Container, g Grant) {\n\tlog.Debug(\"%s: setting demotion preferences...\", c.PrettyName())\n\n\t// System containers should not be demoted.\n\tif c.GetNamespace() == kubernetes.NamespaceSystem {\n\t\tc.SetPageMigration(nil)\n\t\treturn\n\t}\n\n\tmemType := g.GetMemoryNode().GetMemoryType()\n\tif memType&memoryDRAM == 0 || memType&memoryPMEM == 0 {\n\t\tc.SetPageMigration(nil)\n\t\treturn\n\t}\n\n\tdram := g.GetMemoryNode().GetMemset(memoryDRAM)\n\tpmem := g.GetMemoryNode().GetMemset(memoryPMEM)\n\n\tlog.Debug(\"%s: eligible for demotion from %s to %s NUMA node(s)\",\n\t\tc.PrettyName(), dram, pmem)\n\n\tc.SetPageMigration(&cache.PageMigrate{\n\t\tSourceNodes: dram,\n\t\tTargetNodes: pmem,\n\t})\n}\n\nfunc (p *policy) filterInsufficientResources(req Request, originals []Node) []Node {\n\tsufficient := make([]Node, 0)\n\n\tfor _, node := range originals {\n\t\t// TODO: Need to filter based on the memory demotion scheme here. For example, if the request is\n\t\t// of memory type memoryAll, the memory used might be PMEM until it's full and after that DRAM. If\n\t\t// it's DRAM, amount of PMEM should not be considered and so on. How to find this out in a live\n\t\t// system?\n\n\t\tsupply := node.FreeSupply()\n\t\treqMemType := req.MemoryType()\n\n\t\tif reqMemType == memoryUnspec {\n\t\t\t// The algorithm for handling unspecified memory allocations is the same as for handling a request\n\t\t\t// with memory type all.\n\t\t\treqMemType = memoryAll\n\t\t}\n\n\t\trequired := req.MemAmountToAllocate()\n\n\t\tfor _, memType := range []memoryType{memoryPMEM, memoryDRAM, memoryHBM} {\n\t\t\tif reqMemType&memType != 0 {\n\t\t\t\textra := supply.ExtraMemoryReservation(memType)\n\t\t\t\tfree := supply.MemoryLimit()[memType]\n\t\t\t\tif extra > free {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif required+extra <= free {\n\t\t\t\t\tsufficient = append(sufficient, node)\n\t\t\t\t\trequired = 0\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif req.ColdStart() > 0 {\n\t\t\t\t\t// For a \"cold start\" request, the memory request must fit completely in the PMEM. So reject the node.\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// Subtracting unsigned integers.\n\t\t\t\t// Here free >= extra, that is, (free - extra) is non-negative,\n\t\t\t\t// and required > free - extra, that is, required stays positive.\n\t\t\t\trequired -= (free - extra)\n\t\t\t}\n\t\t}\n\t\tif required > 0 {\n\t\t\tlog.Debug(\"%s: filtered out %s with insufficient memory\", req.GetContainer().PrettyName(), node.Name())\n\t\t}\n\t}\n\treturn sufficient\n}\n\n// Score pools against the request and sort them by score.\nfunc (p *policy) sortPoolsByScore(req Request, aff map[int]int32) (map[int]Score, []Node) {\n\tscores := make(map[int]Score, p.nodeCnt)\n\n\tp.root.DepthFirst(func(n Node) error {\n\t\tscores[n.NodeID()] = n.GetScore(req)\n\t\treturn nil\n\t})\n\n\t// Filter out pools which don't have enough uncompressible resources\n\t// (memory) to satisfy the request.\n\tfilteredPools := p.filterInsufficientResources(req, p.pools)\n\n\tsort.Slice(filteredPools, func(i, j int) bool {\n\t\treturn p.compareScores(req, filteredPools, scores, aff, i, j)\n\t})\n\n\treturn scores, filteredPools\n}\n\n// Compare two pools by scores for allocation preference.\nfunc (p *policy) compareScores(request Request, pools []Node, scores map[int]Score,\n\taffinity map[int]int32, i int, j int) bool {\n\tnode1, node2 := pools[i], pools[j]\n\tdepth1, depth2 := node1.RootDistance(), node2.RootDistance()\n\tid1, id2 := node1.NodeID(), node2.NodeID()\n\tscore1, score2 := scores[id1], scores[id2]\n\tcpuType := request.CPUType()\n\tisolated1, reserved1, shared1 := score1.IsolatedCapacity(), score1.ReservedCapacity(), score1.SharedCapacity()\n\tisolated2, reserved2, shared2 := score2.IsolatedCapacity(), score2.ReservedCapacity(), score2.SharedCapacity()\n\ta1 := affinityScore(affinity, node1)\n\ta2 := affinityScore(affinity, node2)\n\n\tlog.Debug(\"comparing scores for %s and %s\", node1.Name(), node2.Name())\n\tlog.Debug(\"  %s: %s, affinity score %f\", node1.Name(), score1.String(), a1)\n\tlog.Debug(\"  %s: %s, affinity score %f\", node2.Name(), score2.String(), a2)\n\n\t//\n\t// Notes:\n\t//\n\t// Our scoring/score sorting algorithm is:\n\t//\n\t// 1) - insufficient isolated, reserved or shared capacity loses\n\t// 2) - if we have affinity, the higher affinity score wins\n\t// 3) - if only one node matches the memory type request, it wins\n\t// 4) - if we have topology hints\n\t//       * better hint score wins\n\t//       * for a tie, prefer the lower node then the smaller id\n\t// 5) - if a node is lower in the tree it wins\n\t// 6) - for reserved allocations\n\t//       * more unallocated reserved capacity per colocated container wins\n\t// 7) - for (non-reserved) isolated allocations\n\t//       * more isolated capacity wins\n\t//       * for a tie, prefer the smaller id\n\t// 8) - for (non-reserved) exclusive allocations\n\t//       * more slicable (shared) capacity wins\n\t//       * for a tie, prefer the smaller id\n\t// 9) - for (non-reserved) shared-only allocations\n\t//       * fewer colocated containers win\n\t//       * for a tie prefer more shared capacity\n\t// 10) - lower id wins\n\t//\n\t// Before this comparison is reached, nodes with insufficient uncompressible resources\n\t// (memory) have been filtered out.\n\n\t// 1) a node with insufficient isolated or shared capacity loses\n\tswitch {\n\tcase cpuType == cpuNormal && ((isolated2 < 0 && isolated1 >= 0) || (shared2 <= 0 && shared1 > 0)):\n\t\tlog.Debug(\"  => %s loses, insufficent isolated or shared\", node2.Name())\n\t\treturn true\n\tcase cpuType == cpuNormal && ((isolated1 < 0 && isolated2 >= 0) || (shared1 <= 0 && shared2 > 0)):\n\t\tlog.Debug(\"  => %s loses, insufficent isolated or shared\", node1.Name())\n\t\treturn false\n\tcase cpuType == cpuReserved && reserved2 < 0 && reserved1 >= 0:\n\t\tlog.Debug(\"  => %s loses, insufficent reserved\", node2.Name())\n\t\treturn true\n\tcase cpuType == cpuReserved && reserved1 < 0 && reserved2 >= 0:\n\t\tlog.Debug(\"  => %s loses, insufficent reserved\", node1.Name())\n\t\treturn false\n\t}\n\n\tlog.Debug(\"  - isolated/reserved/shared insufficiency is a TIE\")\n\n\t// 2) higher affinity score wins\n\tif a1 > a2 {\n\t\tlog.Debug(\"  => %s loses on affinity\", node2.Name())\n\t\treturn true\n\t}\n\tif a2 > a1 {\n\t\tlog.Debug(\"  => %s loses on affinity\", node1.Name())\n\t\treturn false\n\t}\n\n\tlog.Debug(\"  - affinity is a TIE\")\n\n\t// 3) matching memory type wins\n\tif reqType := request.MemoryType(); reqType != memoryUnspec {\n\t\tif node1.HasMemoryType(reqType) && !node2.HasMemoryType(reqType) {\n\t\t\tlog.Debug(\"  => %s WINS on memory type\", node1.Name())\n\t\t\treturn true\n\t\t}\n\t\tif !node1.HasMemoryType(reqType) && node2.HasMemoryType(reqType) {\n\t\t\tlog.Debug(\"  => %s WINS on memory type\", node2.Name())\n\t\t\treturn false\n\t\t}\n\n\t\tlog.Debug(\"  - memory type is a TIE\")\n\t}\n\n\t// 4) better topology hint score wins\n\thScores1 := score1.HintScores()\n\tif len(hScores1) > 0 {\n\t\thScores2 := score2.HintScores()\n\t\ths1, nz1 := combineHintScores(hScores1)\n\t\ths2, nz2 := combineHintScores(hScores2)\n\n\t\tif hs1 > hs2 {\n\t\t\tlog.Debug(\"  => %s WINS on hints\", node1.Name())\n\t\t\treturn true\n\t\t}\n\t\tif hs2 > hs1 {\n\t\t\tlog.Debug(\"  => %s WINS on hints\", node2.Name())\n\t\t\treturn false\n\t\t}\n\n\t\tlog.Debug(\"  - hints are a TIE\")\n\n\t\tif hs1 == 0 {\n\t\t\tif nz1 > nz2 {\n\t\t\t\tlog.Debug(\"  => %s WINS on non-zero hints\", node1.Name())\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif nz2 > nz1 {\n\t\t\t\tlog.Debug(\"  => %s WINS on non-zero hints\", node2.Name())\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tlog.Debug(\"  - non-zero hints are a TIE\")\n\t\t}\n\n\t\t// for a tie, prefer lower nodes and smaller ids\n\t\tif hs1 == hs2 && nz1 == nz2 && (hs1 != 0 || nz1 != 0) {\n\t\t\tif depth1 > depth2 {\n\t\t\t\tlog.Debug(\"  => %s WINS as it is lower\", node1.Name())\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif depth1 < depth2 {\n\t\t\t\tlog.Debug(\"  => %s WINS as it is lower\", node2.Name())\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tlog.Debug(\"  => %s WINS based on equal hint socres, lower id\",\n\t\t\t\tmap[bool]string{true: node1.Name(), false: node2.Name()}[id1 < id2])\n\n\t\t\treturn id1 < id2\n\t\t}\n\t}\n\n\t// 5) a lower node wins\n\tif depth1 > depth2 {\n\t\tlog.Debug(\"  => %s WINS on depth\", node1.Name())\n\t\treturn true\n\t}\n\tif depth1 < depth2 {\n\t\tlog.Debug(\"  => %s WINS on depth\", node2.Name())\n\t\treturn false\n\t}\n\n\tlog.Debug(\"  - depth is a TIE\")\n\n\tif request.CPUType() == cpuReserved {\n\t\t// 6) if requesting reserved CPUs, more reserved\n\t\t//    capacity per colocated container wins. Reserved\n\t\t//    CPUs cannot be precisely accounted as they run\n\t\t//    also BestEffort containers that do not carry\n\t\t//    information on their CPU needs.\n\t\tif reserved1/(score1.Colocated()+1) > reserved2/(score2.Colocated()+1) {\n\t\t\treturn true\n\t\t}\n\t\tif reserved2/(score2.Colocated()+1) > reserved1/(score1.Colocated()+1) {\n\t\t\treturn false\n\t\t}\n\t\tlog.Debug(\"  - reserved capacity is a TIE\")\n\t} else if request.CPUType() == cpuNormal {\n\t\t// 7) more isolated capacity wins\n\t\tif request.Isolate() && (isolated1 > 0 || isolated2 > 0) {\n\t\t\tif isolated1 > isolated2 {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif isolated2 > isolated1 {\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tlog.Debug(\"  => %s WINS based on equal isolated capacity, lower id\",\n\t\t\t\tmap[bool]string{true: node1.Name(), false: node2.Name()}[id1 < id2])\n\n\t\t\treturn id1 < id2\n\t\t}\n\n\t\t// 8) more slicable shared capacity wins\n\t\tif request.FullCPUs() > 0 && (shared1 > 0 || shared2 > 0) {\n\t\t\tif shared1 > shared2 {\n\t\t\t\tlog.Debug(\"  => %s WINS on more slicable capacity\", node1.Name())\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif shared2 > shared1 {\n\t\t\t\tlog.Debug(\"  => %s WINS on more slicable capacity\", node2.Name())\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tlog.Debug(\"  => %s WINS based on equal slicable capacity, lower id\",\n\t\t\t\tmap[bool]string{true: node1.Name(), false: node2.Name()}[id1 < id2])\n\n\t\t\treturn id1 < id2\n\t\t}\n\n\t\t// 9) fewer colocated containers win\n\t\tif score1.Colocated() < score2.Colocated() {\n\t\t\tlog.Debug(\"  => %s WINS on colocation score\", node1.Name())\n\t\t\treturn true\n\t\t}\n\t\tif score2.Colocated() < score1.Colocated() {\n\t\t\tlog.Debug(\"  => %s WINS on colocation score\", node2.Name())\n\t\t\treturn false\n\t\t}\n\n\t\tlog.Debug(\"  - colocation score is a TIE\")\n\n\t\t// more shared capacity wins\n\t\tif shared1 > shared2 {\n\t\t\tlog.Debug(\"  => %s WINS on more shared capacity\", node1.Name())\n\t\t\treturn true\n\t\t}\n\t\tif shared2 > shared1 {\n\t\t\tlog.Debug(\"  => %s WINS on more shared capacity\", node2.Name())\n\t\t\treturn false\n\t\t}\n\t}\n\n\t// 10) lower id wins\n\tlog.Debug(\"  => %s WINS based on lower id\",\n\t\tmap[bool]string{true: node1.Name(), false: node2.Name()}[id1 < id2])\n\n\treturn id1 < id2\n}\n\n// affinityScore calculate the 'goodness' of the affinity for a node.\nfunc affinityScore(affinities map[int]int32, node Node) float64 {\n\tQ := 0.75\n\n\t// Calculate affinity for every node as a combination of\n\t// affinities of the nodes on the path from the node to\n\t// the root and the nodes in the subtree under the node.\n\t//\n\t// The combined affinity for node n is Sum_x(A_x*D_x),\n\t// where for every node x, A_x is the affinity for x and\n\t// D_x is Q ** (number of links from node to x). IOW, the\n\t// effective affinity is the sum of the affinity of n and\n\t// the affinity of each node x of the above mentioned set\n\t// diluted proprotionally to the distance of x to n, with\n\t// Q being 0.75.\n\n\tvar score float64\n\tfor n, q := node.Parent(), Q; !n.IsNil(); n, q = n.Parent(), q*Q {\n\t\ta := affinities[n.NodeID()]\n\t\tscore += q * float64(a)\n\t}\n\tnode.BreadthFirst(func(n Node) error {\n\t\tdiff := float64(n.RootDistance() - node.RootDistance())\n\t\tq := math.Pow(Q, diff)\n\t\ta := affinities[n.NodeID()]\n\t\tscore += q * float64(a)\n\t\treturn nil\n\t})\n\treturn score\n}\n\n// hintScores calculates combined full and zero-filtered hint scores.\nfunc combineHintScores(scores map[string]float64) (float64, float64) {\n\tif len(scores) == 0 {\n\t\treturn 0.0, 0.0\n\t}\n\n\tcombined, filtered := 1.0, 0.0\n\tfor _, score := range scores {\n\t\tcombined *= score\n\t\tif score != 0.0 {\n\t\t\tif filtered == 0.0 {\n\t\t\t\tfiltered = score\n\t\t\t} else {\n\t\t\t\tfiltered *= score\n\t\t\t}\n\t\t}\n\t}\n\treturn combined, filtered\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/pools_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"testing\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tpolicyapi \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nfunc findNodeWithID(id int, nodes []Node) Node {\n\tfor _, node := range nodes {\n\t\tif node.NodeID() == id {\n\t\t\treturn node\n\t\t}\n\t}\n\tpanic(\"No node found with id \" + fmt.Sprintf(\"%d\", id))\n}\n\nfunc findNodeWithName(name string, nodes []Node) Node {\n\tfor _, node := range nodes {\n\t\tif node.Name() == name {\n\t\t\treturn node\n\t\t}\n\t}\n\tpanic(\"No node found with name \" + name)\n}\n\nfunc setLinks(nodes []Node, tree map[int][]int) {\n\thasParent := map[int]struct{}{}\n\tfor parent, children := range tree {\n\t\tparentNode := findNodeWithID(parent, nodes)\n\t\tfor _, child := range children {\n\t\t\tchildNode := findNodeWithID(child, nodes)\n\t\t\tchildNode.LinkParent(parentNode)\n\t\t\thasParent[child] = struct{}{}\n\t\t}\n\t}\n\torphans := []int{}\n\tfor id := range tree {\n\t\tif _, ok := hasParent[id]; !ok {\n\t\t\tnode := findNodeWithID(id, nodes)\n\t\t\tnode.LinkParent(nilnode)\n\t\t\torphans = append(orphans, id)\n\t\t}\n\t}\n\tif len(orphans) != 1 {\n\t\tpanic(fmt.Sprintf(\"expected one root node, got %d with IDs %v\", len(orphans), orphans))\n\t}\n}\n\nfunc TestMemoryLimitFiltering(t *testing.T) {\n\n\t// Test the scoring algorithm with synthetic data. The assumptions are:\n\n\t// 1. The first node in \"nodes\" is the root of the tree.\n\n\ttcases := []struct {\n\t\tname                   string\n\t\tnodes                  []Node\n\t\tnumaNodes              []system.Node\n\t\treq                    Request\n\t\taffinities             map[int]int32\n\t\ttree                   map[int][]int\n\t\texpectedRemainingNodes []int\n\t}{\n\t\t{\n\t\t\tname: \"single node memory limit (fits)\",\n\t\t\tnodes: []Node{\n\t\t\t\t&numanode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      100,\n\t\t\t\t\t\tname:    \"testnode0\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t\tid: 0, // system node id\n\t\t\t\t},\n\t\t\t},\n\t\t\tnumaNodes: []system.Node{\n\t\t\t\t&mockSystemNode{id: 0, memFree: 10001, memTotal: 10001},\n\t\t\t},\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   defaultMemoryType,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{100},\n\t\t\ttree:                   map[int][]int{100: {}},\n\t\t},\n\t\t{\n\t\t\tname: \"single node memory limit (doesn't fit)\",\n\t\t\tnodes: []Node{\n\t\t\t\t&numanode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      100,\n\t\t\t\t\t\tname:    \"testnode0\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(9999, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(9999, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t\tid: 0, // system node id\n\t\t\t\t},\n\t\t\t},\n\t\t\tnumaNodes: []system.Node{\n\t\t\t\t&mockSystemNode{id: 0, memFree: 9999, memTotal: 9999},\n\t\t\t},\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   defaultMemoryType,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{},\n\t\t\ttree:                   map[int][]int{100: {}},\n\t\t},\n\t\t{\n\t\t\tname: \"two node memory limit (fits to leaf)\",\n\t\t\tnodes: []Node{\n\t\t\t\t&virtualnode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      100,\n\t\t\t\t\t\tname:    \"testnode0\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t&numanode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      101,\n\t\t\t\t\t\tname:    \"testnode1\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(10001, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t\tid: 0, // system node id\n\t\t\t\t},\n\t\t\t},\n\t\t\tnumaNodes: []system.Node{\n\t\t\t\t&mockSystemNode{id: 0, memFree: 10001, memTotal: 10001},\n\t\t\t},\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   defaultMemoryType,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{100, 101},\n\t\t\ttree:                   map[int][]int{100: {101}, 101: {}},\n\t\t},\n\t\t{\n\t\t\tname: \"three node memory limit (fits to root)\",\n\t\t\tnodes: []Node{\n\t\t\t\t&virtualnode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      100,\n\t\t\t\t\t\tname:    \"testnode0\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(12000, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(12000, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t&numanode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      101,\n\t\t\t\t\t\tname:    \"testnode1\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t\tid: 0, // system node id\n\t\t\t\t},\n\t\t\t\t&numanode{\n\t\t\t\t\tnode: node{\n\t\t\t\t\t\tid:      102,\n\t\t\t\t\t\tname:    \"testnode2\",\n\t\t\t\t\t\tkind:    UnknownNode,\n\t\t\t\t\t\tnoderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t\tfreeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0, createMemoryMap(6000, 0, 0), createMemoryMap(0, 0, 0)),\n\t\t\t\t\t},\n\t\t\t\t\tid: 1, // system node id\n\t\t\t\t},\n\t\t\t},\n\t\t\tnumaNodes: []system.Node{\n\t\t\t\t&mockSystemNode{id: 0, memFree: 6000, memTotal: 6000},\n\t\t\t\t&mockSystemNode{id: 1, memFree: 6000, memTotal: 6000},\n\t\t\t},\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   defaultMemoryType,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{100},\n\t\t\ttree:                   map[int][]int{100: {101, 102}, 101: {}, 102: {}},\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tsetLinks(tc.nodes, tc.tree)\n\t\t\tpolicy := &policy{\n\t\t\t\tsys: &mockSystem{\n\t\t\t\t\tnodes: tc.numaNodes,\n\t\t\t\t},\n\t\t\t\tpools:       tc.nodes,\n\t\t\t\tcache:       &mockCache{},\n\t\t\t\troot:        tc.nodes[0],\n\t\t\t\tnodeCnt:     len(tc.nodes),\n\t\t\t\tallocations: allocations{},\n\t\t\t}\n\t\t\t// back pointers\n\t\t\tfor _, node := range tc.nodes {\n\t\t\t\tswitch node.(type) {\n\t\t\t\tcase *numanode:\n\t\t\t\t\tnumaNode := node.(*numanode)\n\t\t\t\t\tnumaNode.self.node = numaNode\n\t\t\t\t\tnoderes := numaNode.noderes.(*supply)\n\t\t\t\t\tnoderes.node = node\n\t\t\t\t\tfreeres := numaNode.freeres.(*supply)\n\t\t\t\t\tfreeres.node = node\n\t\t\t\t\tnumaNode.policy = policy\n\t\t\t\tcase *virtualnode:\n\t\t\t\t\tvirtualNode := node.(*virtualnode)\n\t\t\t\t\tvirtualNode.self.node = virtualNode\n\t\t\t\t\tnoderes := virtualNode.noderes.(*supply)\n\t\t\t\t\tnoderes.node = node\n\t\t\t\t\tfreeres := virtualNode.freeres.(*supply)\n\t\t\t\t\tfreeres.node = node\n\t\t\t\t\tvirtualNode.policy = policy\n\t\t\t\t}\n\t\t\t}\n\t\t\tpolicy.allocations.policy = policy\n\n\t\t\tscores, filteredPools := policy.sortPoolsByScore(tc.req, tc.affinities)\n\t\t\tfmt.Printf(\"scores: %v, remaining pools: %v\\n\", scores, filteredPools)\n\n\t\t\tif len(filteredPools) != len(tc.expectedRemainingNodes) {\n\t\t\t\tt.Errorf(\"Wrong nodes in the filtered pool: expected %v but got %v\", tc.expectedRemainingNodes, filteredPools)\n\t\t\t}\n\n\t\t\tfor _, id := range tc.expectedRemainingNodes {\n\t\t\t\tfound := false\n\t\t\t\tfor _, node := range filteredPools {\n\t\t\t\t\tif node.NodeID() == id {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tt.Errorf(\"Did not find id %d in filtered pools: %v\", id, filteredPools)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestPoolCreation(t *testing.T) {\n\n\t// Test pool creation with \"real\" sysfs data.\n\n\t// Create a temporary directory for the test data.\n\tdir, err := os.MkdirTemp(\"\", \"cri-resource-manager-test-sysfs-\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer os.RemoveAll(dir)\n\n\t// Uncompress the test data to the directory.\n\terr = utils.UncompressTbz2(path.Join(\"testdata\", \"sysfs.tar.bz2\"), dir)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\ttcases := []struct {\n\t\tpath                    string\n\t\tname                    string\n\t\treq                     Request\n\t\taffinities              map[int]int32\n\t\texpectedRemainingNodes  []int\n\t\texpectedFirstNodeMemory memoryType\n\t\texpectedLeafNodeCPUs    int\n\t\texpectedRootNodeCPUs    int\n\t\t// TODO: expectedRootNodeMemory   int\n\t}{\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"desktop\", \"sys\"),\n\t\t\tname: \"sysfs pool creation from a desktop system\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryAll,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes:  []int{0},\n\t\t\texpectedFirstNodeMemory: memoryDRAM,\n\t\t\texpectedLeafNodeCPUs:    20,\n\t\t\texpectedRootNodeCPUs:    20,\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"sysfs pool creation from a server system\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryDRAM,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes:  []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\texpectedFirstNodeMemory: memoryDRAM | memoryPMEM,\n\t\t\texpectedLeafNodeCPUs:    28,\n\t\t\texpectedRootNodeCPUs:    112,\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"pmem request on a server system\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryDRAM | memoryPMEM,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes:  []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\texpectedFirstNodeMemory: memoryDRAM | memoryPMEM,\n\t\t\texpectedLeafNodeCPUs:    28,\n\t\t\texpectedRootNodeCPUs:    112,\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"4-socket-server-nosnc\", \"sys\"),\n\t\t\tname: \"sysfs pool creation from a 4 socket server with SNC disabled\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryAll,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes:  []int{0, 1, 2, 3, 4},\n\t\t\texpectedFirstNodeMemory: memoryDRAM,\n\t\t\texpectedLeafNodeCPUs:    36,\n\t\t\texpectedRootNodeCPUs:    36 * 4,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tsys, err := system.DiscoverSystemAt(tc.path)\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\n\t\t\treserved, _ := resapi.ParseQuantity(\"750m\")\n\t\t\tpolicyOptions := &policyapi.BackendOptions{\n\t\t\t\tCache:  &mockCache{},\n\t\t\t\tSystem: sys,\n\t\t\t\tReserved: policyapi.ConstraintSet{\n\t\t\t\t\tpolicyapi.DomainCPU: reserved,\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tlog.EnableDebug()\n\t\t\tpolicy := CreateTopologyAwarePolicy(policyOptions).(*policy)\n\n\t\t\tif policy.root.GetSupply().SharableCPUs().Size()+policy.root.GetSupply().IsolatedCPUs().Size()+policy.root.GetSupply().ReservedCPUs().Size() != tc.expectedRootNodeCPUs {\n\t\t\t\tt.Errorf(\"Expected %d CPUs, got %d\", tc.expectedRootNodeCPUs,\n\t\t\t\t\tpolicy.root.GetSupply().SharableCPUs().Size()+policy.root.GetSupply().IsolatedCPUs().Size()+policy.root.GetSupply().ReservedCPUs().Size())\n\t\t\t}\n\n\t\t\tfor _, p := range policy.pools {\n\t\t\t\tif p.IsLeafNode() {\n\t\t\t\t\tif len(p.Children()) != 0 {\n\t\t\t\t\t\tt.Errorf(\"Leaf node %v had %d children\", p, len(p.Children()))\n\t\t\t\t\t}\n\t\t\t\t\tif p.GetSupply().SharableCPUs().Size()+p.GetSupply().IsolatedCPUs().Size()+p.GetSupply().ReservedCPUs().Size() != tc.expectedLeafNodeCPUs {\n\t\t\t\t\t\tt.Errorf(\"Expected %d CPUs, got %d (%s)\", tc.expectedLeafNodeCPUs,\n\t\t\t\t\t\t\tp.GetSupply().SharableCPUs().Size()+p.GetSupply().IsolatedCPUs().Size()+p.GetSupply().ReservedCPUs().Size(),\n\t\t\t\t\t\t\tp.GetSupply().DumpCapacity())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tscores, filteredPools := policy.sortPoolsByScore(tc.req, tc.affinities)\n\t\t\tfmt.Printf(\"scores: %v, remaining pools: %v\\n\", scores, filteredPools)\n\n\t\t\tif len(filteredPools) != len(tc.expectedRemainingNodes) {\n\t\t\t\tt.Errorf(\"Wrong number of nodes in the filtered pool: expected %d but got %d\", len(tc.expectedRemainingNodes), len(filteredPools))\n\t\t\t}\n\n\t\t\tfor _, id := range tc.expectedRemainingNodes {\n\t\t\t\tfound := false\n\t\t\t\tfor _, node := range filteredPools {\n\t\t\t\t\tif node.NodeID() == id {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tt.Errorf(\"Did not find id %d in filtered pools: %s\", id, filteredPools)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif len(filteredPools) > 0 && filteredPools[0].GetMemoryType() != tc.expectedFirstNodeMemory {\n\t\t\t\tt.Errorf(\"Expected first node memory type %v, got %v\", tc.expectedFirstNodeMemory, filteredPools[0].GetMemoryType())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestWorkloadPlacement(t *testing.T) {\n\n\t// Do some workloads (containers) and see how they are placed in the\n\t// server system.\n\n\t// Create a temporary directory for the test data.\n\tdir, err := os.MkdirTemp(\"\", \"cri-resource-manager-test-sysfs-\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer os.RemoveAll(dir)\n\n\t// Uncompress the test data to the directory.\n\terr = utils.UncompressTbz2(path.Join(\"testdata\", \"sysfs.tar.bz2\"), dir)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\ttcases := []struct {\n\t\tpath                   string\n\t\tname                   string\n\t\treq                    Request\n\t\taffinities             map[int]int32\n\t\texpectedRemainingNodes []int\n\t\texpectedLeafNode       bool\n\t}{\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"workload placement on a server system leaf node\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:  10000,\n\t\t\t\tmemLim:  10000,\n\t\t\t\tmemType: memoryUnspec,\n\t\t\t\tisolate: false,\n\t\t\t\tfull:    25, // 28 - 2 isolated = 26: but fully exhausting the shared CPU subpool is disallowed\n\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\texpectedLeafNode:       true,\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"workload placement on a server system root node: CPUs don't fit to leaf\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      29,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\texpectedLeafNode:       false,\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"workload placement on a server system root node: memory doesn't fit to leaf\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    190000000000,\n\t\t\t\tmemLim:    190000000000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      28,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\texpectedRemainingNodes: []int{2, 6},\n\t\t\texpectedLeafNode:       false,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tsys, err := system.DiscoverSystemAt(tc.path)\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\n\t\t\treserved, _ := resapi.ParseQuantity(\"750m\")\n\t\t\tpolicyOptions := &policyapi.BackendOptions{\n\t\t\t\tCache:  &mockCache{},\n\t\t\t\tSystem: sys,\n\t\t\t\tReserved: policyapi.ConstraintSet{\n\t\t\t\t\tpolicyapi.DomainCPU: reserved,\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tlog.EnableDebug()\n\t\t\tpolicy := CreateTopologyAwarePolicy(policyOptions).(*policy)\n\n\t\t\tscores, filteredPools := policy.sortPoolsByScore(tc.req, tc.affinities)\n\t\t\tfmt.Printf(\"scores: %v, remaining pools: %v\\n\", scores, filteredPools)\n\n\t\t\tif len(filteredPools) != len(tc.expectedRemainingNodes) {\n\t\t\t\tt.Errorf(\"Wrong number of nodes in the filtered pool: expected %d but got %d\", len(tc.expectedRemainingNodes), len(filteredPools))\n\t\t\t}\n\n\t\t\tfor _, id := range tc.expectedRemainingNodes {\n\t\t\t\tfound := false\n\t\t\t\tfor _, node := range filteredPools {\n\t\t\t\t\tif node.NodeID() == id {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif !found {\n\t\t\t\t\tt.Errorf(\"Did not find id %d in filtered pools: %s\", id, filteredPools)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif filteredPools[0].IsLeafNode() != tc.expectedLeafNode {\n\t\t\t\tt.Errorf(\"Workload should have been placed in a leaf node: %t\", tc.expectedLeafNode)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestContainerMove(t *testing.T) {\n\n\t// In case there's not enough memory to guarantee that the\n\t// containers running on child nodes won't get OOM killed, they need\n\t// to be moved upwards in the tree.\n\n\t// Create a temporary directory for the test data.\n\tdir, err := os.MkdirTemp(\"\", \"cri-resource-manager-test-sysfs-\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer os.RemoveAll(dir)\n\n\t// Uncompress the test data to the directory.\n\terr = utils.UncompressTbz2(path.Join(\"testdata\", \"sysfs.tar.bz2\"), dir)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\ttcases := []struct {\n\t\tpath                          string\n\t\tname                          string\n\t\tcontainer1                    cache.Container\n\t\tcontainer2                    cache.Container\n\t\tcontainer3                    cache.Container\n\t\taffinities                    map[int]int32\n\t\texpectedLeafNodeForContainer1 bool\n\t\texpectedLeafNodeForContainer2 bool\n\t\texpectedLeafNodeForContainer3 bool\n\t\texpectedChangeForContainer1   bool\n\t\texpectedChangeForContainer2   bool\n\t\texpectedChangeForContainer3   bool\n\t}{\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"workload placement on a server system leaf node\",\n\t\t\tcontainer1: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tLimits: v1.ResourceList{\n\t\t\t\t\t\tv1.ResourceCPU:    resapi.MustParse(\"2\"),\n\t\t\t\t\t\tv1.ResourceMemory: resapi.MustParse(\"1000\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\treturnValueForGetCacheID: \"first\",\n\t\t\t},\n\t\t\tcontainer2: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tLimits: v1.ResourceList{\n\t\t\t\t\t\tv1.ResourceCPU:    resapi.MustParse(\"2\"),\n\t\t\t\t\t\tv1.ResourceMemory: resapi.MustParse(\"1000\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\treturnValueForGetCacheID: \"second\",\n\t\t\t},\n\t\t\tcontainer3: &mockContainer{\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tLimits: v1.ResourceList{\n\t\t\t\t\t\tv1.ResourceCPU:    resapi.MustParse(\"2\"),\n\t\t\t\t\t\tv1.ResourceMemory: resapi.MustParse(\"1000\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\treturnValueForGetCacheID: \"third\",\n\t\t\t},\n\t\t\texpectedLeafNodeForContainer1: true,\n\t\t\texpectedLeafNodeForContainer2: true,\n\t\t\texpectedLeafNodeForContainer3: true,\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"workload placement on a server system non-leaf node\",\n\t\t\tcontainer1: &mockContainer{\n\t\t\t\tname: \"c1\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tLimits: v1.ResourceList{\n\t\t\t\t\t\tv1.ResourceCPU:    resapi.MustParse(\"2\"),\n\t\t\t\t\t\tv1.ResourceMemory: resapi.MustParse(\"1000\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\treturnValueForGetCacheID: \"first\",\n\t\t\t},\n\t\t\tcontainer2: &mockContainer{\n\t\t\t\tname: \"c2\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tLimits: v1.ResourceList{\n\t\t\t\t\t\tv1.ResourceCPU:    resapi.MustParse(\"2\"),\n\t\t\t\t\t\tv1.ResourceMemory: resapi.MustParse(\"190000000000\"), // 180 GB\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\treturnValueForGetCacheID: \"second\",\n\t\t\t},\n\t\t\tcontainer3: &mockContainer{\n\t\t\t\tname: \"c3\",\n\t\t\t\treturnValueForGetResourceRequirements: v1.ResourceRequirements{\n\t\t\t\t\tLimits: v1.ResourceList{\n\t\t\t\t\t\tv1.ResourceCPU:    resapi.MustParse(\"2\"),\n\t\t\t\t\t\tv1.ResourceMemory: resapi.MustParse(\"140000000000\"), // 130 GB\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\treturnValueForGetCacheID: \"third\",\n\t\t\t},\n\t\t\texpectedLeafNodeForContainer1: false,\n\t\t\texpectedLeafNodeForContainer2: false,\n\t\t\texpectedLeafNodeForContainer3: false,\n\t\t\texpectedChangeForContainer1:   true,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tsys, err := system.DiscoverSystemAt(tc.path)\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\n\t\t\treserved, _ := resapi.ParseQuantity(\"750m\")\n\t\t\tpolicyOptions := &policyapi.BackendOptions{\n\t\t\t\tCache:  &mockCache{},\n\t\t\t\tSystem: sys,\n\t\t\t\tReserved: policyapi.ConstraintSet{\n\t\t\t\t\tpolicyapi.DomainCPU: reserved,\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tlog.EnableDebug()\n\t\t\tpolicy := CreateTopologyAwarePolicy(policyOptions).(*policy)\n\n\t\t\tgrant1, err := policy.allocatePool(tc.container1, \"\")\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t\tfmt.Printf(\"grant 1 memsets: dram %s, pmem %s\\n\", grant1.GetMemoryNode().GetMemset(memoryDRAM), grant1.GetMemoryNode().GetMemset(memoryPMEM))\n\n\t\t\tgrant2, err := policy.allocatePool(tc.container2, \"\")\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t\tfmt.Printf(\"grant 2 memsets: dram %s, pmem %s\\n\", grant2.GetMemoryNode().GetMemset(memoryDRAM), grant2.GetMemoryNode().GetMemset(memoryPMEM))\n\n\t\t\tgrant3, err := policy.allocatePool(tc.container3, \"\")\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t\tfmt.Printf(\"grant 3 memsets: dram %s, pmem %s\\n\", grant3.GetMemoryNode().GetMemset(memoryDRAM), grant3.GetMemoryNode().GetMemset(memoryPMEM))\n\n\t\t\tif (grant1.GetCPUNode().IsSameNode(grant1.GetMemoryNode())) && tc.expectedChangeForContainer1 {\n\t\t\t\tt.Errorf(\"Workload 1 should have been relocated: %t, node: %s\", tc.expectedChangeForContainer1, grant1.GetMemoryNode().Name())\n\t\t\t}\n\t\t\tif (grant2.GetCPUNode().IsSameNode(grant2.GetMemoryNode())) && tc.expectedChangeForContainer2 {\n\t\t\t\tt.Errorf(\"Workload 2 should have been relocated: %t, node: %s\", tc.expectedChangeForContainer2, grant2.GetMemoryNode().Name())\n\t\t\t}\n\t\t\tif (grant3.GetCPUNode().IsSameNode(grant3.GetMemoryNode())) && tc.expectedChangeForContainer3 {\n\t\t\t\tt.Errorf(\"Workload 3 should have been relocated: %t, node: %s\", tc.expectedChangeForContainer3, grant3.GetMemoryNode().Name())\n\t\t\t}\n\n\t\t\tif grant1.GetMemoryNode().IsLeafNode() != tc.expectedLeafNodeForContainer1 {\n\t\t\t\tt.Errorf(\"Workload 1 should have been placed in a leaf node: %t, node: %s\", tc.expectedLeafNodeForContainer1, grant1.GetMemoryNode().Name())\n\t\t\t}\n\t\t\tif grant2.GetMemoryNode().IsLeafNode() != tc.expectedLeafNodeForContainer2 {\n\t\t\t\tt.Errorf(\"Workload 2 should have been placed in a leaf node: %t, node: %s\", tc.expectedLeafNodeForContainer2, grant2.GetMemoryNode().Name())\n\t\t\t}\n\t\t\tif grant3.GetMemoryNode().IsLeafNode() != tc.expectedLeafNodeForContainer3 {\n\t\t\t\tt.Errorf(\"Workload 3 should have been placed in a leaf node: %t, node: %s\", tc.expectedLeafNodeForContainer3, grant3.GetMemoryNode().Name())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestAffinities(t *testing.T) {\n\t//\n\t// Test how (already pre-calculated) affinities affect workload placement.\n\t//\n\n\t// Create a temporary directory for the test data.\n\tdir, err := os.MkdirTemp(\"\", \"cri-resource-manager-test-sysfs-\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer os.RemoveAll(dir)\n\n\t// Uncompress the test data to the directory.\n\terr = utils.UncompressTbz2(path.Join(\"testdata\", \"sysfs.tar.bz2\"), dir)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\ttcases := []struct {\n\t\tpath       string\n\t\tname       string\n\t\treq        Request\n\t\taffinities map[string]int32\n\t\texpected   string\n\t}{\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"no affinities\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{},\n\t\t\texpected:   \"NUMA node #2\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"reserved - no affinities\",\n\t\t\treq: &request{\n\t\t\t\tcpuType:   cpuReserved,\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      0,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{},\n\t\t\texpected:   \"NUMA node #0\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"affinity to NUMA node #1\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #1\": 1,\n\t\t\t},\n\t\t\texpected: \"NUMA node #1\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"affinity to socket #1\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"socket #1\": 1,\n\t\t\t},\n\t\t\texpected: \"socket #1\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"equal affinities to NUMA node #1, socket #1\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"socket #1\":    1,\n\t\t\t\t\"NUMA node #1\": 1,\n\t\t\t},\n\t\t\texpected: \"NUMA node #1\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"equal affinities to NUMA node #1, NUMA node #3\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #1\": 1,\n\t\t\t\t\"NUMA node #3\": 1,\n\t\t\t},\n\t\t\texpected: \"socket #1\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"double affinity to NUMA node #1 vs. #3\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #1\": 2,\n\t\t\t\t\"NUMA node #3\": 1,\n\t\t\t},\n\t\t\texpected: \"socket #1\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"triple affinity to NUMA node #1 vs. #3\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #1\": 3,\n\t\t\t\t\"NUMA node #3\": 1,\n\t\t\t},\n\t\t\texpected: \"NUMA node #1\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"double affinity to NUMA node #0,#3 vs. socket #1\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #0\": 2,\n\t\t\t\t\"NUMA node #3\": 2,\n\t\t\t\t\"socket #1\":    1,\n\t\t\t},\n\t\t\texpected: \"root\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"equal affinity to NUMA node #0,#3 vs. socket #1\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #0\": 1,\n\t\t\t\t\"NUMA node #3\": 1,\n\t\t\t\t\"socket #1\":    1,\n\t\t\t},\n\t\t\texpected: \"root\",\n\t\t},\n\t\t{\n\t\t\tpath: path.Join(dir, \"sysfs\", \"server\", \"sys\"),\n\t\t\tname: \"half the affinity to NUMA node #0,#3 vs. socket #1\",\n\t\t\treq: &request{\n\t\t\t\tmemReq:    10000,\n\t\t\t\tmemLim:    10000,\n\t\t\t\tmemType:   memoryUnspec,\n\t\t\t\tisolate:   false,\n\t\t\t\tfull:      3,\n\t\t\t\tcontainer: &mockContainer{},\n\t\t\t},\n\t\t\taffinities: map[string]int32{\n\t\t\t\t\"NUMA node #0\": 1,\n\t\t\t\t\"NUMA node #3\": 1,\n\t\t\t\t\"socket #1\":    2,\n\t\t\t},\n\t\t\texpected: \"socket #1\",\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tsys, err := system.DiscoverSystemAt(tc.path)\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\n\t\t\treserved, _ := resapi.ParseQuantity(\"750m\")\n\t\t\tpolicyOptions := &policyapi.BackendOptions{\n\t\t\t\tCache:  &mockCache{},\n\t\t\t\tSystem: sys,\n\t\t\t\tReserved: policyapi.ConstraintSet{\n\t\t\t\t\tpolicyapi.DomainCPU: reserved,\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tlog.EnableDebug()\n\t\t\tpolicy := CreateTopologyAwarePolicy(policyOptions).(*policy)\n\n\t\t\taffinities := map[int]int32{}\n\t\t\tfor name, weight := range tc.affinities {\n\t\t\t\taffinities[findNodeWithName(name, policy.pools).NodeID()] = weight\n\t\t\t}\n\n\t\t\tlog.EnableDebug()\n\t\t\tscores, filteredPools := policy.sortPoolsByScore(tc.req, affinities)\n\t\t\tfmt.Printf(\"scores: %v, remaining pools: %v\\n\", scores, filteredPools)\n\n\t\t\tif len(filteredPools) < 1 {\n\t\t\t\tt.Errorf(\"pool scoring failed to find any pools\")\n\t\t\t}\n\n\t\t\tnode := filteredPools[0]\n\t\t\tif node.Name() != tc.expected {\n\t\t\t\tt.Errorf(\"expected best pool %s, got %s\", tc.expected, node.Name())\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/resources.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"time\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// Supply represents avaialbe CPU and memory capacity of a node.\ntype Supply interface {\n\t// GetNode returns the node supplying this capacity.\n\tGetNode() Node\n\t// Clone creates a copy of this supply.\n\tClone() Supply\n\t// IsolatedCPUs returns the isolated cpuset in this supply.\n\tIsolatedCPUs() cpuset.CPUSet\n\t// ReservedCPUs returns the reserved cpuset in this supply.\n\tReservedCPUs() cpuset.CPUSet\n\t// SharableCPUs returns the sharable cpuset in this supply.\n\tSharableCPUs() cpuset.CPUSet\n\t// GrantedReserved returns the locally granted reserved CPU capacity in this supply.\n\tGrantedReserved() int\n\t// GrantedShared returns the locally granted shared CPU capacity in this supply.\n\tGrantedShared() int\n\t// GrantedMemory returns the locally granted memory capacity in this supply.\n\tGrantedMemory(memoryType) uint64\n\t// Cumulate cumulates the given supply into this one.\n\tCumulate(Supply)\n\t// AssignMemory adds extra memory to this supply (for extra NUMA nodes assigned to a pool).\n\tAssignMemory(mem memoryMap)\n\t// AccountAllocateCPU accounts for (removes) allocated exclusive capacity from the supply.\n\tAccountAllocateCPU(Grant)\n\t// AccountReleaseCPU accounts for (reinserts) released exclusive capacity into the supply.\n\tAccountReleaseCPU(Grant)\n\t// GetScore calculates how well this supply fits/fulfills the given request.\n\tGetScore(Request) Score\n\t// AllocatableSharedCPU calculates the allocatable amount of shared CPU of this supply.\n\tAllocatableSharedCPU(...bool) int\n\t// Allocate allocates CPU capacity from this supply and returns it as a grant.\n\tAllocate(Request) (Grant, error)\n\t// ReleaseCPU releases a previously allocated CPU grant from this supply.\n\tReleaseCPU(Grant)\n\t// ReleaseMemory releases a previously allocated memory grant from this supply.\n\tReleaseMemory(Grant)\n\t// ReallocateMemory updates the Grant to allocate memory from this supply.\n\tReallocateMemory(Grant) error\n\t// ExtraMemoryReservation returns the memory reservation.\n\tExtraMemoryReservation(memoryType) uint64\n\t// SetExtraMemroyReservation sets the extra memory reservation based on the granted memory.\n\tSetExtraMemoryReservation(Grant)\n\t// ReleaseExtraMemoryReservation removes the extra memory reservations based on the granted memory.\n\tReleaseExtraMemoryReservation(Grant)\n\t// MemoryLimit returns the amount of various memory types belonging to this grant.\n\tMemoryLimit() memoryMap\n\n\t// Reserve accounts for CPU grants after reloading cached allocations.\n\tReserve(Grant) error\n\t// ReserveMemory accounts for memory grants after reloading cached allocations.\n\tReserveMemory(Grant) error\n\t// DumpCapacity returns a printable representation of the supply's resource capacity.\n\tDumpCapacity() string\n\t// DumpAllocatable returns a printable representation of the supply's alloctable resources.\n\tDumpAllocatable() string\n\t// DumpMemoryState dumps the state of the available and allocated memory.\n\tDumpMemoryState(string)\n}\n\n// Request represents CPU and memory resources requested by a container.\ntype Request interface {\n\t// GetContainer returns the container requesting CPU capacity.\n\tGetContainer() cache.Container\n\t// String returns a printable representation of this request.\n\tString() string\n\t// CPUType returns the type of requested CPU.\n\tCPUType() cpuClass\n\t// SetCPUType sets the type of requested CPU.\n\tSetCPUType(cpuType cpuClass)\n\t// FullCPUs return the number of full CPUs requested.\n\tFullCPUs() int\n\t// CPUFraction returns the amount of fractional milli-CPU requested.\n\tCPUFraction() int\n\t// Isolate returns whether isolated CPUs are preferred for this request.\n\tIsolate() bool\n\t// MemoryType returns the type(s) of requested memory.\n\tMemoryType() memoryType\n\t// MemAmountToAllocate retuns how much memory we need to reserve for a request.\n\tMemAmountToAllocate() uint64\n\t// ColdStart returns the cold start timeout.\n\tColdStart() time.Duration\n}\n\n// Grant represents CPU and memory capacity allocated to a container from a node.\ntype Grant interface {\n\t// SetCPUPortion sets the fraction CPU portion for the grant.\n\tSetCPUPortion(fraction int)\n\t// SetMemoryAllocation sets the memory allocation for the grant.\n\tSetMemoryAllocation(memoryType, memoryMap, time.Duration)\n\t// Clone creates a copy of this grant.\n\tClone() Grant\n\t// RefetchNodes updates the stored cpu and memory nodes of this grant by name.\n\tRefetchNodes() error\n\t// GetContainer returns the container CPU capacity is granted to.\n\tGetContainer() cache.Container\n\t// GetCPUNode returns the node that granted CPU capacity to the container.\n\tGetCPUNode() Node\n\t// GetMemoryNode returns the node which granted memory capacity to\n\t// the container.\n\tGetMemoryNode() Node\n\t// CPUType returns the type of granted CPUs\n\tCPUType() cpuClass\n\t// CPUPortion returns granted milli-CPUs of non-full CPUs of CPUType().\n\t// CPUPortion() == ReservedPortion() + SharedPortion().\n\tCPUPortion() int\n\t// ExclusiveCPUs returns the exclusively granted non-isolated cpuset.\n\tExclusiveCPUs() cpuset.CPUSet\n\t// ReservedCPUs returns the reserved granted cpuset.\n\tReservedCPUs() cpuset.CPUSet\n\t// ReservedPortion() returns the amount of CPUs in milli-CPU granted.\n\tReservedPortion() int\n\t// SharedCPUs returns the shared granted cpuset.\n\tSharedCPUs() cpuset.CPUSet\n\t// SharedPortion returns the amount of CPUs in milli-CPU granted.\n\tSharedPortion() int\n\t// IsolatedCpus returns the exclusively granted isolated cpuset.\n\tIsolatedCPUs() cpuset.CPUSet\n\t// MemoryType returns the type(s) of granted memory.\n\tMemoryType() memoryType\n\t// SetMemoryNode updates the grant memory controllers.\n\tSetMemoryNode(Node)\n\t// Memset returns the granted memory controllers as a string.\n\tMemset() idset.IDSet\n\t// ExpandMemset() makes the memory controller set larger as the grant\n\t// is moved up in the node hierarchy.\n\tExpandMemset() (bool, error)\n\t// MemLimit returns the amount of memory that the container is\n\t// allowed to use.\n\tMemLimit() memoryMap\n\t// String returns a printable representation of this grant.\n\tString() string\n\t// Release releases the grant from all the Supplys it uses.\n\tRelease()\n\t// AccountAllocateCPU accounts for (removes) allocated exclusive capacity for this grant.\n\tAccountAllocateCPU()\n\t// AccountReleaseCPU accounts for (reinserts) released exclusive capacity for this grant.\n\tAccountReleaseCPU()\n\t// UpdateExtraMemoryReservation() updates the reservations in the subtree\n\t// of nodes under the node from which the memory was granted.\n\tUpdateExtraMemoryReservation()\n\t// RestoreMemset restores the granted memory set to node maximum\n\t// and reapplies the grant.\n\tRestoreMemset()\n\t// ColdStart returns the cold start timeout.\n\tColdStart() time.Duration\n\t// AddTimer adds a cold start timer.\n\tAddTimer(*time.Timer)\n\t// StopTimer stops a cold start timer.\n\tStopTimer()\n\t// ClearTimer clears the cold start timer pointer.\n\tClearTimer()\n}\n\n// Score represents how well a supply can satisfy a request.\ntype Score interface {\n\t// Calculate the actual score from the collected parameters.\n\tEval() float64\n\t// Supply returns the supply associated with this score.\n\tSupply() Supply\n\t// Request returns the request associated with this score.\n\tRequest() Request\n\n\tIsolatedCapacity() int\n\tReservedCapacity() int\n\tSharedCapacity() int\n\tColocated() int\n\tHintScores() map[string]float64\n\n\tString() string\n}\n\ntype memoryMap map[memoryType]uint64\n\n// supply implements our Supply interface.\ntype supply struct {\n\tnode                 Node                // node supplying CPUs and memory\n\tisolated             cpuset.CPUSet       // isolated CPUs at this node\n\treserved             cpuset.CPUSet       // reserved CPUs at this node\n\tsharable             cpuset.CPUSet       // sharable CPUs at this node\n\tgrantedReserved      int                 // amount of reserved CPUs allocated\n\tgrantedShared        int                 // amount of shareable CPUs allocated\n\tmem                  memoryMap           // available memory for this node\n\tgrantedMem           memoryMap           // total memory granted\n\textraMemReservations map[Grant]memoryMap // how much memory each workload above has requested\n}\n\nvar _ Supply = &supply{}\n\n// request implements our Request interface.\ntype request struct {\n\tcontainer cache.Container // container for this request\n\tfull      int             // number of full CPUs requested\n\tfraction  int             // amount of fractional CPU requested\n\tisolate   bool            // prefer isolated exclusive CPUs\n\tcpuType   cpuClass        // preferred CPU type (normal, reserved)\n\n\tmemReq  uint64     // memory request\n\tmemLim  uint64     // memory limit\n\tmemType memoryType // requested types of memory\n\n\t// coldStart tells the timeout (in milliseconds) how long to wait until\n\t// a DRAM memory controller should be added to a container asking for a\n\t// mixed DRAM/PMEM memory allocation. This allows for a \"cold start\" where\n\t// initial memory requests are made to the PMEM memory. A value of 0\n\t// indicates that cold start is not explicitly requested.\n\tcoldStart time.Duration\n}\n\nvar _ Request = &request{}\n\n// grant implements our Grant interface.\ntype grant struct {\n\tcontainer      cache.Container // container CPU is granted to\n\tnode           Node            // node CPU is supplied from\n\tmemoryNode     Node            // node memory is supplied from\n\texclusive      cpuset.CPUSet   // exclusive CPUs\n\tcpuType        cpuClass        // type of CPUs (normal, reserved, ...)\n\tcpuPortion     int             // milliCPUs granted from CPUs of cpuType\n\tmemType        memoryType      // requested types of memory\n\tmemset         idset.IDSet     // assigned memory nodes\n\tallocatedMem   memoryMap       // memory limit\n\tcoldStart      time.Duration   // how long until cold start is done\n\tcoldStartTimer *time.Timer     // timer to trigger cold start timeout\n}\n\nvar _ Grant = &grant{}\n\n// score implements our Score interface.\ntype score struct {\n\tsupply    Supply             // CPU supply (node)\n\treq       Request            // CPU request (container)\n\tisolated  int                // remaining isolated CPUs\n\treserved  int                // remaining reserved CPUs\n\tshared    int                // remaining shared capacity\n\tcolocated int                // number of colocated containers\n\thints     map[string]float64 // hint scores\n}\n\nvar _ Score = &score{}\n\n// newSupply creates CPU supply for the given node, cpusets and existing grant.\n\nfunc newSupply(n Node, isolated, reserved, sharable cpuset.CPUSet, grantedReserved int, grantedShared int, mem, grantedMem memoryMap) Supply {\n\tif mem == nil {\n\t\tmem = createMemoryMap(0, 0, 0)\n\t}\n\tif grantedMem == nil {\n\t\tgrantedMem = createMemoryMap(0, 0, 0)\n\t}\n\treturn &supply{\n\t\tnode:                 n,\n\t\tisolated:             isolated.Clone(),\n\t\treserved:             reserved.Clone(),\n\t\tsharable:             sharable.Clone(),\n\t\tgrantedReserved:      grantedReserved,\n\t\tgrantedShared:        grantedShared,\n\t\tmem:                  mem,\n\t\tgrantedMem:           grantedMem,\n\t\textraMemReservations: make(map[Grant]memoryMap),\n\t}\n}\n\nfunc createMemoryMap(dram, pmem, hbm uint64) memoryMap {\n\treturn memoryMap{\n\t\tmemoryDRAM:   dram,\n\t\tmemoryPMEM:   pmem,\n\t\tmemoryHBM:    hbm,\n\t\tmemoryAll:    dram + pmem + hbm,\n\t\tmemoryUnspec: 0,\n\t}\n}\n\nfunc (m memoryMap) Add(dram, pmem, hbm uint64) {\n\tm[memoryDRAM] += dram\n\tm[memoryPMEM] += pmem\n\tm[memoryPMEM] += hbm\n\tm[memoryAll] += dram + pmem + hbm\n}\n\nfunc (m memoryMap) AddDRAM(dram uint64) {\n\tm[memoryDRAM] += dram\n\tm[memoryAll] += dram\n}\n\nfunc (m memoryMap) AddPMEM(pmem uint64) {\n\tm[memoryPMEM] += pmem\n\tm[memoryAll] += pmem\n}\n\nfunc (m memoryMap) AddHBM(hbm uint64) {\n\tm[memoryHBM] += hbm\n\tm[memoryAll] += hbm\n}\n\nfunc (m memoryMap) String() string {\n\tmem, sep := \"\", \"\"\n\n\tdram, pmem, hbm, types := m[memoryDRAM], m[memoryPMEM], m[memoryHBM], 0\n\tif dram > 0 || pmem > 0 || hbm > 0 {\n\t\tif dram > 0 {\n\t\t\tmem += \"DRAM \" + prettyMem(dram)\n\t\t\tsep = \", \"\n\t\t\ttypes++\n\t\t}\n\t\tif pmem > 0 {\n\t\t\tmem += sep + \"PMEM \" + prettyMem(pmem)\n\t\t\tsep = \", \"\n\t\t\ttypes++\n\t\t}\n\t\tif hbm > 0 {\n\t\t\tmem += sep + \"HBM \" + prettyMem(hbm)\n\t\t\ttypes++\n\t\t}\n\t\tif types > 1 {\n\t\t\tmem += sep + \"total \" + prettyMem(pmem+dram+hbm)\n\t\t}\n\t}\n\n\treturn mem\n}\n\n// GetNode returns the node supplying CPU and memory.\nfunc (cs *supply) GetNode() Node {\n\treturn cs.node\n}\n\n// Clone clones the given CPU supply.\nfunc (cs *supply) Clone() Supply {\n\t// Copy the maps.\n\tmem := make(memoryMap)\n\tfor key, value := range cs.mem {\n\t\tmem[key] = value\n\t}\n\tgrantedMem := make(memoryMap)\n\tfor key, value := range cs.grantedMem {\n\t\tgrantedMem[key] = value\n\t}\n\treturn newSupply(cs.node, cs.isolated, cs.reserved, cs.sharable, cs.grantedReserved, cs.grantedShared, mem, grantedMem)\n}\n\n// IsolatedCpus returns the isolated CPUSet of this supply.\nfunc (cs *supply) IsolatedCPUs() cpuset.CPUSet {\n\treturn cs.isolated.Clone()\n}\n\n// ReservedCpus returns the reserved CPUSet of this supply.\nfunc (cs *supply) ReservedCPUs() cpuset.CPUSet {\n\treturn cs.reserved.Clone()\n}\n\n// SharableCpus returns the sharable CPUSet of this supply.\nfunc (cs *supply) SharableCPUs() cpuset.CPUSet {\n\treturn cs.sharable.Clone()\n}\n\n// GrantedReserved returns the locally granted reserved CPU capacity.\nfunc (cs *supply) GrantedReserved() int {\n\treturn cs.grantedReserved\n}\n\n// GrantedShared returns the locally granted sharable CPU capacity.\nfunc (cs *supply) GrantedShared() int {\n\treturn cs.grantedShared\n}\n\nfunc (cs *supply) GrantedMemory(memType memoryType) uint64 {\n\t// Return only granted memory of correct type\n\treturn cs.grantedMem[memType]\n}\n\nfunc (cs *supply) MemoryLimit() memoryMap {\n\treturn cs.mem\n}\n\n// Cumulate more CPU to supply.\nfunc (cs *supply) Cumulate(more Supply) {\n\tmcs := more.(*supply)\n\n\tcs.isolated = cs.isolated.Union(mcs.isolated)\n\tcs.reserved = cs.reserved.Union(mcs.reserved)\n\tcs.sharable = cs.sharable.Union(mcs.sharable)\n\tcs.grantedReserved += mcs.grantedReserved\n\tcs.grantedShared += mcs.grantedShared\n\n\tfor key, value := range mcs.mem {\n\t\tcs.mem[key] += value\n\t}\n\tfor key, value := range mcs.grantedMem {\n\t\tcs.grantedMem[key] += value\n\t}\n}\n\n// AssignMemory adds memory (for extra NUMA nodes assigned to a pool node).\nfunc (cs *supply) AssignMemory(mem memoryMap) {\n\tfor key, value := range mem {\n\t\tcs.mem[key] += value\n\t}\n}\n\n// AccountAllocateCPU accounts for (removes) allocated exclusive capacity from the supply.\nfunc (cs *supply) AccountAllocateCPU(g Grant) {\n\tif cs.node.IsSameNode(g.GetCPUNode()) {\n\t\treturn\n\t}\n\texclusive := g.ExclusiveCPUs()\n\tcs.isolated = cs.isolated.Difference(exclusive)\n\tcs.sharable = cs.sharable.Difference(exclusive)\n}\n\n// AccountReleaseCPU accounts for (reinserts) released exclusive capacity into the supply.\nfunc (cs *supply) AccountReleaseCPU(g Grant) {\n\tif cs.node.IsSameNode(g.GetCPUNode()) {\n\t\treturn\n\t}\n\n\tncs := cs.node.GetSupply()\n\tnodecpus := ncs.IsolatedCPUs().Union(ncs.SharableCPUs())\n\tgrantcpus := g.ExclusiveCPUs().Intersection(nodecpus)\n\n\tisolated := grantcpus.Intersection(ncs.IsolatedCPUs())\n\tsharable := grantcpus.Intersection(ncs.SharableCPUs())\n\tcs.isolated = cs.isolated.Union(isolated)\n\tcs.sharable = cs.sharable.Union(sharable)\n}\n\n// allocateMemory tries to fulfill the memory allocation part of a request.\nfunc (cs *supply) allocateMemory(r Request) (memoryMap, error) {\n\treqType := r.MemoryType()\n\tif reqType == memoryUnspec {\n\t\treqType = memoryAll\n\t}\n\n\tallocated := createMemoryMap(0, 0, 0)\n\trequested := r.MemAmountToAllocate()\n\tremaining := requested\n\n\t//\n\t// Notes:\n\t//   We try to allocate PMEM, then DRAM, and finally HBM, honoring\n\t//   the types allowed by the request. We don't need to care about\n\t//   extra memory reservations for this node as all the nodes with\n\t//   insufficient memory have been filtered out before allocation.\n\t//\n\t//   However, for cold started containers we do check if there is\n\t//   enough PMEM free to accomodate the full request and bail out\n\t//   if that check fails.\n\t//\n\n\tfor _, memType := range []memoryType{memoryPMEM, memoryDRAM, memoryHBM} {\n\t\tif remaining > 0 && (reqType&memType) != 0 {\n\t\t\tavailable := cs.mem[memType]\n\n\t\t\tlog.Debug(\"%s: trying %s %s of %s available\",\n\t\t\t\tr.GetContainer().PrettyName(),\n\t\t\t\tprettyMem(remaining), memType.String(), prettyMem(available))\n\n\t\t\tif remaining <= available {\n\t\t\t\tallocated[memType] = remaining\n\t\t\t} else {\n\t\t\t\tallocated[memType] = available\n\t\t\t}\n\n\t\t\tcs.grantedMem[memType] += allocated[memType]\n\t\t\tcs.mem[memType] -= allocated[memType]\n\t\t\tremaining -= allocated[memType]\n\t\t}\n\n\t\tif remaining > 0 {\n\t\t\tif r.ColdStart() > 0 && memType == memoryPMEM {\n\t\t\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\t\t\"not enough PMEM for cold start at %s\", cs.GetNode().Name())\n\t\t\t}\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif remaining > 0 {\n\t\tlog.Debug(\"%s: %s allocation from %s fell short %s\",\n\t\t\tr.GetContainer().PrettyName(),\n\t\t\treqType.String(), cs.GetNode().Name(), prettyMem(remaining))\n\n\t\tfor memType, amount := range allocated {\n\t\t\tif amount > 0 {\n\t\t\t\tcs.grantedMem[memType] -= amount\n\t\t\t\tcs.mem[memType] += amount\n\t\t\t}\n\t\t}\n\n\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\"not enough memory at %s\", cs.node.Name())\n\t}\n\n\tcs.grantedMem[memoryAll] += requested\n\tcs.mem[memoryAll] -= requested\n\n\treturn allocated, nil\n}\n\n// Allocate allocates a grant from the supply.\nfunc (cs *supply) Allocate(r Request) (Grant, error) {\n\tgrant, err := cs.AllocateCPU(r)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tmemory, err := cs.allocateMemory(r)\n\tif err != nil {\n\t\tcs.ReleaseCPU(grant)\n\t\treturn nil, err\n\t}\n\n\tgrant.SetMemoryAllocation(r.MemoryType(), memory, r.ColdStart())\n\n\treturn grant, nil\n}\n\n// AllocateCPU allocates CPU for a grant from the supply.\nfunc (cs *supply) AllocateCPU(r Request) (Grant, error) {\n\tvar exclusive cpuset.CPUSet\n\tvar err error\n\n\tcr := r.(*request)\n\n\tfull := cr.full\n\tfraction := cr.fraction\n\n\tcpuType := cr.cpuType\n\n\tif cpuType == cpuReserved && full > 0 {\n\t\tlog.Warn(\"exclusive reserved CPUs not supported, allocating %d full CPUs as fractions\", full)\n\t\tfraction += full * 1000\n\t\tfull = 0\n\t}\n\n\tif cpuType == cpuReserved && fraction > 0 && cs.AllocatableReservedCPU() < fraction {\n\t\tlog.Warn(\"possible misconfiguration of reserved resources:\")\n\t\tlog.Warn(\"  %s: allocatable %s\", cs.GetNode().Name(), cs.DumpAllocatable())\n\t\tlog.Warn(\"  %s: needs %d reserved, only %d available\",\n\t\t\tcr.GetContainer().PrettyName(), fraction, cs.AllocatableReservedCPU())\n\t\tlog.Warn(\"  falling back to using normal unreserved CPUs instead...\")\n\t\tcpuType = cpuNormal\n\t}\n\n\t// allocate isolated exclusive CPUs or slice them off the sharable set\n\tswitch {\n\tcase full > 0 && cs.isolated.Size() >= full && cr.isolate:\n\t\texclusive, err = cs.takeCPUs(&cs.isolated, nil, full)\n\t\tif err != nil {\n\t\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\t\"%s: can't take %d exclusive isolated CPUs from %s: %v\",\n\t\t\t\tcs.node.Name(), full, cs.isolated, err)\n\t\t}\n\n\tcase full > 0 && cs.AllocatableSharedCPU() > 1000*full:\n\t\texclusive, err = cs.takeCPUs(&cs.sharable, nil, full)\n\t\tif err != nil {\n\t\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\t\"%s: can't take %d exclusive CPUs from %s: %v\",\n\t\t\t\tcs.node.Name(), full, cs.sharable, err)\n\t\t}\n\n\tcase full > 0:\n\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\"%s: can't slice %d exclusive CPUs from %s, %dm available\",\n\t\t\tcs.node.Name(), full, cs.sharable, cs.AllocatableSharedCPU())\n\t}\n\n\tgrant := newGrant(cs.node, cr.GetContainer(), cpuType, exclusive, 0, 0, nil, 0)\n\tgrant.AccountAllocateCPU()\n\n\tif fraction > 0 {\n\t\tif cpuType == cpuNormal {\n\t\t\t// allocate requested portion of shared CPUs\n\t\t\tif cs.AllocatableSharedCPU() < fraction {\n\t\t\t\tcs.ReleaseCPU(grant)\n\t\t\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\t\t\"%s: not enough %dm sharable CPU for %dm, %dm available\",\n\t\t\t\t\tcs.node.Name(), fraction, cs.sharable, cs.AllocatableSharedCPU())\n\t\t\t}\n\t\t\tcs.grantedShared += fraction\n\t\t} else if cpuType == cpuReserved {\n\t\t\t// allocate requested portion of reserved CPUs\n\t\t\tif cs.AllocatableReservedCPU() < fraction {\n\t\t\t\tcs.ReleaseCPU(grant)\n\t\t\t\treturn nil, policyError(\"internal error: \"+\n\t\t\t\t\t\"%s: not enough reserved CPU: %dm requested, %dm available\",\n\t\t\t\t\tcs.node.Name(), fraction, cs.AllocatableReservedCPU())\n\t\t\t}\n\t\t\tcs.grantedReserved += fraction\n\t\t}\n\t\tgrant.SetCPUPortion(fraction)\n\t}\n\n\treturn grant, nil\n}\n\nfunc (cs *supply) ReallocateMemory(g Grant) error {\n\tlog.Debug(\"%s: reallocating memory (%s) from %s to %s\",\n\t\tg.GetContainer().PrettyName(),\n\t\tg.MemLimit().String(),\n\t\tg.GetMemoryNode().Name(),\n\t\tcs.GetNode().Name())\n\n\t// The grant has been previously allocated from another supply. Reallocate it here.\n\tg.GetMemoryNode().FreeSupply().ReleaseMemory(g)\n\n\tmem := uint64(0)\n\tallocatedMemory := g.MemLimit()\n\tfor key, value := range allocatedMemory {\n\t\tif cs.mem[key] < value {\n\t\t\treturn policyError(\"internal error: not enough memory for reallocation at %s (released from %s)\", cs.GetNode().Name(), g.GetMemoryNode().Name())\n\t\t}\n\t\tcs.mem[key] -= value\n\t\tcs.grantedMem[key] += value\n\t\tmem += value\n\t}\n\tcs.grantedMem[memoryAll] += mem\n\tcs.mem[memoryAll] -= mem\n\treturn nil\n}\n\nfunc (cs *supply) ReleaseCPU(g Grant) {\n\tisolated := g.ExclusiveCPUs().Intersection(cs.node.GetSupply().IsolatedCPUs())\n\tsharable := g.ExclusiveCPUs().Difference(isolated)\n\n\tcs.isolated = cs.isolated.Union(isolated)\n\tcs.sharable = cs.sharable.Union(sharable)\n\tcs.grantedReserved -= g.ReservedPortion()\n\tcs.grantedShared -= g.SharedPortion()\n\n\tg.AccountReleaseCPU()\n}\n\n// ReleaseMemory returns memory from the given grant to the supply.\nfunc (cs *supply) ReleaseMemory(g Grant) {\n\treleasedMemory := uint64(0)\n\n\tlog.Debug(\"%s: releasing granted memory (%s) from %s\",\n\t\tg.GetContainer().PrettyName(),\n\t\tg.MemLimit().String(), cs.GetNode().Name())\n\n\tfor key, value := range g.MemLimit() {\n\t\tcs.grantedMem[key] -= value\n\t\tcs.mem[key] += value\n\t\treleasedMemory += value\n\t}\n\tcs.grantedMem[memoryAll] -= releasedMemory\n\tcs.mem[memoryAll] += releasedMemory\n\n\tcs.node.DepthFirst(func(n Node) error {\n\t\tn.FreeSupply().ReleaseExtraMemoryReservation(g)\n\t\treturn nil\n\t})\n}\n\nfunc (cs *supply) ExtraMemoryReservation(memType memoryType) uint64 {\n\textra := uint64(0)\n\tfor _, res := range cs.extraMemReservations {\n\t\textra += res[memType]\n\t}\n\treturn extra\n}\n\nfunc (cs *supply) ReleaseExtraMemoryReservation(g Grant) {\n\tif mems, ok := cs.extraMemReservations[g]; ok {\n\t\tlog.Debug(\"%s: releasing extra memory reservation (%s) from %s\",\n\t\t\tg.GetContainer().PrettyName(), mems.String(),\n\t\t\tcs.GetNode().Name())\n\t\tdelete(cs.extraMemReservations, g)\n\t}\n}\n\nfunc (cs *supply) SetExtraMemoryReservation(g Grant) {\n\tres := make(memoryMap)\n\textraMemory := uint64(0)\n\tfor key, value := range g.MemLimit() {\n\t\tres[key] = value\n\t\textraMemory += value\n\t}\n\tres[memoryAll] = extraMemory\n\tcs.extraMemReservations[g] = res\n}\n\nfunc (cs *supply) Reserve(g Grant) error {\n\tif g.CPUType() == cpuNormal {\n\t\tisolated := g.IsolatedCPUs()\n\t\texclusive := g.ExclusiveCPUs().Difference(isolated)\n\t\tsharedPortion := g.SharedPortion()\n\t\tif !cs.isolated.Intersection(isolated).Equals(isolated) {\n\t\t\treturn policyError(\"can't reserve isolated CPUs (%s) of %s from %s\",\n\t\t\t\tisolated.String(), g.String(), cs.DumpAllocatable())\n\t\t}\n\t\tif !cs.sharable.Intersection(exclusive).Equals(exclusive) {\n\t\t\treturn policyError(\"can't reserve exclusive CPUs (%s) of %s from %s\",\n\t\t\t\texclusive.String(), g.String(), cs.DumpAllocatable())\n\t\t}\n\t\tif cs.AllocatableSharedCPU() < 1000*exclusive.Size()+sharedPortion {\n\t\t\treturn policyError(\"can't reserve %d shared CPUs of %s from %s\",\n\t\t\t\tsharedPortion, g.String(), cs.DumpAllocatable())\n\t\t}\n\t\tcs.isolated = cs.isolated.Difference(isolated)\n\t\tcs.sharable = cs.sharable.Difference(exclusive)\n\t\tcs.grantedShared += sharedPortion\n\t} else if g.CPUType() == cpuReserved {\n\t\tsharedPortion := 1000*g.ExclusiveCPUs().Size() + g.SharedPortion()\n\t\tif sharedPortion > 0 && cs.AllocatableReservedCPU() < sharedPortion {\n\t\t\treturn policyError(\"can't reserve %d reserved CPUs of %s from %s\",\n\t\t\t\tsharedPortion, g.String(), cs.DumpAllocatable())\n\t\t}\n\t\tcs.grantedReserved += sharedPortion\n\t}\n\n\tg.AccountAllocateCPU()\n\n\treturn nil\n}\n\nfunc (cs *supply) ReserveMemory(g Grant) error {\n\tmem := uint64(0)\n\tallocatedMemory := g.MemLimit()\n\tfor key, value := range allocatedMemory {\n\t\tif cs.mem[key] < value {\n\t\t\treturn policyError(\"internal error: not enough memory for allocation at %s\", g.GetMemoryNode().Name())\n\t\t}\n\t\tcs.mem[key] -= value\n\t\tcs.grantedMem[key] += value\n\t\tmem += value\n\t}\n\tcs.grantedMem[memoryAll] += mem\n\tcs.mem[memoryAll] -= mem\n\tg.UpdateExtraMemoryReservation()\n\treturn nil\n}\n\n// takeCPUs takes up to cnt CPUs from a given CPU set to another.\nfunc (cs *supply) takeCPUs(from, to *cpuset.CPUSet, cnt int) (cpuset.CPUSet, error) {\n\tcset, err := cs.node.Policy().cpuAllocator.AllocateCpus(from, cnt, cpuallocator.PriorityHigh)\n\tif err != nil {\n\t\treturn cset, err\n\t}\n\n\tif to != nil {\n\t\t*to = to.Union(cset)\n\t}\n\n\treturn cset, err\n}\n\n// DumpCapacity returns a printable representation of the supply's resource capacity.\nfunc (cs *supply) DumpCapacity() string {\n\tcpu, mem, sep := \"\", cs.mem.String(), \"\"\n\n\tif !cs.isolated.IsEmpty() {\n\t\tcpu = fmt.Sprintf(\"isolated:%s\", cpuset.ShortCPUSet(cs.isolated))\n\t\tsep = \", \"\n\t}\n\tif !cs.reserved.IsEmpty() {\n\t\tcpu += sep + fmt.Sprintf(\"reserved:%s (%dm)\", cpuset.ShortCPUSet(cs.reserved),\n\t\t\t1000*cs.reserved.Size())\n\t\tsep = \", \"\n\t}\n\tif !cs.sharable.IsEmpty() {\n\t\tcpu += sep + fmt.Sprintf(\"sharable:%s (%dm)\", cpuset.ShortCPUSet(cs.sharable),\n\t\t\t1000*cs.sharable.Size())\n\t}\n\n\tcapacity := \"<\" + cs.node.Name() + \" capacity: \"\n\n\tif cpu == \"\" && mem == \"\" {\n\t\tcapacity += \"-\"\n\t} else {\n\t\tsep = \"\"\n\t\tif cpu != \"\" {\n\t\t\tcapacity += \"CPU: \" + cpu\n\t\t\tsep = \", \"\n\t\t}\n\t\tif mem != \"\" {\n\t\t\tcapacity += sep + \"MemLimit: \" + mem\n\t\t}\n\t}\n\tcapacity += \">\"\n\n\treturn capacity\n}\n\n// DumpAllocatable returns a printable representation of the supply's resource capacity.\nfunc (cs *supply) DumpAllocatable() string {\n\tcpu, mem, sep := \"\", cs.mem.String(), \"\"\n\n\tif !cs.isolated.IsEmpty() {\n\t\tcpu = fmt.Sprintf(\"isolated:%s\", cpuset.ShortCPUSet(cs.isolated))\n\t\tsep = \", \"\n\t}\n\tif !cs.reserved.IsEmpty() {\n\t\tcpu += sep + fmt.Sprintf(\"reserved:%s (allocatable: %dm)\", cpuset.ShortCPUSet(cs.reserved), cs.AllocatableReservedCPU())\n\t\tsep = \", \"\n\t\tif cs.grantedReserved > 0 {\n\t\t\tcpu += sep + fmt.Sprintf(\"grantedReserved:%dm\", cs.grantedReserved)\n\t\t}\n\t}\n\tlocal_grantedShared := cs.grantedShared\n\ttotal_grantedShared := cs.node.GrantedSharedCPU()\n\tif !cs.sharable.IsEmpty() {\n\t\tcpu += sep + fmt.Sprintf(\"sharable:%s (\", cpuset.ShortCPUSet(cs.sharable))\n\t\tsep = \"\"\n\t\tif local_grantedShared > 0 || total_grantedShared > 0 {\n\t\t\tcpu += fmt.Sprintf(\"grantedShared:\")\n\t\t\tkind := \"\"\n\t\t\tif local_grantedShared > 0 {\n\t\t\t\tcpu += fmt.Sprintf(\"%dm\", local_grantedShared)\n\t\t\t\tkind = \"local\"\n\t\t\t\tsep = \"/\"\n\t\t\t}\n\t\t\tif total_grantedShared > 0 {\n\t\t\t\tcpu += sep + fmt.Sprintf(\"%dm\", total_grantedShared)\n\t\t\t\tkind += sep + \"subtree\"\n\t\t\t}\n\t\t\tcpu += \" \" + kind\n\t\t\tsep = \", \"\n\t\t}\n\t\tcpu += sep + fmt.Sprintf(\"allocatable:%dm)\", cs.AllocatableSharedCPU(true))\n\t}\n\n\tallocatable := \"<\" + cs.node.Name() + \" allocatable: \"\n\n\tif cpu == \"\" && mem == \"\" {\n\t\tallocatable += \"-\"\n\t} else {\n\t\tsep = \"\"\n\t\tif cpu != \"\" {\n\t\t\tallocatable += \"CPU: \" + cpu\n\t\t\tsep = \", \"\n\t\t}\n\t\tif mem != \"\" {\n\t\t\tallocatable += sep + \"MemLimit: \" + mem\n\t\t}\n\t}\n\tallocatable += \">\"\n\n\treturn allocatable\n}\n\n// prettyMem formats the given amount as k, M, G, or T units.\nfunc prettyMem(value uint64) string {\n\tunits := []string{\"k\", \"M\", \"G\", \"T\"}\n\tcoeffs := []uint64{1 << 10, 1 << 20, 1 << 30, 1 << 40}\n\n\tc, u := uint64(1), \"\"\n\tfor i := 0; i < len(units); i++ {\n\t\tif coeffs[i] > value {\n\t\t\tbreak\n\t\t}\n\t\tc, u = coeffs[i], units[i]\n\t}\n\tv := float64(value) / float64(c)\n\n\treturn strconv.FormatFloat(v, 'f', 2, 64) + u\n}\n\n// DumpMemoryState dumps the state of the available and allocated memory.\nfunc (cs *supply) DumpMemoryState(prefix string) {\n\tmemTypes := []memoryType{memoryDRAM, memoryPMEM, memoryHBM}\n\ttotalFree := uint64(0)\n\ttotalGranted := uint64(0)\n\tfor _, kind := range memTypes {\n\t\tfree := cs.mem[kind]\n\t\tgranted := cs.grantedMem[kind]\n\t\tif free != 0 || granted != 0 {\n\t\t\tlog.Debug(prefix+\"- %s: free: %s, granted %s\",\n\t\t\t\tkind, prettyMem(free), prettyMem(granted))\n\t\t}\n\t\ttotalFree += free\n\t\ttotalGranted += granted\n\t}\n\tlog.Debug(prefix+\"- total free: %s, total granted %s\",\n\t\tprettyMem(totalFree), prettyMem(totalGranted))\n\n\tprintHdr := true\n\tif len(cs.extraMemReservations) > 0 {\n\t\tfor g, memMap := range cs.extraMemReservations {\n\t\t\tsplit := \"\"\n\t\t\tsep := \"\"\n\t\t\ttotal := uint64(0)\n\t\t\tif mem := memMap[memoryDRAM]; mem > 0 {\n\t\t\t\tsplit = \"DRAM \" + prettyMem(mem)\n\t\t\t\tsep = \", \"\n\t\t\t\ttotal += mem\n\t\t\t}\n\t\t\tif mem := memMap[memoryPMEM]; mem > 0 {\n\t\t\t\tsplit += sep + \"PMEM \" + prettyMem(mem)\n\t\t\t\tsep = \", \"\n\t\t\t\ttotal += mem\n\t\t\t}\n\t\t\tif mem := memMap[memoryHBM]; mem > 0 {\n\t\t\t\tsplit += sep + \"HBMEM \" + prettyMem(mem)\n\t\t\t\tsep = \", \"\n\t\t\t\ttotal += mem\n\t\t\t}\n\t\t\tif total > 0 {\n\t\t\t\tif printHdr {\n\t\t\t\t\tlog.Debug(prefix + \"- extra reservations:\")\n\t\t\t\t\tprintHdr = false\n\t\t\t\t}\n\t\t\t\tlog.Debug(prefix+\"  - %s: %s (%s)\",\n\t\t\t\t\tg.GetContainer().PrettyName(), prettyMem(total), split)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// newRequest creates a new request for the given container.\nfunc newRequest(container cache.Container) Request {\n\tpod, _ := container.GetPod()\n\tfull, fraction, isolate, cpuType := cpuAllocationPreferences(pod, container)\n\treq, lim, mtype := memoryAllocationPreference(pod, container)\n\tcoldStart := time.Duration(0)\n\n\tlog.Debug(\"%s: CPU preferences: cpuType=%s, full=%v, fraction=%v, isolate=%v\",\n\t\tcontainer.PrettyName(), cpuType, full, fraction, isolate)\n\n\tif mtype == memoryUnspec {\n\t\tmtype = defaultMemoryType\n\t}\n\n\tif mtype&memoryPMEM != 0 && mtype&memoryDRAM != 0 {\n\t\tparsedColdStart, err := coldStartPreference(pod, container)\n\t\tif err != nil {\n\t\t\tlog.Error(\"Failed to parse cold start preference\")\n\t\t} else {\n\t\t\tif parsedColdStart.Duration > 0 {\n\t\t\t\tif coldStartOff {\n\t\t\t\t\tlog.Error(\"coldstart disabled (movable non-DRAM memory zones present)\")\n\t\t\t\t} else {\n\t\t\t\t\tcoldStart = time.Duration(parsedColdStart.Duration)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else if mtype == memoryPMEM {\n\t\tif coldStartOff {\n\t\t\tmtype = mtype | memoryDRAM\n\t\t\tlog.Error(\"%s: forced also DRAM usage (movable non-DRAM memory zones present)\",\n\t\t\t\tcontainer.PrettyName())\n\t\t}\n\t}\n\n\treturn &request{\n\t\tcontainer: container,\n\t\tfull:      full,\n\t\tfraction:  fraction,\n\t\tisolate:   isolate,\n\t\tcpuType:   cpuType,\n\t\tmemReq:    req,\n\t\tmemLim:    lim,\n\t\tmemType:   mtype,\n\t\tcoldStart: coldStart,\n\t}\n}\n\n// GetContainer returns the container requesting CPU.\nfunc (cr *request) GetContainer() cache.Container {\n\treturn cr.container\n}\n\n// String returns aprintable representation of the CPU request.\nfunc (cr *request) String() string {\n\tmem := \"<Memory request: limit:\" + prettyMem(cr.memLim) + \", req:\" + prettyMem(cr.memReq) + \">\"\n\tisolated := map[bool]string{false: \"\", true: \"isolated \"}[cr.isolate]\n\tswitch {\n\tcase cr.full == 0 && cr.fraction == 0:\n\t\treturn \"<CPU request \" + cr.container.PrettyName() + \": ->\" + mem\n\n\tcase cr.full > 0 && cr.fraction > 0:\n\t\treturn fmt.Sprintf(\"<CPU request \"+cr.container.PrettyName()+\": \"+\n\t\t\t\"%sexclusive: %d, shared: %d>\", isolated, cr.full, cr.fraction) + mem\n\n\tcase cr.full > 0:\n\t\treturn fmt.Sprintf(\"<CPU request \"+\n\t\t\tcr.container.PrettyName()+\": %sexclusive: %d>\", isolated, cr.full) + mem\n\n\tdefault:\n\t\treturn fmt.Sprintf(\"<CPU request \"+\n\t\t\tcr.container.PrettyName()+\": shared: %d>\", cr.fraction) + mem\n\t}\n}\n\n// CPUType returns the requested type of CPU for the grant.\nfunc (cr *request) CPUType() cpuClass {\n\treturn cr.cpuType\n}\n\n// SetCPUType sets the requested type of CPU for the grant.\nfunc (cr *request) SetCPUType(cpuType cpuClass) {\n\tcr.cpuType = cpuType\n}\n\n// FullCPUs return the number of full CPUs requested.\nfunc (cr *request) FullCPUs() int {\n\treturn cr.full\n}\n\n// CPUFraction returns the amount of fractional milli-CPU requested.\nfunc (cr *request) CPUFraction() int {\n\treturn cr.fraction\n}\n\n// Isolate returns whether isolated CPUs are preferred for this request.\nfunc (cr *request) Isolate() bool {\n\treturn cr.isolate\n}\n\n// MemAmountToAllocate retuns how much memory we need to reserve for a request.\nfunc (cr *request) MemAmountToAllocate() uint64 {\n\tvar amount uint64 = 0\n\tswitch cr.GetContainer().GetQOSClass() {\n\tcase v1.PodQOSBurstable:\n\t\t// May be a request and/or limit. We focus on the limit because we\n\t\t// need to prepare for the case when all containers are using all\n\t\t// the memory they are allowed to. If limit is not set then we'll\n\t\t// allocate the request (which the container will get).\n\t\tif cr.memLim > 0 {\n\t\t\tamount = cr.memLim\n\t\t} else {\n\t\t\tamount = cr.memReq\n\t\t}\n\tcase v1.PodQOSGuaranteed:\n\t\t// Limit and request are the same.\n\t\tamount = cr.memLim\n\tcase v1.PodQOSBestEffort:\n\t\t// No requests or limits.\n\t\tamount = 0\n\t}\n\treturn amount\n}\n\n// MemoryType returns the requested type of memory for the grant.\nfunc (cr *request) MemoryType() memoryType {\n\treturn cr.memType\n}\n\n// ColdStart returns the cold start timeout (in milliseconds).\nfunc (cr *request) ColdStart() time.Duration {\n\treturn cr.coldStart\n}\n\n// Score collects data for scoring this supply wrt. the given request.\nfunc (cs *supply) GetScore(req Request) Score {\n\tscore := &score{\n\t\tsupply: cs,\n\t\treq:    req,\n\t}\n\n\tcr := req.(*request)\n\tfull, part := cr.full, cr.fraction\n\tif full == 0 && part == 0 {\n\t\tpart = 1\n\t}\n\n\tscore.reserved = cs.AllocatableReservedCPU()\n\tscore.shared = cs.AllocatableSharedCPU()\n\n\tif cr.CPUType() == cpuReserved {\n\t\t// calculate free reserved capacity\n\t\tscore.reserved -= part\n\t} else {\n\t\t// calculate isolated node capacity CPU\n\t\tif cr.isolate {\n\t\t\tscore.isolated = cs.isolated.Size() - full\n\t\t}\n\n\t\t// if we don't want isolated or there is not enough, calculate slicable capacity\n\t\tif !cr.isolate || score.isolated < 0 {\n\t\t\tscore.shared -= 1000 * full\n\t\t}\n\n\t\t// calculate fractional capacity\n\t\tscore.shared -= part\n\t}\n\n\t// calculate colocation score\n\tfor _, grant := range cs.node.Policy().allocations.grants {\n\t\tif cr.CPUType() == grant.CPUType() && grant.GetCPUNode().NodeID() == cs.node.NodeID() {\n\t\t\tscore.colocated++\n\t\t}\n\t}\n\n\t// calculate real hint scores\n\thints := cr.container.GetTopologyHints()\n\tscore.hints = make(map[string]float64, len(hints))\n\n\tfor provider, hint := range cr.container.GetTopologyHints() {\n\t\tif provider == topology.ProviderKubelet {\n\t\t\tlog.Warn(\" - ignoring topology pseudo-hint from kubelet allocation %s\", hint)\n\t\t\tcontinue\n\t\t}\n\t\tlog.Debug(\" - evaluating topology hint %s\", hint)\n\t\tscore.hints[provider] = cs.node.HintScore(hint)\n\t}\n\n\treturn score\n}\n\n// AllocatableReservedCPU calculates the allocatable amount of reserved CPU of this supply.\nfunc (cs *supply) AllocatableReservedCPU() int {\n\tif cs.reserved.Size() == 0 {\n\t\t// This supply has no room for reserved (not even of zero-sized)\n\t\treturn -1\n\t}\n\treserved := 1000*cs.reserved.Size() - cs.node.GrantedReservedCPU()\n\tfor node := cs.node.Parent(); !node.IsNil(); node = node.Parent() {\n\t\tpSupply := node.FreeSupply()\n\t\tpReserved := 1000*pSupply.ReservedCPUs().Size() - pSupply.GetNode().GrantedReservedCPU()\n\t\tif pReserved < reserved {\n\t\t\treserved = pReserved\n\t\t}\n\t}\n\treturn reserved\n}\n\n// AllocatableSharedCPU calculates the allocatable amount of shared CPU of this supply.\nfunc (cs *supply) AllocatableSharedCPU(quiet ...bool) int {\n\tverbose := !(len(quiet) > 0 && quiet[0])\n\n\t// Notes:\n\t//   Take into account the supplies/grants in all ancestors, making sure\n\t//   none of them gets overcommitted as the result of fulfilling this request.\n\tshared := 1000*cs.sharable.Size() - cs.node.GrantedSharedCPU()\n\tif verbose {\n\t\tlog.Debug(\"%s: unadjusted free shared CPU: %dm\", cs.node.Name(), shared)\n\t}\n\tfor node := cs.node.Parent(); !node.IsNil(); node = node.Parent() {\n\t\tpSupply := node.FreeSupply()\n\t\tpShared := 1000*pSupply.SharableCPUs().Size() - pSupply.GetNode().GrantedSharedCPU()\n\t\tif pShared < shared {\n\t\t\tif verbose {\n\t\t\t\tlog.Debug(\"%s: capping free shared CPU (%dm -> %dm) to avoid overcommit of %s\",\n\t\t\t\t\tcs.node.Name(), shared, pShared, node.Name())\n\t\t\t}\n\t\t\tshared = pShared\n\t\t}\n\t}\n\tif verbose {\n\t\tlog.Debug(\"%s: ancestor-adjusted free shared CPU: %dm\", cs.node.Name(), shared)\n\t}\n\treturn shared\n}\n\n// Eval...\nfunc (score *score) Eval() float64 {\n\treturn 1.0\n}\n\nfunc (score *score) Supply() Supply {\n\treturn score.supply\n}\n\nfunc (score *score) Request() Request {\n\treturn score.req\n}\n\nfunc (score *score) IsolatedCapacity() int {\n\treturn score.isolated\n}\n\nfunc (score *score) ReservedCapacity() int {\n\treturn score.reserved\n}\n\nfunc (score *score) SharedCapacity() int {\n\treturn score.shared\n}\n\nfunc (score *score) Colocated() int {\n\treturn score.colocated\n}\n\nfunc (score *score) HintScores() map[string]float64 {\n\treturn score.hints\n}\n\nfunc (score *score) String() string {\n\treturn fmt.Sprintf(\"<CPU score: node %s, isolated:%d, reserved:%d, shared:%d, colocated:%d, hints: %v>\",\n\t\tscore.supply.GetNode().Name(), score.isolated, score.reserved, score.shared, score.colocated, score.hints)\n}\n\n// newGrant creates a CPU grant from the given node for the container.\nfunc newGrant(n Node, c cache.Container, cpuType cpuClass, exclusive cpuset.CPUSet, cpuPortion int, mt memoryType, allocated memoryMap, coldstart time.Duration) Grant {\n\tgrant := &grant{\n\t\tnode:       n,\n\t\tmemoryNode: n,\n\t\tcontainer:  c,\n\t\tcpuType:    cpuType,\n\t\texclusive:  exclusive,\n\t\tcpuPortion: cpuPortion,\n\t}\n\tif allocated != nil {\n\t\tgrant.SetMemoryAllocation(mt, allocated, coldstart)\n\t}\n\treturn grant\n}\n\n// SetCPUPortion sets the fractional CPU portion for the grant.\nfunc (cg *grant) SetCPUPortion(fraction int) {\n\tcg.cpuPortion = fraction\n}\n\n// SetMemoryAllocation sets the memory allocation for the grant.\nfunc (cg *grant) SetMemoryAllocation(mt memoryType, allocated memoryMap, coldstart time.Duration) {\n\tinitial := memoryPMEM\n\tif coldstart <= 0 {\n\t\tinitial = mt\n\t}\n\tmems := cg.node.GetMemset(initial)\n\tif mems.Size() == 0 {\n\t\tmems = cg.node.GetMemset(memoryDRAM)\n\t\tif mems.Size() == 0 {\n\t\t\tmems = cg.node.GetMemset(memoryAll)\n\t\t}\n\t}\n\tmems = mems.Clone()\n\n\tcg.memType = mt\n\tcg.memset = mems\n\tcg.allocatedMem = allocated\n\tcg.coldStart = coldstart\n}\n\n// Clone creates a copy of this grant.\nfunc (cg *grant) Clone() Grant {\n\treturn &grant{\n\t\tnode:         cg.GetCPUNode(),\n\t\tmemoryNode:   cg.GetMemoryNode(),\n\t\tcontainer:    cg.GetContainer(),\n\t\texclusive:    cg.ExclusiveCPUs(),\n\t\tcpuType:      cg.CPUType(),\n\t\tcpuPortion:   cg.SharedPortion(),\n\t\tmemType:      cg.MemoryType(),\n\t\tmemset:       cg.Memset().Clone(),\n\t\tallocatedMem: cg.MemLimit(),\n\t\tcoldStart:    cg.ColdStart(),\n\t}\n}\n\n// RefetchNodes updates the stored cpu and memory nodes of this grant by name.\nfunc (cg *grant) RefetchNodes() error {\n\tnode, ok := cg.node.Policy().nodes[cg.node.Name()]\n\tif !ok {\n\t\treturn policyError(\"failed to refetch grant cpu node %s\", cg.node.Name())\n\t}\n\tmemoryNode, ok := cg.memoryNode.Policy().nodes[cg.memoryNode.Name()]\n\tif !ok {\n\t\treturn policyError(\"failed to refetch grant memory node %s\", cg.memoryNode.Name())\n\t}\n\tcg.node = node\n\tcg.memoryNode = memoryNode\n\treturn nil\n}\n\n// GetContainer returns the container this grant is valid for.\nfunc (cg *grant) GetContainer() cache.Container {\n\treturn cg.container\n}\n\n// GetNode returns the Node this grant gets its CPU allocation from.\nfunc (cg *grant) GetCPUNode() Node {\n\treturn cg.node\n}\n\n// GetNode returns the Node this grant gets its memory allocation from.\nfunc (cg *grant) GetMemoryNode() Node {\n\treturn cg.memoryNode\n}\n\nfunc (cg *grant) SetMemoryNode(n Node) {\n\tcg.memoryNode = n\n\tcg.memset = n.GetMemset(cg.MemoryType())\n}\n\n// CPUType returns the requested type of CPU for the grant.\nfunc (cg *grant) CPUType() cpuClass {\n\treturn cg.cpuType\n}\n\n// CPUPortion returns granted milli-CPUs of non-full CPUs of CPUType().\nfunc (cg *grant) CPUPortion() int {\n\treturn cg.cpuPortion\n}\n\n// ExclusiveCPUs returns the non-isolated exclusive CPUSet in this grant.\nfunc (cg *grant) ExclusiveCPUs() cpuset.CPUSet {\n\treturn cg.exclusive\n}\n\n// ReservedCPUs returns the reserved CPUSet in the supply of this grant.\nfunc (cg *grant) ReservedCPUs() cpuset.CPUSet {\n\treturn cg.node.GetSupply().ReservedCPUs()\n}\n\n// ReservedPortion returns the milli-CPU allocation for the reserved CPUSet in this grant.\nfunc (cg *grant) ReservedPortion() int {\n\tif cg.cpuType == cpuReserved {\n\t\treturn cg.cpuPortion\n\t}\n\treturn 0\n}\n\n// SharedCPUs returns the shared CPUSet in the supply of this grant.\nfunc (cg *grant) SharedCPUs() cpuset.CPUSet {\n\treturn cg.node.FreeSupply().SharableCPUs()\n}\n\n// SharedPortion returns the milli-CPU allocation for the shared CPUSet in this grant.\nfunc (cg *grant) SharedPortion() int {\n\tif cg.cpuType == cpuNormal {\n\t\treturn cg.cpuPortion\n\t}\n\treturn 0\n}\n\n// ExclusiveCPUs returns the isolated exclusive CPUSet in this grant.\nfunc (cg *grant) IsolatedCPUs() cpuset.CPUSet {\n\treturn cg.node.GetSupply().IsolatedCPUs().Intersection(cg.exclusive)\n}\n\n// MemoryType returns the requested type of memory for the grant.\nfunc (cg *grant) MemoryType() memoryType {\n\treturn cg.memType\n}\n\n// Memset returns the granted memory controllers as an IDSet.\nfunc (cg *grant) Memset() idset.IDSet {\n\treturn cg.memset\n}\n\n// MemLimit returns the granted memory.\nfunc (cg *grant) MemLimit() memoryMap {\n\treturn cg.allocatedMem\n}\n\n// String returns a printable representation of the CPU grant.\nfunc (cg *grant) String() string {\n\tvar cpuType, isolated, exclusive, reserved, shared string\n\tcpuType = fmt.Sprintf(\"cputype: %s\", cg.cpuType)\n\tisol := cg.IsolatedCPUs()\n\tif !isol.IsEmpty() {\n\t\tisolated = fmt.Sprintf(\", isolated: %s\", isol)\n\t}\n\tif !cg.exclusive.IsEmpty() {\n\t\texclusive = fmt.Sprintf(\", exclusive: %s\", cg.exclusive)\n\t}\n\tif cg.ReservedPortion() > 0 {\n\t\treserved = fmt.Sprintf(\", reserved: %s (%dm)\",\n\t\t\tcg.node.FreeSupply().ReservedCPUs(), cg.ReservedPortion())\n\t}\n\tif cg.SharedPortion() > 0 {\n\t\tshared = fmt.Sprintf(\", shared: %s (%dm)\",\n\t\t\tcg.node.FreeSupply().SharableCPUs(), cg.SharedPortion())\n\t}\n\n\tmem := cg.allocatedMem.String()\n\tif mem != \"\" {\n\t\tmem = \", MemLimit: \" + mem\n\t}\n\n\treturn fmt.Sprintf(\"<grant for %s from %s: %s%s%s%s%s%s>\",\n\t\tcg.container.PrettyName(), cg.node.Name(), cpuType, isolated, exclusive, reserved, shared, mem)\n}\n\nfunc (cg *grant) AccountAllocateCPU() {\n\tcg.node.DepthFirst(func(n Node) error {\n\t\tn.FreeSupply().AccountAllocateCPU(cg)\n\t\treturn nil\n\t})\n\tfor node := cg.node.Parent(); !node.IsNil(); node = node.Parent() {\n\t\tnode.FreeSupply().AccountAllocateCPU(cg)\n\t}\n}\n\nfunc (cg *grant) Release() {\n\tcg.GetCPUNode().FreeSupply().ReleaseCPU(cg)\n\tcg.GetMemoryNode().FreeSupply().ReleaseMemory(cg)\n\tcg.StopTimer()\n}\n\nfunc (cg *grant) AccountReleaseCPU() {\n\tcg.node.DepthFirst(func(n Node) error {\n\t\tn.FreeSupply().AccountReleaseCPU(cg)\n\t\treturn nil\n\t})\n\tfor node := cg.node.Parent(); !node.IsNil(); node = node.Parent() {\n\t\tnode.FreeSupply().AccountReleaseCPU(cg)\n\t}\n}\n\nfunc (cg *grant) RestoreMemset() {\n\tmems := cg.GetMemoryNode().GetMemset(cg.memType)\n\tcg.memset = mems\n\tcg.GetMemoryNode().Policy().applyGrant(cg)\n}\n\nfunc (cg *grant) ExpandMemset() (bool, error) {\n\tsupply := cg.GetMemoryNode().FreeSupply()\n\tnode := cg.GetMemoryNode()\n\tparent := node.Parent()\n\n\t// We have to assume that the memory has been allocated how we granted it (if PMEM ran out\n\t// the allocations have been made from DRAM and so on).\n\n\t// Figure out if there is enough memory now to have grant as-is.\n\textra := supply.ExtraMemoryReservation(memoryAll)\n\tfree := supply.MemoryLimit()[memoryAll]\n\tif extra <= free {\n\t\t// The grant fits in the node even with extra reservations\n\t\treturn false, nil\n\t}\n\t// Else it doesn't fit, so move the grant up in the memory tree.\n\trequired := uint64(0)\n\tfor _, memType := range []memoryType{memoryPMEM, memoryDRAM, memoryHBM} {\n\t\trequired += cg.MemLimit()[memType]\n\t}\n\tlog.Debug(\"out-of-memory risk in %s: extra reservations %s > free %s -> moving up %s total memory grant from %s\",\n\t\tcg, prettyMem(extra), prettyMem(free), prettyMem(required), node.Name())\n\n\t// Find an ancestor where the grant fits. As reservations in\n\t// child nodes do not show up in free + extra in parent nodes,\n\t// releasing the grant is not necessary before searching.\n\tfor ; !parent.IsNil(); parent = parent.Parent() {\n\t\tpSupply := parent.FreeSupply()\n\t\tparentFree := pSupply.MemoryLimit()[memoryAll]\n\t\tparentExtra := pSupply.ExtraMemoryReservation(memoryAll)\n\t\tif parentExtra+required <= parentFree {\n\t\t\trequired = 0\n\t\t\tbreak\n\t\t}\n\t\tlog.Debug(\"- %s has %s free but %s extra reservations, moving further up\",\n\t\t\tparent.Name(), prettyMem(parentFree), prettyMem(parentExtra))\n\t}\n\tif required > 0 {\n\t\treturn false, fmt.Errorf(\"internal error: cannot find enough memory (%s) for %s from ancestors of %s\", prettyMem(required), cg, node.Name())\n\t}\n\n\t// Release granted memory from the node and allocate it from the parent node.\n\terr := parent.FreeSupply().ReallocateMemory(cg)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tcg.SetMemoryNode(parent)\n\tcg.UpdateExtraMemoryReservation()\n\n\t// Make the container to use the new memory set.\n\t// FIXME: this could be done in a second pass to avoid doing this many times\n\tcg.GetMemoryNode().Policy().applyGrant(cg)\n\n\treturn true, nil\n}\n\nfunc (cg *grant) UpdateExtraMemoryReservation() {\n\t// For every subnode, make sure that this grant is added to the extra memory allocation.\n\tcg.GetMemoryNode().DepthFirst(func(n Node) error {\n\t\t// No extra allocation should be done to the node itself.\n\t\tif !n.IsSameNode(cg.GetMemoryNode()) {\n\t\t\tsupply := n.FreeSupply()\n\t\t\tsupply.SetExtraMemoryReservation(cg)\n\t\t}\n\t\treturn nil\n\t})\n}\n\nfunc (cg *grant) ColdStart() time.Duration {\n\treturn cg.coldStart\n}\n\nfunc (cg *grant) AddTimer(timer *time.Timer) {\n\tcg.coldStartTimer = timer\n}\n\nfunc (cg *grant) StopTimer() {\n\tif cg.coldStartTimer != nil {\n\t\tcg.coldStartTimer.Stop()\n\t\tcg.coldStartTimer = nil\n\t}\n}\n\nfunc (cg *grant) ClearTimer() {\n\tif cg.coldStartTimer != nil {\n\t\tcg.coldStartTimer = nil\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/builtin/topology-aware/topology-aware-policy.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topologyaware\n\nimport (\n\t\"errors\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tresapi \"k8s.io/apimachinery/pkg/api/resource\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cpuallocator\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\n\tpolicyapi \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nconst (\n\t// PolicyName is the name used to activate this policy implementation.\n\tPolicyName = \"topology-aware\"\n\t// PolicyDescription is a short description of this policy.\n\tPolicyDescription = \"A policy for prototyping memory tiering.\"\n\t// PolicyPath is the path of this policy in the configuration hierarchy.\n\tPolicyPath = \"policy.\" + PolicyName\n\t// AliasName is the 'memtier' alias name for this policy.\n\tAliasName = \"memtier\"\n\t// AliasPath is the 'memtier' alias configuration path for this policy.\n\tAliasPath = \"policy.\" + AliasName\n\n\t// ColdStartDone is the event generated for the end of a container cold start period.\n\tColdStartDone = \"cold-start-done\"\n)\n\n// allocations is our cache.Cachable for saving resource allocations in the cache.\ntype allocations struct {\n\tpolicy *policy\n\tgrants map[string]Grant\n}\n\n// policy is our runtime state for this policy.\ntype policy struct {\n\toptions      *policyapi.BackendOptions // options we were created or reconfigured with\n\tcache        cache.Cache               // pod/container cache\n\tsys          system.System             // system/HW topology info\n\tallowed      cpuset.CPUSet             // bounding set of CPUs we're allowed to use\n\treserved     cpuset.CPUSet             // system-/kube-reserved CPUs\n\treserveCnt   int                       // number of CPUs to reserve if given as resource.Quantity\n\tisolated     cpuset.CPUSet             // (our allowed set of) isolated CPUs\n\tnodes        map[string]Node           // pool nodes by name\n\tpools        []Node                    // pre-populated node slice for scoring, etc...\n\troot         Node                      // root of our pool/partition tree\n\tnodeCnt      int                       // number of pools\n\tdepth        int                       // tree depth\n\tallocations  allocations               // container pool assignments\n\tcpuAllocator cpuallocator.CPUAllocator // CPU allocator used by the policy\n\tcoldstartOff bool                      // coldstart forced off (have movable PMEM zones)\n\tisAlias      bool                      // whether started by referencing AliasName\n}\n\n// Make sure policy implements the policy.Backend interface.\nvar _ policyapi.Backend = &policy{}\n\n// Whether we have coldstart forced off due to PMEM in movable memory zones.\nvar coldStartOff bool\n\n// CreateTopologyAwarePolicy creates a new policy instance.\nfunc CreateTopologyAwarePolicy(opts *policyapi.BackendOptions) policyapi.Backend {\n\treturn createPolicy(opts, false)\n}\n\n// CreateMemtierPolicy creates a new policy instance, aliased as 'memtier'.\nfunc CreateMemtierPolicy(opts *policyapi.BackendOptions) policyapi.Backend {\n\treturn createPolicy(opts, true)\n}\n\n// createPolicy creates a new policy instance.\nfunc createPolicy(opts *policyapi.BackendOptions, isAlias bool) policyapi.Backend {\n\tp := &policy{\n\t\tcache:        opts.Cache,\n\t\tsys:          opts.System,\n\t\toptions:      opts,\n\t\tcpuAllocator: cpuallocator.NewCPUAllocator(opts.System),\n\t\tisAlias:      isAlias,\n\t}\n\n\tif isAlias {\n\t\t*opt = *aliasOpt\n\t}\n\n\tif err := p.initialize(); err != nil {\n\t\tlog.Fatal(\"failed to initialize %s policy: %v\", PolicyName, err)\n\t}\n\n\tp.registerImplicitAffinities()\n\n\tconfig.GetModule(policyapi.ConfigPath).AddNotify(p.configNotify)\n\n\treturn p\n}\n\n// Name returns the name of this policy.\nfunc (p *policy) Name() string {\n\treturn PolicyName\n}\n\n// Description returns the description for this policy.\nfunc (p *policy) Description() string {\n\treturn PolicyDescription\n}\n\n// Start prepares this policy for accepting allocation/release requests.\nfunc (p *policy) Start(add []cache.Container, del []cache.Container) error {\n\tif err := p.restoreCache(); err != nil {\n\t\treturn policyError(\"failed to start: %v\", err)\n\t}\n\n\t// Turn coldstart forcibly off if we have movable non-DRAM memory.\n\t// Note that although this can change dynamically we only check it\n\t// during startup and trust users to either not fiddle with memory\n\t// or restart us if they do.\n\tp.checkColdstartOff()\n\n\tp.root.Dump(\"<post-start>\")\n\n\treturn p.Sync(add, del)\n}\n\n// Sync synchronizes the state of this policy.\nfunc (p *policy) Sync(add []cache.Container, del []cache.Container) error {\n\tlog.Debug(\"synchronizing state...\")\n\tfor _, c := range del {\n\t\tp.ReleaseResources(c)\n\t}\n\tfor _, c := range add {\n\t\tp.AllocateResources(c)\n\t}\n\n\treturn nil\n}\n\n// AllocateResources is a resource allocation request for this policy.\nfunc (p *policy) AllocateResources(container cache.Container) error {\n\tlog.Debug(\"allocating resources for %s...\", container.PrettyName())\n\n\tgrant, err := p.allocatePool(container, \"\")\n\tif err != nil {\n\t\treturn policyError(\"failed to allocate resources for %s: %v\",\n\t\t\tcontainer.PrettyName(), err)\n\t}\n\tp.applyGrant(grant)\n\tp.updateSharedAllocations(&grant)\n\n\tp.root.Dump(\"<post-alloc>\")\n\n\treturn nil\n}\n\n// ReleaseResources is a resource release request for this policy.\nfunc (p *policy) ReleaseResources(container cache.Container) error {\n\tlog.Debug(\"releasing resources of %s...\", container.PrettyName())\n\n\tif grant, found := p.releasePool(container); found {\n\t\tp.updateSharedAllocations(&grant)\n\t}\n\n\tp.root.Dump(\"<post-release>\")\n\n\treturn nil\n}\n\n// UpdateResources is a resource allocation update request for this policy.\nfunc (p *policy) UpdateResources(c cache.Container) error {\n\tlog.Debug(\"(not) updating container %s...\", c.PrettyName())\n\treturn nil\n}\n\n// Rebalance tries to find an optimal allocation of resources for the current containers.\nfunc (p *policy) Rebalance() (bool, error) {\n\tvar errors error\n\n\tcontainers := p.cache.GetContainers()\n\tmovable := []cache.Container{}\n\n\tfor _, c := range containers {\n\t\tif c.GetQOSClass() != v1.PodQOSGuaranteed {\n\t\t\tp.ReleaseResources(c)\n\t\t\tmovable = append(movable, c)\n\t\t}\n\t}\n\n\tfor _, c := range movable {\n\t\tif err := p.AllocateResources(c); err != nil {\n\t\t\tif errors == nil {\n\t\t\t\terrors = err\n\t\t\t} else {\n\t\t\t\terrors = policyError(\"%v, %v\", errors, err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn true, errors\n}\n\n// HandleEvent handles policy-specific events.\nfunc (p *policy) HandleEvent(e *events.Policy) (bool, error) {\n\tlog.Debug(\"received policy event %s.%s with data %v...\", e.Source, e.Type, e.Data)\n\n\tswitch e.Type {\n\tcase events.ContainerStarted:\n\t\tc, ok := e.Data.(cache.Container)\n\t\tif !ok {\n\t\t\treturn false, policyError(\"%s event: expecting cache.Container Data, got %T\",\n\t\t\t\te.Type, e.Data)\n\t\t}\n\t\tlog.Info(\"triggering coldstart period (if necessary) for %s\", c.PrettyName())\n\t\treturn false, p.triggerColdStart(c)\n\tcase ColdStartDone:\n\t\tid, ok := e.Data.(string)\n\t\tif !ok {\n\t\t\treturn false, policyError(\"%s event: expecting container ID Data, got %T\",\n\t\t\t\te.Type, e.Data)\n\t\t}\n\t\tc, ok := p.cache.LookupContainer(id)\n\t\tif !ok {\n\t\t\t// TODO: This is probably a race condition. Should we return nil error here?\n\t\t\treturn false, policyError(\"%s event: failed to lookup container %s\", id)\n\t\t}\n\t\tlog.Info(\"finishing coldstart period for %s\", c.PrettyName())\n\t\treturn p.finishColdStart(c)\n\t}\n\treturn false, nil\n}\n\n// Introspect provides data for external introspection.\nfunc (p *policy) Introspect(state *introspect.State) {\n\tpools := make(map[string]*introspect.Pool, len(p.pools))\n\tfor _, node := range p.nodes {\n\t\tcpus := node.GetSupply()\n\t\tpool := &introspect.Pool{\n\t\t\tName:   node.Name(),\n\t\t\tCPUs:   cpus.SharableCPUs().Union(cpus.IsolatedCPUs()).String(),\n\t\t\tMemory: node.GetMemset(memoryAll).String(),\n\t\t}\n\t\tif parent := node.Parent(); !parent.IsNil() {\n\t\t\tpool.Parent = parent.Name()\n\t\t}\n\t\tif children := node.Children(); len(children) > 0 {\n\t\t\tpool.Children = make([]string, 0, len(children))\n\t\t\tfor _, c := range children {\n\t\t\t\tpool.Children = append(pool.Children, c.Name())\n\t\t\t}\n\t\t}\n\t\tpools[pool.Name] = pool\n\t}\n\tstate.Pools = pools\n\n\tassignments := make(map[string]*introspect.Assignment, len(p.allocations.grants))\n\tfor _, g := range p.allocations.grants {\n\t\ta := &introspect.Assignment{\n\t\t\tContainerID:   g.GetContainer().GetID(),\n\t\t\tCPUShare:      g.SharedPortion(),\n\t\t\tExclusiveCPUs: g.ExclusiveCPUs().Union(g.IsolatedCPUs()).String(),\n\t\t\tPool:          g.GetCPUNode().Name(),\n\t\t}\n\t\tif g.SharedPortion() > 0 || a.ExclusiveCPUs == \"\" {\n\t\t\ta.SharedCPUs = g.SharedCPUs().String()\n\t\t}\n\t\tassignments[a.ContainerID] = a\n\t}\n\tstate.Assignments = assignments\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *policy) DescribeMetrics() []*prometheus.Desc {\n\treturn nil\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *policy) PollMetrics() policyapi.Metrics {\n\treturn nil\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *policy) CollectMetrics(policyapi.Metrics) ([]prometheus.Metric, error) {\n\treturn nil, nil\n}\n\n// ExportResourceData provides resource data to export for the container.\nfunc (p *policy) ExportResourceData(c cache.Container) map[string]string {\n\tgrant, ok := p.allocations.grants[c.GetCacheID()]\n\tif !ok {\n\t\treturn nil\n\t}\n\n\tdata := map[string]string{}\n\tshared := grant.SharedCPUs().String()\n\tisolated := grant.ExclusiveCPUs().Intersection(grant.GetCPUNode().GetSupply().IsolatedCPUs())\n\texclusive := grant.ExclusiveCPUs().Difference(isolated).String()\n\n\tif grant.SharedPortion() > 0 && shared != \"\" {\n\t\tdata[policyapi.ExportSharedCPUs] = shared\n\t}\n\tif isolated.String() != \"\" {\n\t\tdata[policyapi.ExportIsolatedCPUs] = isolated.String()\n\t}\n\tif exclusive != \"\" {\n\t\tdata[policyapi.ExportExclusiveCPUs] = exclusive\n\t}\n\n\tmems := grant.Memset()\n\tdram := idset.NewIDSet()\n\tpmem := idset.NewIDSet()\n\thbm := idset.NewIDSet()\n\tfor _, id := range mems.SortedMembers() {\n\t\tnode := p.sys.Node(id)\n\t\tswitch node.GetMemoryType() {\n\t\tcase system.MemoryTypeDRAM:\n\t\t\tdram.Add(id)\n\t\tcase system.MemoryTypePMEM:\n\t\t\tpmem.Add(id)\n\t\t\t/*\n\t\t\t\tcase system.MemoryTypeHBM:\n\t\t\t\t\thbm.Add(id)\n\t\t\t*/\n\t\t}\n\t}\n\tdata[\"ALL_MEMS\"] = mems.String()\n\tif dram.Size() > 0 {\n\t\tdata[\"DRAM_MEMS\"] = dram.String()\n\t}\n\tif pmem.Size() > 0 {\n\t\tdata[\"PMEM_MEMS\"] = pmem.String()\n\t}\n\tif hbm.Size() > 0 {\n\t\tdata[\"HBM_MEMS\"] = hbm.String()\n\t}\n\n\treturn data\n}\n\n// reallocateResources reallocates the given containers using the given pool hints\nfunc (p *policy) reallocateResources(containers []cache.Container, pools map[string]string) error {\n\terrs := []error{}\n\n\tlog.Info(\"reallocating resources...\")\n\n\tcache.SortContainers(containers)\n\n\tfor _, c := range containers {\n\t\tp.releasePool(c)\n\t}\n\tfor _, c := range containers {\n\t\tlog.Debug(\"reallocating resources for %s...\", c.PrettyName())\n\n\t\tgrant, err := p.allocatePool(c, pools[c.GetCacheID()])\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t} else {\n\t\t\tp.applyGrant(grant)\n\t\t}\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn errors.Join(errs...)\n\t}\n\n\tp.updateSharedAllocations(nil)\n\n\tp.root.Dump(\"<post-realloc>\")\n\n\treturn nil\n}\n\nfunc (p *policy) configNotify(event config.Event, source config.Source) error {\n\tpolicyName := PolicyName\n\tif p.isAlias {\n\t\tpolicyName = AliasName\n\t\t*opt = *aliasOpt\n\t}\n\tlog.Info(\"%s configuration %s:\", policyName, event)\n\tlog.Info(\"  - pin containers to CPUs: %v\", opt.PinCPU)\n\tlog.Info(\"  - pin containers to memory: %v\", opt.PinMemory)\n\tlog.Info(\"  - prefer isolated CPUs: %v\", opt.PreferIsolated)\n\tlog.Info(\"  - prefer shared CPUs: %v\", opt.PreferShared)\n\tlog.Info(\"  - reserved pool namespaces: %v\", opt.ReservedPoolNamespaces)\n\n\tvar allowed, reserved cpuset.CPUSet\n\tvar reinit bool\n\n\tif cpus, ok := p.options.Available[policyapi.DomainCPU]; ok {\n\t\tif cset, ok := cpus.(cpuset.CPUSet); ok {\n\t\t\tallowed = cset\n\t\t}\n\t}\n\tif cpus, ok := p.options.Reserved[policyapi.DomainCPU]; ok {\n\t\tswitch v := cpus.(type) {\n\t\tcase cpuset.CPUSet:\n\t\t\treserved = v\n\t\tcase resapi.Quantity:\n\t\t\treserveCnt := (int(v.MilliValue()) + 999) / 1000\n\t\t\tif reserveCnt != p.reserveCnt {\n\t\t\t\tlog.Warn(\"CPU reservation has changed (%v, was %v)\",\n\t\t\t\t\treserveCnt, p.reserveCnt)\n\t\t\t\treinit = true\n\t\t\t}\n\t\t}\n\t}\n\n\tif !allowed.Equals(p.allowed) {\n\t\tif !(allowed.Size() == 0 && p.allowed.Size() == 0) {\n\t\t\tlog.Warn(\"allowed cpuset changed (%s, was %s)\",\n\t\t\t\tallowed.String(), p.allowed.String())\n\t\t\treinit = true\n\t\t}\n\t}\n\tif !reserved.Equals(p.reserved) {\n\t\tif !(reserved.Size() == 0 && p.reserved.Size() == 0) {\n\t\t\tlog.Warn(\"reserved cpuset changed (%s, was %s)\",\n\t\t\t\treserved.String(), p.reserved.String())\n\t\t\treinit = true\n\t\t}\n\t}\n\n\t//\n\t// Notes:\n\t//   If the allowed or reserved resources have changed, we need to\n\t//   rebuild our pool hierarchy using the updated constraints and\n\t//   also update the existing allocations accordingly. We do this\n\t//   first reinitializing the policy then reloading the allocations\n\t//   from the cache. If we fail, we restore the original state of\n\t//   the policy and reject the new configuration.\n\t//\n\n\tif reinit {\n\t\tlog.Warn(\"reinitializing %s policy...\", PolicyName)\n\n\t\tsavedPolicy := *p\n\t\tallocations := savedPolicy.allocations.clone()\n\n\t\tif err := p.initialize(); err != nil {\n\t\t\t*p = savedPolicy\n\t\t\treturn policyError(\"failed to reconfigure: %v\", err)\n\t\t}\n\n\t\tfor _, grant := range allocations.grants {\n\t\t\tif err := grant.RefetchNodes(); err != nil {\n\t\t\t\t*p = savedPolicy\n\t\t\t\treturn policyError(\"failed to reconfigure: %v\", err)\n\t\t\t}\n\t\t}\n\n\t\tlog.Warn(\"updating existing allocations...\")\n\t\tif err := p.restoreAllocations(&allocations); err != nil {\n\t\t\t*p = savedPolicy\n\t\t\treturn policyError(\"failed to reconfigure: %v\", err)\n\t\t}\n\n\t\tp.root.Dump(\"<post-config>\")\n\t}\n\n\treturn nil\n}\n\n// Initialize or reinitialize the policy.\nfunc (p *policy) initialize() error {\n\tp.nodes = nil\n\tp.pools = nil\n\tp.root = nil\n\tp.nodeCnt = 0\n\tp.depth = 0\n\tp.allocations = p.newAllocations()\n\n\tif err := p.checkConstraints(); err != nil {\n\t\treturn err\n\t}\n\n\tif err := p.buildPoolsByTopology(); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Check the constraints passed to us.\nfunc (p *policy) checkConstraints() error {\n\tif c, ok := p.options.Available[policyapi.DomainCPU]; ok {\n\t\tp.allowed = c.(cpuset.CPUSet)\n\t} else {\n\t\t// default to all online cpus\n\t\tp.allowed = p.sys.CPUSet().Difference(p.sys.Offlined())\n\t}\n\n\tp.isolated = p.sys.Isolated().Intersection(p.allowed)\n\n\tc, ok := p.options.Reserved[policyapi.DomainCPU]\n\tif !ok {\n\t\treturn policyError(\"cannot start without CPU reservation\")\n\t}\n\n\tswitch c.(type) {\n\tcase cpuset.CPUSet:\n\t\tp.reserved = c.(cpuset.CPUSet)\n\t\t// check that all reserved CPUs are in the allowed set\n\t\tif !p.reserved.Difference(p.allowed).IsEmpty() {\n\t\t\treturn policyError(\"invalid reserved cpuset %s, some CPUs (%s) are not \"+\n\t\t\t\t\"part of the online allowed cpuset (%s)\", p.reserved,\n\t\t\t\tp.reserved.Difference(p.allowed), p.allowed)\n\t\t}\n\t\t// check that none of the reserved CPUs are isolated\n\t\tif !p.reserved.Intersection(p.isolated).IsEmpty() {\n\t\t\treturn policyError(\"invalid reserved cpuset %s, some CPUs (%s) are also isolated\",\n\t\t\t\tp.reserved.Intersection(p.isolated))\n\t\t}\n\n\tcase resapi.Quantity:\n\t\tqty := c.(resapi.Quantity)\n\t\tp.reserveCnt = (int(qty.MilliValue()) + 999) / 1000\n\t\t// Use CpuAllocator to pick reserved CPUs among\n\t\t// allowed ones. Because using those CPUs is allowed,\n\t\t// they remain (they are put back) in the allowed set.\n\t\tcset, err := p.cpuAllocator.AllocateCpus(&p.allowed, p.reserveCnt, cpuallocator.PriorityNormal)\n\t\tp.allowed = p.allowed.Union(cset)\n\t\tif err != nil {\n\t\t\tlog.Fatal(\"cannot reserve %dm CPUs for ReservedResources from AvailableResources: %s\", qty.MilliValue(), err)\n\t\t}\n\t\tp.reserved = cset\n\t}\n\n\tif p.reserved.IsEmpty() {\n\t\treturn policyError(\"cannot start without CPU reservation\")\n\t}\n\n\treturn nil\n}\n\nfunc (p *policy) restoreCache() error {\n\tallocations := p.newAllocations()\n\tif p.cache.GetPolicyEntry(keyAllocations, &allocations) {\n\t\tif err := p.restoreAllocations(&allocations); err != nil {\n\t\t\treturn policyError(\"failed to restore allocations from cache: %v\", err)\n\t\t}\n\t\tp.allocations.Dump(log.Info, \"restored \")\n\t}\n\tp.saveAllocations()\n\n\treturn nil\n}\n\nfunc (p *policy) checkColdstartOff() {\n\tfor _, id := range p.sys.NodeIDs() {\n\t\tnode := p.sys.Node(id)\n\t\tif node.GetMemoryType() == system.MemoryTypePMEM {\n\t\t\tif !node.HasNormalMemory() {\n\t\t\t\tcoldStartOff = true\n\t\t\t\tlog.Error(\"coldstart forced off: NUMA node #%d does not have normal memory\", id)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// newAllocations returns a new initialized empty set of allocations.\nfunc (p *policy) newAllocations() allocations {\n\treturn allocations{policy: p, grants: make(map[string]Grant)}\n}\n\n// clone creates a copy of the allocation.\nfunc (a *allocations) clone() allocations {\n\to := allocations{policy: a.policy, grants: make(map[string]Grant)}\n\tfor id, grant := range a.grants {\n\t\to.grants[id] = grant.Clone()\n\t}\n\treturn o\n}\n\n// getContainerPoolHints creates container pool hints for the current grants.\nfunc (a *allocations) getContainerPoolHints() ([]cache.Container, map[string]string) {\n\tcontainers := make([]cache.Container, 0, len(a.grants))\n\thints := make(map[string]string)\n\tfor _, grant := range a.grants {\n\t\tc := grant.GetContainer()\n\t\tcontainers = append(containers, c)\n\t\thints[c.GetCacheID()] = grant.GetCPUNode().Name()\n\t}\n\treturn containers, hints\n}\n\n// Register us as a policy implementation.\nfunc init() {\n\tpolicyapi.Register(PolicyName, PolicyDescription, CreateTopologyAwarePolicy)\n\tpolicyapi.Register(AliasName, PolicyDescription, CreateMemtierPolicy)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/error.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage policy\n\nimport (\n\t\"fmt\"\n)\n\nfunc policyError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"policy: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/flags.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage policy\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cgroups\"\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n)\n\nconst (\n\t// NonePolicy is the name of our no-op policy.\n\tNonePolicy = \"none\"\n\t// DefaultPolicy is the name of our default policy.\n\tDefaultPolicy = NonePolicy\n\t// ConfigPath is the configuration module path for the generic policy layer.\n\tConfigPath = \"policy\"\n)\n\n// Options captures our configurable parameters.\ntype options struct {\n\t// Policy is the name of the policy backend to activate.\n\tPolicy string `json:\"Active\"`\n\t// Available hardware resources to use.\n\tAvailable ConstraintSet `json:\"AvailableResources,omitempty\"`\n\t// Reserved hardware resources, for system and kube tasks.\n\tReserved ConstraintSet `json:\"ReservedResources,omitempty\"`\n}\n\n// Our runtime configuration.\nvar opt = defaultOptions().(*options)\n\n// MarshalJSON implements JSON marshalling for ConstraintSets.\nfunc (cs ConstraintSet) MarshalJSON() ([]byte, error) {\n\tobj := map[string]interface{}{}\n\tfor domain, constraint := range cs {\n\t\tname := string(domain)\n\t\tswitch constraint.(type) {\n\t\tcase cpuset.CPUSet:\n\t\t\tobj[name] = \"cpuset:\" + constraint.(cpuset.CPUSet).String()\n\t\tcase resource.Quantity:\n\t\t\tqty := constraint.(resource.Quantity)\n\t\t\tobj[name] = qty.String()\n\t\tcase int:\n\t\t\tobj[name] = strconv.Itoa(constraint.(int))\n\t\tdefault:\n\t\t\treturn nil, policyError(\"invalid %v constraint of type %T\", domain, constraint)\n\t\t}\n\t}\n\treturn json.Marshal(obj)\n}\n\n// UnmarshalJSON implements JSON unmarshalling for ConstraintSets.\nfunc (cs *ConstraintSet) UnmarshalJSON(raw []byte) error {\n\tset := make(ConstraintSet)\n\tobj := map[string]interface{}{}\n\tif err := json.Unmarshal(raw, &obj); err != nil {\n\t\treturn policyError(\"failed to unmarshal ConstraintSet: %v\", err)\n\t}\n\n\tfor name, value := range obj {\n\t\tswitch strings.ToUpper(name) {\n\t\tcase string(DomainCPU):\n\t\t\tswitch v := value.(type) {\n\t\t\tcase string:\n\t\t\t\tif err := set.parseCPU(v); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase int:\n\t\t\t\tset.setCPUMilliQuantity(v)\n\t\t\tcase float64:\n\t\t\t\tset.setCPUMilliQuantity(int(1000.0 * v))\n\t\t\tdefault:\n\t\t\t\treturn policyError(\"invalid CPU constraint of type %T\", value)\n\t\t\t}\n\t\tdefault:\n\t\t\treturn policyError(\"internal error: unhandled ConstraintSet domain %s\", name)\n\t\t}\n\t}\n\n\t*cs = set\n\treturn nil\n}\n\nfunc (cs *ConstraintSet) String() string {\n\tret := \"\"\n\tsep := \"\"\n\tfor domain, value := range *cs {\n\t\tret += sep + string(domain) + \"=\" + ConstraintToString(value)\n\t\tsep = \",\"\n\t}\n\treturn ret\n}\n\nfunc (cs *ConstraintSet) parseCPU(value string) error {\n\tkind, spec := \"\", \"\"\n\tif sep := strings.IndexByte(value, ':'); sep != -1 {\n\t\tkind = value[:sep]\n\t\tspec = value[sep+1:]\n\t} else {\n\t\tspec = value\n\t}\n\tif len(spec) == 0 {\n\t\treturn policyError(\"missing CPU constraint value\")\n\t}\n\n\tswitch {\n\tcase kind == \"cgroup\" || spec[0] == '/':\n\t\tif err := cs.parseCPUFromCgroup(spec); err != nil {\n\t\t\treturn err\n\t\t}\n\tcase kind == \"cpuset\" || strings.IndexAny(spec, \"-,\") != -1:\n\t\tif err := cs.parseCPUSet(spec); err != nil {\n\t\t\treturn err\n\t\t}\n\tcase kind == \"\":\n\t\tif err := cs.parseCPUQuantity(spec); err != nil {\n\t\t\treturn err\n\t\t}\n\tdefault:\n\t\treturn policyError(\"invalid CPU constraint qualifier %q\", kind)\n\t}\n\n\treturn nil\n}\n\nfunc (cs *ConstraintSet) parseCPUSet(value string) error {\n\tcset, err := cpuset.Parse(value)\n\tif err != nil {\n\t\treturn policyError(\"failed to parse CPU cpuset constraint %q: %v\",\n\t\t\tvalue, err)\n\t}\n\t(*cs)[DomainCPU] = cset\n\treturn nil\n}\n\nfunc (cs *ConstraintSet) parseCPUQuantity(value string) error {\n\tqty, err := resource.ParseQuantity(value)\n\tif err != nil {\n\t\treturn policyError(\"failed to parse CPU Quantity constraint %q: %v\",\n\t\t\tvalue, err)\n\t}\n\t(*cs)[DomainCPU] = qty\n\treturn nil\n}\n\nfunc (cs *ConstraintSet) parseCPUFromCgroup(dir string) error {\n\tpathToCpuset := func(outPath *string, fragments ...string) bool {\n\t\t*outPath = filepath.Join(filepath.Join(fragments...), \"cpuset.cpus\")\n\t\t_, err := os.Stat(*outPath)\n\t\treturn !errors.Is(err, os.ErrNotExist)\n\t}\n\tpath := \"\"\n\tswitch {\n\tcase len(dir) == 0:\n\t\treturn policyError(\"empty CPU cgroup constraint\")\n\tcase dir[0] == '/' && pathToCpuset(&path, dir):\n\t\t// dir is a direct, absolute path to an existing cgroup\n\tcase pathToCpuset(&path, cgroups.GetMountDir(), dir):\n\t\t// dir is a relative path starting from the cgroup mount point\n\tcase pathToCpuset(&path, cgroups.Cpuset.Path(), dir):\n\t\t// dir is a relative path starting from the cpuset controller (cgroup v1)\n\tdefault:\n\t\t// dir is none of the previous\n\t\treturn policyError(\"failed to find cpuset.cpus for CPU cgroup constraint %q\", dir)\n\t}\n\tbytes, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn policyError(\"failed read CPU cpuset cgroup constraint %q: %v\",\n\t\t\tpath, err)\n\t}\n\tcpus := strings.TrimSuffix(string(bytes), \"\\n\")\n\tcset, err := cpuset.Parse(cpus)\n\tif err != nil {\n\t\treturn policyError(\"failed to parse cpuset cgroup constraint %q: %v\",\n\t\t\tcpus, err)\n\t}\n\t(*cs)[DomainCPU] = cset\n\treturn nil\n}\n\nfunc (cs *ConstraintSet) setCPUMilliQuantity(value int) {\n\tqty := resource.NewMilliQuantity(int64(value), resource.DecimalSI)\n\t(*cs)[DomainCPU] = *qty\n}\n\n// AvailablePolicy describes an available policy.\ntype AvailablePolicy struct {\n\t// Name is the name of the policy.\n\tName string\n\t// Description is a short description of the policy.\n\tDescription string\n}\n\n// AvailablePolicies returns the available policies and their descriptions.\nfunc AvailablePolicies() []*AvailablePolicy {\n\tpolicies := make([]*AvailablePolicy, 0, len(backends)+1)\n\tfor name, be := range backends {\n\t\tpolicies = append(policies, &AvailablePolicy{\n\t\t\tName:        name,\n\t\t\tDescription: be.description,\n\t\t})\n\t}\n\tsort.Slice(policies, func(i, j int) bool { return policies[i].Name < policies[j].Name })\n\n\treturn policies\n}\n\n// defaultOptions returns a new options instance, all initialized to defaults.\nfunc defaultOptions() interface{} {\n\treturn &options{\n\t\tPolicy:    DefaultPolicy,\n\t\tAvailable: ConstraintSet{},\n\t\tReserved:  ConstraintSet{},\n\t}\n}\n\n// Register us for configuration handling.\nfunc init() {\n\tconfig.Register(ConfigPath, \"Generic policy layer.\", opt, defaultOptions,\n\t\tconfig.WithNotify(configNotify))\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/policy/policy.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage policy\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/blockio\"\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/agent\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control/rdt\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\tsystem \"github.com/intel/cri-resource-manager/pkg/sysfs\"\n)\n\n// Domain represents a hardware resource domain that can be policied by a backend.\ntype Domain string\n\nconst (\n\t// DomainCPU is the CPU resource domain.\n\tDomainCPU Domain = \"CPU\"\n\t// DomainMemory is the memory resource domain.\n\tDomainMemory Domain = \"Memory\"\n\t// DomainHugePage is the hugepages resource domain.\n\tDomainHugePage Domain = \"HugePages\"\n\t// DomainCache is the CPU cache resource domain.\n\tDomainCache Domain = \"Cache\"\n\t// DomainMemoryBW is the memory resource bandwidth.\n\tDomainMemoryBW Domain = \"MBW\"\n)\n\n// Constraint describes constraint of one hardware domain\ntype Constraint interface{}\n\n// ConstraintSet describes, per hardware domain, the resources available for a policy.\ntype ConstraintSet map[Domain]Constraint\n\n// Options describes policy options\ntype Options struct {\n\t// Client interface to cri-resmgr agent\n\tAgentCli agent.Interface\n\t// SendEvent is the function for delivering events back to the resource manager.\n\tSendEvent SendEventFn\n}\n\n// BackendOptions describes the options for a policy backend instance\ntype BackendOptions struct {\n\t// System provides system/HW/topology information\n\tSystem system.System\n\t// System state/cache\n\tCache cache.Cache\n\t// Resource availibility constraint\n\tAvailable ConstraintSet\n\t// Resource reservation constraint\n\tReserved ConstraintSet\n\t// Client interface to cri-resmgr agent\n\tAgentCli agent.Interface\n\t// SendEvent is the function for delivering events up to the resource manager.\n\tSendEvent SendEventFn\n}\n\n// CreateFn is the type for functions used to create a policy instance.\ntype CreateFn func(*BackendOptions) Backend\n\n// SendEventFn is the type for a function to send events back to the resource manager.\ntype SendEventFn func(interface{}) error\n\nconst (\n\t// ExportedResources is the basename of the file container resources are exported to.\n\tExportedResources = \"resources.sh\"\n\t// ExportSharedCPUs is the shell variable used to export shared container CPUs.\n\tExportSharedCPUs = \"SHARED_CPUS\"\n\t// ExportIsolatedCPUs is the shell variable used to export isolated container CPUs.\n\tExportIsolatedCPUs = \"ISOLATED_CPUS\"\n\t// ExportExclusiveCPUs is the shell variable used to export exclusive container CPUs.\n\tExportExclusiveCPUs = \"EXCLUSIVE_CPUS\"\n)\n\n// Backend is the policy (decision making logic) interface exposed by implementations.\n//\n// A backends operates in a set of policy domains. Currently each policy domain\n// corresponds to some particular hardware resource (CPU, memory, cache, etc).\ntype Backend interface {\n\t// Name gets the well-known name of this policy.\n\tName() string\n\t// Description gives a verbose description about the policy implementation.\n\tDescription() string\n\t// Start up and sycnhronizes the policy, using the given cache and resource constraints.\n\tStart([]cache.Container, []cache.Container) error\n\t// Sync synchronizes the policy, allocating/releasing the given containers.\n\tSync([]cache.Container, []cache.Container) error\n\t// AllocateResources allocates resources to/for a container.\n\tAllocateResources(cache.Container) error\n\t// ReleaseResources release resources of a container.\n\tReleaseResources(cache.Container) error\n\t// UpdateResources updates resource allocations of a container.\n\tUpdateResources(cache.Container) error\n\t// Rebalance tries an optimal allocation of resources for the current container.\n\tRebalance() (bool, error)\n\t// HandleEvent processes the given event. The returned boolean indicates whether\n\t// changes have been made to any of the containers while handling the event.\n\tHandleEvent(*events.Policy) (bool, error)\n\t// ExportResourceData provides resource data to export for the container.\n\tExportResourceData(cache.Container) map[string]string\n\t// Introspect provides data for external introspection.\n\tIntrospect(*introspect.State)\n\t// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\n\tDescribeMetrics() []*prometheus.Desc\n\t// PollMetrics provides policy metrics for monitoring.\n\tPollMetrics() Metrics\n\t// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\n\tCollectMetrics(Metrics) ([]prometheus.Metric, error)\n}\n\n// Policy is the exposed interface for container resource allocations decision making.\ntype Policy interface {\n\t// Start starts up policy, prepare for serving resource management requests.\n\tStart([]cache.Container, []cache.Container) error\n\t// Sync synchronizes the state of the active policy.\n\tSync([]cache.Container, []cache.Container) error\n\t// AllocateResources allocates resources to a container.\n\tAllocateResources(cache.Container) error\n\t// ReleaseResources releases resources of a container.\n\tReleaseResources(cache.Container) error\n\t// UpdateResources updates resource allocations of a container.\n\tUpdateResources(cache.Container) error\n\t// Rebalance tries to find an optimal allocation of resources for the current containers.\n\tRebalance() (bool, error)\n\t// HandleEvent passes on the given event to the active policy. The returned boolean\n\t// indicates whether changes have been made to any of the containers while handling\n\t// the event.\n\tHandleEvent(*events.Policy) (bool, error)\n\t// ExportResourceData exports/updates resource data for the container.\n\tExportResourceData(cache.Container)\n\t// Introspect provides data for external introspection.\n\tIntrospect() *introspect.State\n\t// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\n\tDescribeMetrics() []*prometheus.Desc\n\t// PollMetrics provides policy metrics for monitoring.\n\tPollMetrics() Metrics\n\t// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\n\tCollectMetrics(Metrics) ([]prometheus.Metric, error)\n}\n\ntype Metrics interface{}\n\n// Policy instance/state.\ntype policy struct {\n\toptions   Options            // policy options\n\tcache     cache.Cache        // system state cache\n\tactive    Backend            // our active backend\n\tsystem    system.System      // system/HW/topology info\n\tinspsys   *introspect.System // ditto for introspection\n\tsendEvent SendEventFn        // function to send event up to the resource manager\n}\n\n// backend is a registered Backend.\ntype backend struct {\n\tname        string   // unqiue backend name\n\tdescription string   // verbose backend description\n\tcreate      CreateFn // backend creation function\n}\n\n// Out logger instance.\nvar log logger.Logger = logger.NewLogger(\"policy\")\n\n// Registered backends.\nvar backends = make(map[string]*backend)\n\n// Options passed to created/activated backend.\nvar backendOpts = &BackendOptions{}\n\n// ActivePolicy returns the name of the policy to be activated.\nfunc ActivePolicy() string {\n\treturn opt.Policy\n}\n\n// NewPolicy creates a policy instance using the selected backend.\nfunc NewPolicy(cache cache.Cache, o *Options) (Policy, error) {\n\tsys, err := system.DiscoverSystem()\n\tif err != nil {\n\t\treturn nil, policyError(\"failed to discover system topology: %v\", err)\n\t}\n\n\tp := &policy{\n\t\tcache:   cache,\n\t\tsystem:  sys,\n\t\toptions: *o,\n\t}\n\n\tactive, ok := backends[opt.Policy]\n\tif !ok {\n\t\treturn nil, policyError(\"unknown policy '%s' requested\", opt.Policy)\n\t}\n\n\tlog.Info(\"activating '%s' policy...\", active.name)\n\n\tif len(opt.Available) != 0 {\n\t\tlog.Info(\"  with available resources:\")\n\t\tfor n, r := range opt.Available {\n\t\t\tlog.Info(\"    - %s=%s\", n, ConstraintToString(r))\n\t\t}\n\t}\n\tif len(opt.Reserved) != 0 {\n\t\tlog.Info(\"  with reserved resources:\")\n\t\tfor n, r := range opt.Reserved {\n\t\t\tlog.Info(\"    - %s=%s\", n, ConstraintToString(r))\n\t\t}\n\t}\n\n\tif log.DebugEnabled() {\n\t\tlogger.Get(opt.Policy).EnableDebug()\n\t}\n\n\tbackendOpts.Cache = p.cache\n\tbackendOpts.System = p.system\n\tbackendOpts.Available = opt.Available\n\tbackendOpts.Reserved = opt.Reserved\n\tbackendOpts.AgentCli = o.AgentCli\n\tbackendOpts.SendEvent = o.SendEvent\n\n\tp.active = active.create(backendOpts)\n\n\treturn p, nil\n}\n\n// Start starts up policy, preparing it for resving requests.\nfunc (p *policy) Start(add []cache.Container, del []cache.Container) error {\n\tlog.Info(\"starting policy '%s'...\", p.active.Name())\n\treturn p.active.Start(add, del)\n}\n\n// Sync synchronizes the active policy state.\nfunc (p *policy) Sync(add []cache.Container, del []cache.Container) error {\n\treturn p.active.Sync(add, del)\n}\n\n// AllocateResources allocates resources for a container.\nfunc (p *policy) AllocateResources(c cache.Container) error {\n\treturn p.active.AllocateResources(c)\n}\n\n// ReleaseResources release resources of a container.\nfunc (p *policy) ReleaseResources(c cache.Container) error {\n\treturn p.active.ReleaseResources(c)\n}\n\n// UpdateResources updates resource allocations of a container.\nfunc (p *policy) UpdateResources(c cache.Container) error {\n\treturn p.active.UpdateResources(c)\n}\n\n// Rebalance tries to find a more optimal allocation of resources for the current containers.\nfunc (p *policy) Rebalance() (bool, error) {\n\treturn p.active.Rebalance()\n}\n\n// HandleEvent passes on the given event to the active policy.\nfunc (p *policy) HandleEvent(e *events.Policy) (bool, error) {\n\treturn p.active.HandleEvent(e)\n}\n\n// ExportResourceData exports/updates resource data for the container.\nfunc (p *policy) ExportResourceData(c cache.Container) {\n\tvar buf bytes.Buffer\n\n\tdata := p.active.ExportResourceData(c)\n\tkeys := []string{}\n\tfor key := range data {\n\t\tkeys = append(keys, key)\n\t}\n\tsort.Strings(keys)\n\n\tfor _, key := range keys {\n\t\tvalue := data[key]\n\t\tif _, err := buf.WriteString(fmt.Sprintf(\"%s=%q\\n\", key, value)); err != nil {\n\t\t\tlog.Error(\"container %s: failed to export resource data (%s=%q)\",\n\t\t\t\tc.PrettyName(), key, value)\n\t\t\tbuf.Reset()\n\t\t\tbreak\n\t\t}\n\t}\n\n\tp.cache.WriteFile(c.GetCacheID(), ExportedResources, 0644, buf.Bytes())\n}\n\n// Introspect provides data for external introspection/visualization.\nfunc (p *policy) Introspect() *introspect.State {\n\tpods := p.cache.GetPods()\n\tstate := &introspect.State{Pods: make(map[string]*introspect.Pod, len(pods))}\n\n\tfor _, p := range pods {\n\t\tcontainers := p.GetContainers()\n\t\tif len(containers) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpod := &introspect.Pod{\n\t\t\tID:         p.GetID(),\n\t\t\tUID:        p.GetUID(),\n\t\t\tName:       p.GetName(),\n\t\t\tContainers: make(map[string]*introspect.Container, len(containers)),\n\t\t}\n\n\t\tfor _, c := range containers {\n\t\t\tcontainer := &introspect.Container{\n\t\t\t\tID:      c.GetID(),\n\t\t\t\tName:    c.GetName(),\n\t\t\t\tCommand: c.GetCommand(),\n\t\t\t\tArgs:    c.GetArgs(),\n\t\t\t\tHints:   introspect.TopologyHints(c.GetTopologyHints()),\n\t\t\t}\n\t\t\tresources := c.GetResourceRequirements()\n\t\t\tif req, ok := resources.Requests[corev1.ResourceCPU]; ok {\n\t\t\t\tif value := req.MilliValue(); value > 0 {\n\t\t\t\t\tcontainer.CPURequest = value\n\t\t\t\t}\n\t\t\t}\n\t\t\tif lim, ok := resources.Limits[corev1.ResourceCPU]; ok {\n\t\t\t\tif value := lim.MilliValue(); value > 0 {\n\t\t\t\t\tcontainer.CPULimit = value\n\t\t\t\t}\n\t\t\t}\n\t\t\tif req, ok := resources.Requests[corev1.ResourceMemory]; ok {\n\t\t\t\tif value := req.Value(); value > 0 {\n\t\t\t\t\tcontainer.MemoryRequest = value\n\t\t\t\t}\n\t\t\t}\n\t\t\tif lim, ok := resources.Limits[corev1.ResourceMemory]; ok {\n\t\t\t\tif value := lim.Value(); value > 0 {\n\t\t\t\t\tcontainer.MemoryLimit = value\n\t\t\t\t}\n\t\t\t}\n\t\t\tpod.Containers[container.ID] = container\n\t\t}\n\t\tstate.Pods[pod.ID] = pod\n\t}\n\n\tif p.inspsys == nil {\n\t\tsys := &introspect.System{\n\t\t\tSockets: make(map[int]*introspect.Socket, p.system.PackageCount()),\n\t\t\tNodes:   make(map[int]*introspect.Node, p.system.NUMANodeCount()),\n\t\t}\n\t\tfor _, id := range p.system.PackageIDs() {\n\t\t\tpkg := p.system.Package(id)\n\t\t\tsys.Sockets[int(id)] = &introspect.Socket{ID: int(id), CPUs: pkg.CPUSet().String()}\n\t\t}\n\t\tfor _, id := range p.system.NodeIDs() {\n\t\t\tnode := p.system.Node(id)\n\t\t\tsys.Nodes[int(id)] = &introspect.Node{ID: int(id), CPUs: node.CPUSet().String()}\n\t\t}\n\t\tsys.Isolated = p.system.Isolated().String()\n\t\tsys.Offlined = p.system.Offlined().String()\n\t\tp.inspsys = sys\n\t}\n\n\trdtClassNames := []string{}\n\tfor _, rdtClass := range rdt.GetClasses() {\n\t\trdtClassNames = append(rdtClassNames, rdtClass.Name())\n\t}\n\tblkioClassNames := []string{}\n\tfor _, blkioClass := range blockio.GetClasses() {\n\t\tblkioClassNames = append(blkioClassNames, blkioClass.Name)\n\t}\n\tp.inspsys.RDTClasses = rdtClassNames\n\tp.inspsys.Policy = opt.Policy\n\n\tstate.System = p.inspsys\n\tp.active.Introspect(state)\n\n\treturn state\n}\n\n// PollMetrics provides policy metrics for monitoring.\nfunc (p *policy) PollMetrics() Metrics {\n\treturn p.active.PollMetrics()\n}\n\n// DescribeMetrics generates policy-specific prometheus metrics data descriptors.\nfunc (p *policy) DescribeMetrics() []*prometheus.Desc {\n\treturn p.active.DescribeMetrics()\n}\n\n// CollectMetrics generates prometheus metrics from cached/polled policy-specific metrics data.\nfunc (p *policy) CollectMetrics(m Metrics) ([]prometheus.Metric, error) {\n\treturn p.active.CollectMetrics(m)\n}\n\n// Register registers a policy backend.\nfunc Register(name, description string, create CreateFn) error {\n\tlog.Info(\"registering policy '%s'...\", name)\n\n\tif o, ok := backends[name]; ok {\n\t\treturn policyError(\"policy %s already registered (%s)\", name, o.description)\n\t}\n\n\tbackends[name] = &backend{\n\t\tname:        name,\n\t\tdescription: description,\n\t\tcreate:      create,\n\t}\n\n\treturn nil\n}\n\n// ConstraintToString returns the given constraint as a string.\nfunc ConstraintToString(value Constraint) string {\n\tswitch value.(type) {\n\tcase cpuset.CPUSet:\n\t\treturn \"#\" + value.(cpuset.CPUSet).String()\n\tcase int:\n\t\treturn strconv.Itoa(value.(int))\n\tcase string:\n\t\treturn value.(string)\n\tcase resource.Quantity:\n\t\tqty := value.(resource.Quantity)\n\t\treturn qty.String()\n\tdefault:\n\t\treturn fmt.Sprintf(\"<???(type:%T)>\", value)\n\t}\n}\n\n// configNotify is the configuration change notification callback for the genric policy layer.\nfunc configNotify(_ config.Event, _ config.Source) error {\n\t// let the active policy know of changes\n\tbackendOpts.Available = opt.Available\n\tbackendOpts.Reserved = opt.Reserved\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/requests.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tconfig \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/events\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/server\"\n)\n\nconst (\n\tkubeAPIVersion = \"0.1.0\"\n)\n\nvar knownRuntimes = []string{\n\t\"containerd\",\n\t\"cri-o\",\n}\n\n// setupRequestProcessing prepares the resource manager for CRI request processing.\nfunc (m *resmgr) setupRequestProcessing() error {\n\tinterceptors := map[string]server.Interceptor{\n\t\t\"RunPodSandbox\":    m.RunPod,\n\t\t\"StopPodSandbox\":   m.StopPod,\n\t\t\"RemovePodSandbox\": m.RemovePod,\n\n\t\t\"CreateContainer\": m.CreateContainer,\n\t\t\"StartContainer\":  m.StartContainer,\n\t\t\"StopContainer\":   m.StopContainer,\n\t\t\"RemoveContainer\": m.RemoveContainer,\n\t\t\"ListContainers\":  m.ListContainers,\n\n\t\t\"UpdateContainerResources\": m.UpdateContainer,\n\t}\n\n\tif err := m.relay.Server().RegisterInterceptors(interceptors); err != nil {\n\t\treturn resmgrError(\"failed to register resource-manager CRI interceptors: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// disambiguate produces disambiguation context for a request/reply dump.\nfunc (m *resmgr) disambiguate(msg interface{}) string {\n\tvar qualifier string\n\n\tm.RLock()\n\tdefer m.RUnlock()\n\n\tswitch req := msg.(type) {\n\tcase *criv1.RunPodSandboxRequest:\n\t\tif req.Config != nil && req.Config.Metadata != nil {\n\t\t\tqualifier = req.Config.Metadata.Name\n\t\t}\n\tcase *criv1.StopPodSandboxRequest:\n\t\tif pod, ok := m.cache.LookupPod(req.PodSandboxId); ok {\n\t\t\tqualifier = pod.GetName()\n\t\t} else {\n\t\t\tqualifier = \"unknown pod \" + req.PodSandboxId\n\t\t}\n\tcase *criv1.RemovePodSandboxRequest:\n\t\tif pod, ok := m.cache.LookupPod(req.PodSandboxId); ok {\n\t\t\tqualifier = pod.GetName()\n\t\t} else {\n\t\t\tqualifier = \"unknown pod \" + req.PodSandboxId\n\t\t}\n\n\tcase *criv1.CreateContainerRequest:\n\t\tswitch {\n\t\tcase req.SandboxConfig == nil || req.SandboxConfig.Metadata == nil:\n\t\t\tqualifier = \"missing pod metadata in request\"\n\t\tcase req.Config == nil || req.Config.Metadata == nil:\n\t\t\tqualifier = \"missing metadata in request\"\n\t\tdefault:\n\t\t\tqualifier = req.SandboxConfig.Metadata.Name + \":\" + req.Config.Metadata.Name\n\t\t}\n\n\tcase *criv1.StartContainerRequest:\n\t\tif container, ok := m.cache.LookupContainer(req.ContainerId); ok {\n\t\t\tqualifier = container.PrettyName()\n\t\t} else {\n\t\t\tqualifier = \"unknown container \" + req.ContainerId\n\t\t}\n\tcase *criv1.StopContainerRequest:\n\t\tif container, ok := m.cache.LookupContainer(req.ContainerId); ok {\n\t\t\tqualifier = container.PrettyName()\n\t\t} else {\n\t\t\tqualifier = \"unknown container \" + req.ContainerId\n\t\t}\n\tcase *criv1.RemoveContainerRequest:\n\t\tif container, ok := m.cache.LookupContainer(req.ContainerId); ok {\n\t\t\tqualifier = container.PrettyName()\n\t\t} else {\n\t\t\tqualifier = \"unknown container \" + req.ContainerId\n\t\t}\n\n\tcase *criv1.UpdateContainerResourcesRequest:\n\t\tif container, ok := m.cache.LookupContainer(req.ContainerId); ok {\n\t\t\tqualifier = container.PrettyName()\n\t\t} else {\n\t\t\tqualifier = \"unknown container \" + req.ContainerId\n\t\t}\n\t}\n\n\tif qualifier != \"\" {\n\t\treturn \"<\" + qualifier + \">\"\n\t}\n\n\treturn \"\"\n}\n\n// startRequestProcessing starts request processing by starting the active policy.\nfunc (m *resmgr) startRequestProcessing() error {\n\tctx := context.Background()\n\tadd, del, err := m.syncWithCRI(ctx)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t//\n\t// Notes:\n\t//   While normally it is enough to release stale containers and allocate\n\t//   newly discovered ones, if we are switching policies we need to force\n\t//   reallocating everything. Otherwise containers already present in the\n\t//   cache would not get properly updated by the new policy.\n\t//\n\tif m.policySwitch {\n\t\tcontainers := m.cache.GetContainers()\n\t\tcache.SortContainers(containers)\n\t\tadd, del = containers, containers\n\t\tm.policySwitch = false\n\t}\n\n\tif err := m.policy.Start(add, del); err != nil {\n\t\treturn resmgrError(\"failed to start policy %s: %v\", policy.ActivePolicy(), err)\n\t}\n\n\tif err := m.runPostReleaseHooks(ctx, \"startup\", del...); err != nil {\n\t\tm.Error(\"startup: failed to run post-release hooks: %v\", err)\n\t}\n\n\treturn m.cache.Save()\n}\n\n// syncWithCRI synchronizes cache pods and containers with the CRI runtime.\nfunc (m *resmgr) syncWithCRI(ctx context.Context) ([]cache.Container, []cache.Container, error) {\n\tif !m.relay.Client().HasRuntimeService() {\n\t\treturn nil, nil, nil\n\t}\n\n\tm.Info(\"synchronizing cache state with CRI runtime...\")\n\n\tadd, del := []cache.Container{}, []cache.Container{}\n\tpods, err := m.relay.Client().ListPodSandbox(ctx, &criv1.ListPodSandboxRequest{})\n\tif err != nil {\n\t\treturn nil, nil, resmgrError(\"cache synchronization pod query failed: %v\", err)\n\t}\n\n\tstatus := map[string]*cache.PodStatus{}\n\tfor _, pod := range pods.Items {\n\t\tif s, err := m.queryPodStatus(ctx, pod.Id); err != nil {\n\t\t\tm.Error(\"%s: failed to query pod status: %v\", pod.Id, err)\n\t\t} else {\n\t\t\tstatus[pod.Id] = s\n\t\t}\n\t}\n\t_, _, deleted := m.cache.RefreshPods(pods, status)\n\tfor _, c := range deleted {\n\t\tm.Info(\"discovered stale container %s...\", c.GetID())\n\t\tdel = append(del, c)\n\t}\n\n\tcontainers, err := m.relay.Client().ListContainers(ctx, &criv1.ListContainersRequest{})\n\tif err != nil {\n\t\treturn nil, nil, resmgrError(\"cache synchronization container query failed: %v\", err)\n\t}\n\tadded, deleted := m.cache.RefreshContainers(containers)\n\tfor _, c := range added {\n\t\tif c.GetState() != cache.ContainerStateRunning {\n\t\t\tm.Info(\"ignoring discovered container %s (in state %v)...\",\n\t\t\t\tc.GetID(), c.GetState())\n\t\t\tcontinue\n\t\t}\n\t\tm.Info(\"discovered out-of-sync running container %s...\", c.GetID())\n\t\tadd = append(add, c)\n\t}\n\tfor _, c := range deleted {\n\t\tm.Info(\"discovered stale container %s...\", c.GetID())\n\t\tdel = append(del, c)\n\t}\n\n\treturn add, del, nil\n}\n\nfunc (m *resmgr) queryPodStatus(ctx context.Context, podID string) (*cache.PodStatus, error) {\n\tresponse, err := m.relay.Client().PodSandboxStatus(ctx,\n\t\t&criv1.PodSandboxStatusRequest{\n\t\t\tPodSandboxId: podID,\n\t\t\tVerbose:      true,\n\t\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn cache.ParsePodStatus(response)\n}\n\n// RunPod intercepts CRI requests for Pod creation.\nfunc (m *resmgr) RunPod(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\treply, rqerr := handler(ctx, request)\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to create pod: %v\", method, rqerr)\n\t\treturn reply, rqerr\n\t}\n\n\tpodID := reply.(*criv1.RunPodSandboxResponse).PodSandboxId\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tpod, err := m.cache.InsertPod(podID, request, nil)\n\tif err != nil {\n\t\tm.Error(\"%s: failed to insert new pod to cache: %v\", method, err)\n\t\treturn nil, resmgrError(\"%s: failed to insert new pod to cache: %v\", method, err)\n\t}\n\tm.updateIntrospection()\n\n\t// search for any lingering old version and clean up if found\n\treleased := false\n\tdel := []cache.Container{}\n\tfor _, p := range m.cache.GetPods() {\n\t\tif p.GetUID() != pod.GetUID() || p == pod {\n\t\t\tcontinue\n\t\t}\n\t\tm.Warn(\"re-creation of pod %s, releasing old one\", p.GetName())\n\t\tfor _, c := range pod.GetInitContainers() {\n\t\t\tm.Info(\"%s: removing stale init-container %s...\", method, c.PrettyName())\n\t\t\tm.policy.ReleaseResources(c)\n\t\t\tc.UpdateState(cache.ContainerStateStale)\n\t\t\treleased = true\n\t\t\tdel = append(del, c)\n\t\t}\n\t\tfor _, c := range pod.GetContainers() {\n\t\t\tm.Info(\"%s: removing stale container %s...\", method, c.PrettyName())\n\t\t\tm.policy.ReleaseResources(c)\n\t\t\tc.UpdateState(cache.ContainerStateStale)\n\t\t\treleased = true\n\t\t\tdel = append(del, c)\n\t\t}\n\t\tm.cache.DeletePod(p.GetID())\n\t}\n\tif released {\n\t\tif err := m.runPostReleaseHooks(ctx, method, del...); err != nil {\n\t\t\tm.Error(\"%s: failed to run post-release hooks for lingering pod %s: %v\",\n\t\t\t\tmethod, pod.GetName(), err)\n\t\t}\n\t}\n\n\tm.Info(\"created pod %s (%s)\", pod.GetName(), podID)\n\n\treturn reply, nil\n}\n\n// StopPod intercepts CRI requests for stopping Pods.\nfunc (m *resmgr) StopPod(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\treply, rqerr := handler(ctx, request)\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tpodID := request.(*criv1.StopPodSandboxRequest).PodSandboxId\n\tpod, ok := m.cache.LookupPod(podID)\n\n\tif !ok {\n\t\tm.Warn(\"%s: failed to look up pod %s, just passing request through\", method, podID)\n\t\treturn reply, rqerr\n\t}\n\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to stop pod %s: %v\", method, podID, rqerr)\n\t\treturn reply, rqerr\n\t}\n\n\tm.Info(\"%s: stopped pod %s (%s)...\", method, pod.GetName(), podID)\n\n\treleased := []cache.Container{}\n\tfor _, c := range pod.GetInitContainers() {\n\t\tm.Info(\"%s: releasing resources for %s...\", method, c.PrettyName())\n\t\tif err := m.policy.ReleaseResources(c); err != nil {\n\t\t\tm.Warn(\"%s: failed to release init-container %s: %v\", method, c.PrettyName(), err)\n\t\t}\n\t\tc.UpdateState(cache.ContainerStateExited)\n\t\treleased = append(released, c)\n\t}\n\tfor _, c := range pod.GetContainers() {\n\t\tm.Info(\"%s: releasing resources for container %s...\", method, c.PrettyName())\n\t\tif err := m.policy.ReleaseResources(c); err != nil {\n\t\t\tm.Warn(\"%s: failed to release container %s: %v\", method, c.PrettyName(), err)\n\t\t}\n\t\tc.UpdateState(cache.ContainerStateExited)\n\t\treleased = append(released, c)\n\t}\n\n\tif err := m.runPostReleaseHooks(ctx, method, released...); err != nil {\n\t\tm.Error(\"%s: failed to run post-release hooks for pod %s: %v\",\n\t\t\tmethod, pod.GetName(), err)\n\t}\n\n\tm.updateIntrospection()\n\n\treturn reply, rqerr\n}\n\n// RemovePod intercepts CRI requests for Pod removal.\nfunc (m *resmgr) RemovePod(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\treply, rqerr := handler(ctx, request)\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tpodID := request.(*criv1.RemovePodSandboxRequest).PodSandboxId\n\tpod, ok := m.cache.LookupPod(podID)\n\n\tif !ok {\n\t\tm.Warn(\"%s: failed to look up pod %s, just passing request through\", method, podID)\n\t\treturn reply, rqerr\n\t}\n\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to remove pod %s: %v\", method, podID, rqerr)\n\t} else {\n\t\tm.Info(\"%s: removed pod %s (%s)...\", method, pod.GetName(), podID)\n\t}\n\n\treleased := []cache.Container{}\n\tfor _, c := range pod.GetInitContainers() {\n\t\tm.Info(\"%s: removing stale init-container %s...\", method, c.PrettyName())\n\t\tif err := m.policy.ReleaseResources(c); err != nil {\n\t\t\tm.Warn(\"%s: failed to release init-container %s: %v\", method, c.PrettyName(), err)\n\t\t}\n\t\tc.UpdateState(cache.ContainerStateStale)\n\t\treleased = append(released, c)\n\t}\n\tfor _, c := range pod.GetContainers() {\n\t\tm.Info(\"%s: removing stale container %s...\", method, c.PrettyName())\n\t\tif err := m.policy.ReleaseResources(c); err != nil {\n\t\t\tm.Warn(\"%s: failed to release container %s: %v\", method, c.PrettyName(), err)\n\t\t}\n\t\tc.UpdateState(cache.ContainerStateStale)\n\t\treleased = append(released, c)\n\t}\n\n\tif err := m.runPostReleaseHooks(ctx, method, released...); err != nil {\n\t\tm.Error(\"%s: failed to run post-release hooks for pod %s: %v\",\n\t\t\tmethod, pod.GetName(), err)\n\t}\n\n\tm.cache.DeletePod(podID)\n\tm.updateIntrospection()\n\n\treturn reply, rqerr\n}\n\n// CreateContainer intercepts CRI requests for Container creation.\nfunc (m *resmgr) CreateContainer(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\t// kubelet doesn't always clean up crashed containers so we try doing it here\n\tif msg, ok := request.(*criv1.CreateContainerRequest); ok {\n\t\tif pod, ok := m.cache.LookupPod(msg.PodSandboxId); ok {\n\t\t\tif msg.Config != nil && msg.Config.Metadata != nil {\n\t\t\t\tif c, ok := pod.GetContainer(msg.Config.Metadata.Name); ok {\n\t\t\t\t\tm.Warn(\"re-creation of container %s, releasing old one\", c.PrettyName())\n\t\t\t\t\tm.policy.ReleaseResources(c)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tcontainer, err := m.cache.InsertContainer(request)\n\tif err != nil {\n\t\tm.Error(\"%s: failed to insert new container to cache: %v\", method, err)\n\t\treturn nil, resmgrError(\"%s: failed to insert new container to cache: %v\", method, err)\n\t}\n\n\tcontainer.SetCRIRequest(request)\n\n\tm.Info(\"%s: creating container %s...\", method, container.PrettyName())\n\n\tif err := m.policy.AllocateResources(container); err != nil {\n\t\tm.Error(\"%s: failed to allocate resources for container %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t\tm.cache.DeleteContainer(container.GetCacheID())\n\t\treturn nil, resmgrError(\"failed to allocate container resources: %v\", err)\n\t}\n\n\tcontainer.InsertMount(&cache.Mount{\n\t\tContainer:   \"/.cri-resmgr\",\n\t\tHost:        m.cache.ContainerDirectory(container.GetCacheID()),\n\t\tReadonly:    true,\n\t\tPropagation: cache.MountHostToContainer,\n\t})\n\n\tif err := m.runPostAllocateHooks(ctx, method); err != nil {\n\t\tm.Error(\"%s: failed to run post-allocate hooks for %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t\tm.policy.ReleaseResources(container)\n\t\tm.runPostReleaseHooks(ctx, method, container)\n\t\tm.cache.DeleteContainer(container.GetCacheID())\n\t\treturn nil, resmgrError(\"failed to allocate container resources: %v\", err)\n\t}\n\n\tcontainer.ClearCRIRequest()\n\treply, rqerr := handler(ctx, request)\n\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to create container %s: %v\", method, container.PrettyName(), rqerr)\n\t\tm.policy.ReleaseResources(container)\n\t\tm.runPostReleaseHooks(ctx, method, container)\n\t\tm.cache.DeleteContainer(container.GetCacheID())\n\t\treturn nil, resmgrError(\"failed to create container: %v\", rqerr)\n\t}\n\n\tm.cache.UpdateContainerID(container.GetCacheID(), reply)\n\tcontainer.UpdateState(cache.ContainerStateCreated)\n\tm.updateIntrospection()\n\n\treturn reply, nil\n}\n\n// StartContainer intercepts CRI requests for starting Containers.\nfunc (m *resmgr) StartContainer(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tcontainerID := request.(*criv1.StartContainerRequest).ContainerId\n\tcontainer, ok := m.cache.LookupContainer(containerID)\n\n\tif !ok {\n\t\tm.Warn(\"%s: failed to look up container %s, just passing request through\",\n\t\t\tmethod, containerID)\n\t\treturn handler(ctx, request)\n\t}\n\n\tm.Info(\"%s: starting container %s...\", method, container.PrettyName())\n\n\tif container.GetState() != cache.ContainerStateCreated {\n\t\tm.Error(\"%s: refusing to start container %s in unexpected state %v\",\n\t\t\tmethod, container.PrettyName(), container.GetState())\n\t\treturn nil, resmgrError(\"refusing to start container %s in unexpexted state %v\",\n\t\t\tcontainer.PrettyName(), container.GetState())\n\t}\n\n\treply, rqerr := handler(ctx, request)\n\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to start container %s: %v\", method, container.PrettyName(), rqerr)\n\t\treturn nil, rqerr\n\t}\n\n\tcontainer.UpdateState(cache.ContainerStateRunning)\n\n\te := &events.Policy{\n\t\tType:   events.ContainerStarted,\n\t\tSource: \"resource-manager\",\n\t\tData:   container,\n\t}\n\tif _, err := m.policy.HandleEvent(e); err != nil {\n\t\tm.Error(\"%s: policy failed to handle event %s: %v\", method, e.Type, err)\n\t}\n\n\tif err := m.runPostStartHooks(method, container); err != nil {\n\t\tm.Error(\"%s: failed to run post-start hooks for %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t}\n\n\tm.updateIntrospection()\n\n\treturn reply, rqerr\n}\n\n// StopContainer intercepts CRI requests for stopping Containers.\nfunc (m *resmgr) StopContainer(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\treply, rqerr := handler(ctx, request)\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tcontainerID := request.(*criv1.StopContainerRequest).ContainerId\n\tcontainer, ok := m.cache.LookupContainer(containerID)\n\n\tif !ok {\n\t\tm.Warn(\"%s: failed to look up container %s, just passing request through\",\n\t\t\tmethod, containerID)\n\t\treturn reply, rqerr\n\t}\n\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to stop container %s: %v\", method, container.PrettyName(), rqerr)\n\t\treturn reply, rqerr\n\t}\n\n\tm.Info(\"%s: stopped container %s...\", method, container.PrettyName())\n\n\t// Notes:\n\t//   For now, we assume any error replies from CRI are about the container not\n\t//   being found, in which case we still go ahead and finish locally stopping it...\n\n\tif err := m.policy.ReleaseResources(container); err != nil {\n\t\tm.Error(\"%s: failed to release resources for container %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t}\n\n\tcontainer.UpdateState(cache.ContainerStateExited)\n\n\tif err := m.runPostReleaseHooks(ctx, method, container); err != nil {\n\t\tm.Error(\"%s: failed to run post-release hooks for %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t}\n\n\tm.updateIntrospection()\n\n\treturn reply, rqerr\n}\n\n// RemoveContainer intercepts CRI requests for Container removal.\nfunc (m *resmgr) RemoveContainer(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\treply, rqerr := handler(ctx, request)\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tcontainerID := request.(*criv1.RemoveContainerRequest).ContainerId\n\tcontainer, ok := m.cache.LookupContainer(containerID)\n\n\tif !ok {\n\t\tm.Warn(\"%s: failed to look up container %s, just passing request through\",\n\t\t\tmethod, containerID)\n\t\treturn reply, rqerr\n\t}\n\n\tif rqerr != nil {\n\t\tm.Error(\"%s: failed to remove container %s: %v\", method, container.PrettyName(), rqerr)\n\t} else {\n\t\tm.Info(\"%s: removed container %s...\", method, container.PrettyName())\n\t}\n\n\tif err := m.policy.ReleaseResources(container); err != nil {\n\t\tm.Error(\"%s: failed to release resources for container %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t}\n\n\tcontainer.UpdateState(cache.ContainerStateStale)\n\n\tif err := m.runPostReleaseHooks(ctx, method, container); err != nil {\n\t\tm.Error(\"%s: failed to run post-release hooks for %s: %v\",\n\t\t\tmethod, container.PrettyName(), err)\n\t}\n\n\tm.updateIntrospection()\n\n\treturn reply, rqerr\n}\n\n// ListContainers intercepts CRI requests for listing Containers.\nfunc (m *resmgr) ListContainers(ctx context.Context, method string, request interface{},\n\thandler server.Handler) (interface{}, error) {\n\n\treply, rqerr := handler(ctx, request)\n\n\tif rqerr != nil {\n\t\treturn reply, rqerr\n\t}\n\n\tif f := request.(*criv1.ListContainersRequest).Filter; f != nil {\n\t\tif f.Id != \"\" || f.State != nil || f.PodSandboxId != \"\" || len(f.LabelSelector) > 0 {\n\t\t\treturn reply, nil\n\t\t}\n\t}\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tclistmap := map[string]*criv1.Container{}\n\treleased := []cache.Container{}\n\tfor _, listed := range reply.(*criv1.ListContainersResponse).Containers {\n\t\tclistmap[listed.Id] = listed\n\t\tif listed.State != criv1.ContainerState_CONTAINER_EXITED {\n\t\t\tcontinue\n\t\t}\n\t\tif c, ok := m.cache.LookupContainer(listed.Id); ok {\n\t\t\tstate := c.GetState()\n\t\t\tif state == cache.ContainerStateRunning || state == cache.ContainerStateCreated {\n\t\t\t\tm.Info(\"%s: exited, releasing its resources...\", c.PrettyName())\n\t\t\t\tif err := m.policy.ReleaseResources(c); err != nil {\n\t\t\t\t\tm.Error(\"%s: failed to release resources for container %s: %v\",\n\t\t\t\t\t\tmethod, c.PrettyName(), err)\n\t\t\t\t}\n\t\t\t\tc.UpdateState(cache.ContainerStateExited)\n\t\t\t\treleased = append(released, c)\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, c := range m.cache.GetContainers() {\n\t\tif c.GetState() == cache.ContainerStateRunning {\n\t\t\tif _, ok := clistmap[c.GetID()]; !ok {\n\t\t\t\tm.Info(\"%s: absent from runtime, releasing its resources...\", c.PrettyName())\n\t\t\t\tif err := m.policy.ReleaseResources(c); err != nil {\n\t\t\t\t\tm.Error(\"%s: failed to release resources for container %s: %v\",\n\t\t\t\t\t\tmethod, c.PrettyName(), err)\n\t\t\t\t}\n\t\t\t\tc.UpdateState(cache.ContainerStateStale)\n\t\t\t\treleased = append(released, c)\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(released) > 0 {\n\t\tif err := m.runPostReleaseHooks(ctx, method, released...); err != nil {\n\t\t\tm.Error(\"%s: failed to run post-release hooks: %v\",\n\t\t\t\tmethod, err)\n\t\t}\n\t}\n\tm.updateIntrospection()\n\n\treturn reply, nil\n}\n\n// UpdateContainer intercepts CRI requests for updating Containers.\nfunc (m *resmgr) UpdateContainer(_ context.Context, _ string, _ interface{},\n\t_ server.Handler) (interface{}, error) {\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\t//\n\t// Notes:\n\t//   Once VPA is fully implemented, we need to start passing these\n\t//   requests on to the active policy.\n\t//\n\t// containerID := request.(*criv1.UpdateContainerResourcesRequest).ContainerId\n\t// container, ok := m.cache.LookupContainer(containerID)\n\t// if !ok {\n\t//     m.Warn(\"%s: failed to look up container %s, just passing request through\",\n\t//         method, containerID)\n\t//     return handler(ctx, request)\n\t// }\n\t//\n\t// err := m.policy.UpdateResources(container)\n\t// if err != nil {\n\t//     m.Error(\"%s: failed to update resources of container %s: %v\", method, containerID, err)\n\t//     return nil, err\n\t// }\n\t//\n\t// err := m.runPostUpdateHooks(ctx, method)\n\t// if err != nil {\n\t//     m.Warn(\"%s: failed to run post-update hooks for update of container %s: %v\",\n\t//         method, containerID, err)\n\t// }\n\t//\n\t// return &criv1.UpdateContainerResourcesResponse{}, nil\n\t//\n\n\tif !m.warnedCRIUpdate {\n\t\tm.Warn(\"CRI UpdateContainerResources request received. Unless Vertical\")\n\t\tm.Warn(\"Pod Autoscaling is fully implemented, this usually indicates that\")\n\t\tm.Warn(\"kubelet is running with CPU Manager enabled and 'static' or some\")\n\t\tm.Warn(\"other than 'none' policy active. This does not make much sense when\")\n\t\tm.Warn(\"CRI Resource Manager is also active and on the kubelet-runtime\")\n\t\tm.Warn(\"signalling path. Please consider disabling CPU Manager or setting\")\n\t\tm.Warn(\"its active policy to 'none'.\")\n\t\tm.warnedCRIUpdate = true\n\t}\n\n\treturn &criv1.UpdateContainerResourcesResponse{}, nil\n}\n\n// RebalanceContainers tries to find a more optimal container resource allocation if necessary.\nfunc (m *resmgr) RebalanceContainers() error {\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tm.Info(\"rebalancing (reallocating) containers...\")\n\n\treturn m.rebalance(\"Rebalance\")\n}\n\n// rebalance triggers a policy-specific rebalancing cycle of containers.\nfunc (m *resmgr) rebalance(method string) error {\n\tif m.policy == nil {\n\t\treturn nil\n\t}\n\n\tchanges, err := m.policy.Rebalance()\n\n\tif err != nil {\n\t\tm.Error(\"%s: rebalancing of containers failed: %v\", method, err)\n\t}\n\n\tif changes {\n\t\tif err := m.runPostUpdateHooks(context.Background(), method); err != nil {\n\t\t\tm.Error(\"%s: failed to run post-update hooks: %v\", method, err)\n\t\t\treturn resmgrError(\"%s: failed to run post-update hooks: %v\", method, err)\n\t\t}\n\t}\n\n\treturn m.cache.Save()\n}\n\n// DeliverPolicyEvent delivers a policy-specific event to the active policy.\nfunc (m *resmgr) DeliverPolicyEvent(e *events.Policy) error {\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tif m.policy == nil {\n\t\treturn nil\n\t}\n\n\tif e.Source == \"\" {\n\t\te.Source = \"unspecified\"\n\t}\n\n\tm.Info(\"delivering policy event %s.%s...\", e.Source, e.Type)\n\n\tmethod := \"DeliverPolicyEvent\"\n\tchanges, err := m.policy.HandleEvent(e)\n\n\tif err != nil {\n\t\tm.Error(\"%s: handling event %s.%s failed: %v\", method, e.Source, e.Type, err)\n\t\treturn err\n\t}\n\n\tif changes {\n\t\tif err = m.runPostUpdateHooks(context.Background(), method); err != nil {\n\t\t\tm.Error(\"%s: failed to run post-update hooks: %v\", method, err)\n\t\t\treturn resmgrError(\"%s: failed to run post-update hooks: %v\", method, err)\n\t\t}\n\t}\n\n\tm.cache.Save()\n\treturn nil\n}\n\n// setConfig activates a new configuration, either from the agent or from a file.\nfunc (m *resmgr) setConfig(v interface{}) error {\n\tvar err error\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tswitch cfg := v.(type) {\n\tcase *config.RawConfig:\n\t\terr = pkgcfg.SetConfig(cfg.Data)\n\tcase string:\n\t\terr = pkgcfg.SetConfigFromFile(cfg)\n\tdefault:\n\t\terr = fmt.Errorf(\"invalid configuration source/type %T\", v)\n\t}\n\tif err != nil {\n\t\tm.Error(\"configuration rejected: %v\", err)\n\t\treturn resmgrError(\"configuration rejected: %v\", err)\n\t}\n\n\tif m.policy != nil {\n\t\t// synchronize state of controllers with new configuration\n\t\tif err = m.control.StartStopControllers(m.cache, m.relay.Client()); err != nil {\n\t\t\tm.Error(\"failed to synchronize controllers with new configuration: %v\", err)\n\t\t\treturn resmgrError(\"failed to synchronize controllers with new configuration: %v\", err)\n\t\t}\n\n\t\tif err = m.runPostUpdateHooks(context.Background(), \"setConfig\"); err != nil {\n\t\t\tm.Error(\"failed to run post-update hooks after reconfiguration: %v\", err)\n\t\t\treturn resmgrError(\"failed to run post-update hooks after reconfiguration: %v\", err)\n\t\t}\n\t}\n\n\t// if we managed to activate a configuration from the agent, store it in the cache\n\tif cfg, ok := v.(*config.RawConfig); ok {\n\t\tm.cache.SetConfig(cfg)\n\t}\n\n\tm.Info(\"successfully switched to new configuration\")\n\n\treturn nil\n}\n\n// runPostAllocateHooks runs the necessary hooks after allocating resources for some containers.\nfunc (m *resmgr) runPostAllocateHooks(ctx context.Context, method string) error {\n\tfor _, c := range m.cache.GetPendingContainers() {\n\t\tswitch c.GetState() {\n\t\tcase cache.ContainerStateRunning, cache.ContainerStateCreated:\n\t\t\tif err := m.control.RunPostUpdateHooks(c); err != nil {\n\t\t\t\tm.Warn(\"%s post-update hook failed for %s: %v\",\n\t\t\t\t\tmethod, c.PrettyName(), err)\n\t\t\t}\n\t\t\tif req, ok := c.ClearCRIRequest(); ok {\n\t\t\t\tif _, err := m.sendCRIRequest(ctx, req); err != nil {\n\t\t\t\t\tm.Warn(\"%s update of container %s failed: %v\",\n\t\t\t\t\t\tmethod, c.PrettyName(), err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tm.policy.ExportResourceData(c)\n\t\tcase cache.ContainerStateCreating:\n\t\t\tif err := m.control.RunPreCreateHooks(c); err != nil {\n\t\t\t\tm.Warn(\"%s pre-create hook failed for %s: %v\",\n\t\t\t\t\tmethod, c.PrettyName(), err)\n\t\t\t}\n\t\t\tm.policy.ExportResourceData(c)\n\t\tdefault:\n\t\t\tm.Warn(\"%s: skipping container %s (in state %v)\", method,\n\t\t\t\tc.PrettyName(), c.GetState())\n\t\t}\n\t}\n\treturn nil\n}\n\n// runPostStartHooks runs the necessary hooks after having started a container.\nfunc (m *resmgr) runPostStartHooks(method string, c cache.Container) error {\n\tif err := m.control.RunPostStartHooks(c); err != nil {\n\t\tm.Error(\"%s: post-start hook failed for %s: %v\", method, c.PrettyName(), err)\n\t}\n\treturn nil\n}\n\n// runPostReleaseHooks runs the necessary hooks after releaseing resources of some containers\nfunc (m *resmgr) runPostReleaseHooks(ctx context.Context, method string, released ...cache.Container) error {\n\tfor _, c := range released {\n\t\tif err := m.control.RunPostStopHooks(c); err != nil {\n\t\t\tm.Warn(\"post-stop hook failed for %s: %v\", c.PrettyName(), err)\n\t\t}\n\t\tif c.GetState() == cache.ContainerStateStale {\n\t\t\tm.cache.DeleteContainer(c.GetCacheID())\n\t\t}\n\t}\n\tfor _, c := range m.cache.GetPendingContainers() {\n\t\tswitch state := c.GetState(); state {\n\t\tcase cache.ContainerStateStale, cache.ContainerStateExited:\n\t\t\tif err := m.control.RunPostStopHooks(c); err != nil {\n\t\t\t\tm.Warn(\"post-stop hook failed for %s: %v\", c.PrettyName(), err)\n\t\t\t}\n\t\t\tif state == cache.ContainerStateStale {\n\t\t\t\tm.cache.DeleteContainer(c.GetCacheID())\n\t\t\t}\n\t\tcase cache.ContainerStateRunning, cache.ContainerStateCreated:\n\t\t\tif err := m.control.RunPostUpdateHooks(c); err != nil {\n\t\t\t\tm.Warn(\"post-update hook failed for %s: %v\", c.PrettyName(), err)\n\t\t\t}\n\t\t\tif req, ok := c.ClearCRIRequest(); ok {\n\t\t\t\tif _, err := m.sendCRIRequest(ctx, req); err != nil {\n\t\t\t\t\tm.Warn(\"update of container %s failed: %v\", c.PrettyName(), err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tm.policy.ExportResourceData(c)\n\t\tdefault:\n\t\t\tm.Warn(\"%s: skipping pending container %s (in state %v)\",\n\t\t\t\tmethod, c.PrettyName(), c.GetState())\n\t\t}\n\t}\n\treturn nil\n}\n\n// runPostUpdateHooks runs the necessary hooks after reconcilation.\nfunc (m *resmgr) runPostUpdateHooks(ctx context.Context, method string) error {\n\tfor _, c := range m.cache.GetPendingContainers() {\n\t\tswitch c.GetState() {\n\t\tcase cache.ContainerStateRunning, cache.ContainerStateCreated:\n\t\t\tif err := m.control.RunPostUpdateHooks(c); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif req, ok := c.GetCRIRequest(); ok {\n\t\t\t\tif _, err := m.sendCRIRequest(ctx, req); err != nil {\n\t\t\t\t\tm.Warn(\"%s update of container %s failed: %v\",\n\t\t\t\t\t\tmethod, c.PrettyName(), err)\n\t\t\t\t} else {\n\t\t\t\t\tc.ClearCRIRequest()\n\t\t\t\t}\n\t\t\t}\n\t\t\tm.policy.ExportResourceData(c)\n\t\tdefault:\n\t\t\tm.Warn(\"%s: skipping container %s (in state %v)\", method,\n\t\t\t\tc.PrettyName(), c.GetState())\n\t\t}\n\t}\n\treturn nil\n}\n\n// sendCRIRequest sends the given CRI request, returning the received reply and error.\nfunc (m *resmgr) sendCRIRequest(ctx context.Context, request interface{}) (interface{}, error) {\n\tclient := m.relay.Client()\n\tswitch request.(type) {\n\tcase *criv1.UpdateContainerResourcesRequest:\n\t\treq := request.(*criv1.UpdateContainerResourcesRequest)\n\t\tm.Debug(\"sending update request for container %s...\", req.ContainerId)\n\t\treturn client.UpdateContainerResources(ctx, req)\n\tdefault:\n\t\treturn nil, resmgrError(\"sendCRIRequest: unhandled request type %T\", request)\n\t}\n}\n\nfunc (m *resmgr) checkRuntime(ctx context.Context) error {\n\tversion, err := m.relay.Client().Version(ctx, &criv1.VersionRequest{\n\t\tVersion: kubeAPIVersion,\n\t})\n\tif err != nil {\n\t\treturn resmgrError(\"failed to query runtime version: %v\", err)\n\t}\n\n\tfor _, name := range knownRuntimes {\n\t\tif strings.HasPrefix(version.RuntimeName, name) {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tif opt.AllowUntestedRuntimes {\n\t\tm.Warnf(\"running with untested/unknown runtime %q\", version.RuntimeName)\n\t\treturn nil\n\t}\n\n\treturn rejectRuntimeError(version.RuntimeName)\n}\n\nfunc rejectRuntimeError(name string) error {\n\treturn resmgrError(\"rejecting untested runtime %s, use --%s to allow it\",\n\t\tname, allowUntestedRuntimesFlag)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/resource-manager.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage resmgr\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"os/signal\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"golang.org/x/sys/unix\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/relay\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/agent\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\tconfig \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/control\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/introspect\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/metrics\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/visualizer\"\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/pidfile\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n\t\"github.com/intel/cri-resource-manager/pkg/topology\"\n\n\tpolicyCollector \"github.com/intel/cri-resource-manager/pkg/policycollector\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n)\n\n// ResourceManager is the interface we expose for controlling the CRI resource manager.\ntype ResourceManager interface {\n\t// Start starts the resource manager.\n\tStart() error\n\t// Stop stops the resource manager.\n\tStop()\n\t// SetConfig dynamically updates the resource manager configuration.\n\tSetConfig(*config.RawConfig) error\n\t// SetAdjustment dynamically updates external adjustments.\n\tSetAdjustment(*config.Adjustment) map[string]error\n\t// SendEvent sends an event to be processed by the resource manager.\n\tSendEvent(event interface{}) error\n\t// Add-ons for testing.\n\tResourceManagerTestAPI\n}\n\n// resmgr is the implementation of ResourceManager.\ntype resmgr struct {\n\tlogger.Logger\n\tsync.RWMutex\n\trelay        relay.Relay        // our CRI relay\n\tcache        cache.Cache        // cached state\n\tpolicy       policy.Policy      // resource manager policy\n\tpolicySwitch bool               // active policy is being switched\n\tconfigServer config.Server      // configuration management server\n\tcontrol      control.Control    // policy controllers/enforcement\n\tagent        agent.Interface    // connection to cri-resmgr agent\n\tconf         *config.RawConfig  // pending for saving in cache\n\tmetrics      *metrics.Metrics   // metrics collector/pre-processor\n\tevents       chan interface{}   // channel for delivering events\n\tstop         chan interface{}   // channel for signalling shutdown to goroutines\n\tsignals      chan os.Signal     // signal channel\n\tintrospect   *introspect.Server // server for external introspection\n\n\twarnedCRIUpdate bool // warned about CRI UpdateContainer calls\n}\n\n// NewResourceManager creates a new ResourceManager instance.\nfunc NewResourceManager() (ResourceManager, error) {\n\tm := &resmgr{Logger: logger.NewLogger(\"resource-manager\")}\n\n\tif err := m.setupCache(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tsysfs.SetSysRoot(opt.HostRoot)\n\ttopology.SetSysRoot(opt.HostRoot)\n\n\tswitch {\n\tcase opt.ResetPolicy && opt.ResetConfig:\n\t\tos.Exit(m.resetCachedPolicy() + m.resetCachedConfig())\n\tcase opt.ResetPolicy:\n\t\tos.Exit(m.resetCachedPolicy())\n\tcase opt.ResetConfig:\n\t\tos.Exit(m.resetCachedConfig())\n\t}\n\n\tif err := m.checkOpts(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupAgentInterface(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.loadConfig(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupConfigServer(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupPolicy(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.registerPolicyMetricsCollector(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupRelay(); err != nil {\n\t\tpid, _ := pidfile.OwnerPid()\n\t\tif pid > 0 {\n\t\t\tm.Error(\"looks like we're already running as pid %d...\", pid)\n\t\t}\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupRequestProcessing(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupEventProcessing(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupControllers(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := m.setupIntrospection(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn m, nil\n}\n\n// Start starts the resource manager.\nfunc (m *resmgr) Start() error {\n\tm.Info(\"starting...\")\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tif err := m.checkRuntime(context.Background()); err != nil {\n\t\treturn err\n\t}\n\n\tif err := m.startControllers(); err != nil {\n\t\treturn err\n\t}\n\n\tif err := m.startRequestProcessing(); err != nil {\n\t\treturn err\n\t}\n\n\tif err := m.startEventProcessing(); err != nil {\n\t\treturn err\n\t}\n\n\tm.startIntrospection()\n\n\tif err := m.relay.Start(); err != nil {\n\t\treturn resmgrError(\"failed to start CRI relay: %v\", err)\n\t}\n\n\tif err := pidfile.Remove(); err != nil {\n\t\treturn resmgrError(\"failed to remove stale/old PID file: %v\", err)\n\t}\n\tif err := pidfile.Write(); err != nil {\n\t\treturn resmgrError(\"failed to write PID file: %v\", err)\n\t}\n\n\tif opt.ForceConfig == \"\" {\n\t\tif err := m.configServer.Start(opt.ConfigSocket); err != nil {\n\t\t\treturn resmgrError(\"failed to start configuration server: %v\", err)\n\t\t}\n\n\t\t// We never store a forced configuration in the cache. However, if we're not\n\t\t// running with a forced configuration, and the configuration is pending to\n\t\t// get stored in the cache (IOW, it is a new one acquired from an agent), then\n\t\t// then store it in the cache now.\n\t\tif m.conf != nil {\n\t\t\tm.cache.SetConfig(m.conf)\n\t\t\tm.conf = nil\n\t\t}\n\t}\n\n\tm.Info(\"up and running\")\n\n\treturn nil\n}\n\n// Stop stops the resource manager.\nfunc (m *resmgr) Stop() {\n\tm.Info(\"shutting down...\")\n\n\tm.Lock()\n\tdefer m.Unlock()\n\n\tif m.signals != nil {\n\t\tclose(m.signals)\n\t\tm.signals = nil\n\t}\n\n\tm.configServer.Stop()\n\tm.relay.Stop()\n\tm.stopIntrospection()\n\tm.stopEventProcessing()\n}\n\n// SetConfig pushes new configuration to the resource manager.\nfunc (m *resmgr) SetConfig(conf *config.RawConfig) error {\n\tm.Info(\"applying new configuration from agent...\")\n\treturn m.setConfig(conf)\n}\n\n// SetAdjustment pushes new external adjustments to the resource manager.\nfunc (m *resmgr) SetAdjustment(adjustment *config.Adjustment) map[string]error {\n\tm.Info(\"applying new adjustments from agent...\")\n\n\tm.Lock()\n\tdefer m.Unlock()\n\treturn m.setAdjustment(adjustment)\n}\n\n// setConfigFromFile pushes new configuration to the resource manager from a file.\nfunc (m *resmgr) setConfigFromFile(path string) error {\n\tm.Info(\"applying new configuration from file %s...\", path)\n\treturn m.setConfig(path)\n}\n\n// setAdjustments pushes new external policies to the resource manager.\nfunc (m *resmgr) setAdjustment(adjustments *config.Adjustment) map[string]error {\n\tm.Info(\"applying new external adjustments from agent...\")\n\n\trebalance, errors := m.cache.SetAdjustment(adjustments)\n\tif rebalance {\n\t\tm.rebalance(\"setAdjustment\")\n\t}\n\n\treturn errors\n}\n\n// resetCachedPolicy resets the cached active policy and all of its data.\nfunc (m *resmgr) resetCachedPolicy() int {\n\tm.Info(\"resetting active policy stored in cache...\")\n\tdefer logger.Flush()\n\n\tif ls, err := utils.IsListeningSocket(opt.RelaySocket); ls || err != nil {\n\t\tm.Error(\"refusing to reset, looks like an instance of %q is active at socket %q...\",\n\t\t\tfilepath.Base(os.Args[0]), opt.RelaySocket)\n\t\treturn 1\n\t}\n\n\tif err := m.cache.ResetActivePolicy(); err != nil {\n\t\tm.Error(\"failed to reset active policy: %v\", err)\n\t\treturn 1\n\t}\n\treturn 0\n}\n\n// resetCachedConfig resets any cached configuration.\nfunc (m *resmgr) resetCachedConfig() int {\n\tm.Info(\"resetting cached configuration...\")\n\tdefer logger.Flush()\n\n\tif ls, err := utils.IsListeningSocket(opt.RelaySocket); ls || err != nil {\n\t\tm.Error(\"refusing to reset, looks like an instance of %q is active at socket %q...\",\n\t\t\tfilepath.Base(os.Args[0]), opt.RelaySocket)\n\t\treturn 1\n\t}\n\n\tif err := m.cache.ResetConfig(); err != nil {\n\t\tm.Error(\"failed to reset cached configuration: %v\", err)\n\t\treturn 1\n\t}\n\treturn 0\n}\n\n// setupCache creates a cache and reloads its last saved state if found.\nfunc (m *resmgr) setupCache() error {\n\tvar err error\n\n\toptions := cache.Options{CacheDir: opt.RelayDir}\n\tif m.cache, err = cache.NewCache(options); err != nil {\n\t\treturn resmgrError(\"failed to create cache: %v\", err)\n\t}\n\n\treturn nil\n\n}\n\n// setupAgentInterface sets up the connection to the node agent.\nfunc (m *resmgr) setupAgentInterface() error {\n\tvar err error\n\n\tif m.agent, err = agent.NewAgentInterface(opt.AgentSocket); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// setupConfigServer sets up our configuration server for agent notifications.\nfunc (m *resmgr) setupConfigServer() error {\n\tvar err error\n\n\tif m.configServer, err = config.NewConfigServer(m.SetConfig, m.SetAdjustment); err != nil {\n\t\treturn resmgrError(\"failed to create configuration notification server: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// checkOpts checks the command line options for obvious errors.\nfunc (m *resmgr) checkOpts() error {\n\tif opt.ForceConfig != \"\" && opt.FallbackConfig != \"\" {\n\t\treturn resmgrError(\"both fallback (%s) and forced (%s) configurations given\",\n\t\t\topt.FallbackConfig, opt.ForceConfig)\n\t}\n\n\treturn nil\n}\n\n// loadConfig tries to pick and load (initial) configuration from a number of sources.\nfunc (m *resmgr) loadConfig() error {\n\t//\n\t// We try to load initial configuration from a number of sources:\n\t//\n\t//    1. use forced configuration file if we were given one\n\t//    2. use last configuration stored in cache, if we have one and it applies\n\t//    3. use fallback configuration file if we were given one\n\t//    4. use empty/builtin default configuration, whatever that is...\n\t//\n\n\tif opt.ForceConfig != \"\" {\n\t\tm.Info(\"using forced configuration %s...\", opt.ForceConfig)\n\t\tif err := pkgcfg.SetConfigFromFile(opt.ForceConfig); err != nil {\n\t\t\treturn resmgrError(\"failed to load forced configuration %s: %v\",\n\t\t\t\topt.ForceConfig, err)\n\t\t}\n\t\treturn m.setupConfigSignal(opt.ForceConfigSignal)\n\t}\n\n\tm.Info(\"trying last cached configuration...\")\n\tif conf := m.cache.GetConfig(); conf != nil {\n\t\terr := pkgcfg.SetConfig(conf.Data)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\t\tm.Error(\"failed to activate cached configuration: %v\", err)\n\t}\n\n\tif opt.FallbackConfig != \"\" {\n\t\tm.Info(\"using fallback configuration %s...\", opt.FallbackConfig)\n\t\tif err := pkgcfg.SetConfigFromFile(opt.FallbackConfig); err != nil {\n\t\t\treturn resmgrError(\"failed to load fallback configuration %s: %v\",\n\t\t\t\topt.FallbackConfig, err)\n\t\t}\n\t\treturn nil\n\t}\n\n\tm.Warn(\"no initial configuration found\")\n\treturn nil\n}\n\n// setupConfigSignal sets up a signal handler for reloading forced configuration.\nfunc (m *resmgr) setupConfigSignal(signame string) error {\n\tif signame == \"\" || strings.HasPrefix(strings.ToLower(signame), \"disable\") {\n\t\treturn nil\n\t}\n\n\tm.Info(\"setting up signal %s to reload forced configuration\", signame)\n\n\tsig := unix.SignalNum(signame)\n\tif int(sig) == 0 {\n\t\treturn resmgrError(\"invalid forced configuration reload signal '%s'\", signame)\n\t}\n\n\tm.signals = make(chan os.Signal, 1)\n\tsignal.Notify(m.signals, sig)\n\n\tgo func(signals <-chan os.Signal) {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase _, ok := <-signals:\n\t\t\t\tif !ok {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tm.Info(\"reloading forced configuration %s...\", opt.ForceConfig)\n\n\t\t\tif err := m.setConfigFromFile(opt.ForceConfig); err != nil {\n\t\t\t\tm.Error(\"failed to reload forced configuration %s: %v\",\n\t\t\t\t\topt.ForceConfig, err)\n\t\t\t}\n\t\t}\n\t}(m.signals)\n\n\treturn nil\n}\n\n// setupPolicy sets up policy with the configured/active backend\nfunc (m *resmgr) setupPolicy() error {\n\tvar err error\n\n\tactive := policy.ActivePolicy()\n\tcached := m.cache.GetActivePolicy()\n\n\tif active != cached {\n\t\tif cached != \"\" {\n\t\t\tif opt.DisablePolicySwitch {\n\t\t\t\tm.Error(\"can't switch policy from %q to %q: policy switching disabled\",\n\t\t\t\t\tcached, active)\n\t\t\t\treturn resmgrError(\"cannot load cache with policy %s for active policy %s\",\n\t\t\t\t\tcached, active)\n\t\t\t}\n\t\t\tif err := m.cache.ResetActivePolicy(); err != nil {\n\t\t\t\treturn resmgrError(\"failed to reset cached policy %q: %v\", cached, err)\n\t\t\t}\n\t\t}\n\t\tm.cache.SetActivePolicy(active)\n\t\tm.policySwitch = true\n\t}\n\n\toptions := &policy.Options{AgentCli: m.agent, SendEvent: m.SendEvent}\n\tif m.policy, err = policy.NewPolicy(m.cache, options); err != nil {\n\t\treturn resmgrError(\"failed to create policy %s: %v\", active, err)\n\t}\n\n\treturn nil\n}\n\n// setupRelay sets up the CRI request relay.\nfunc (m *resmgr) setupRelay() error {\n\tvar err error\n\n\toptions := relay.Options{\n\t\tRelaySocket:   opt.RelaySocket,\n\t\tImageSocket:   opt.ImageSocket,\n\t\tRuntimeSocket: opt.RuntimeSocket,\n\t\tQualifyReqFn:  m.disambiguate,\n\t}\n\n\toptions.ImageSocket = strings.TrimPrefix(options.ImageSocket, \"unix://\")\n\toptions.RuntimeSocket = strings.TrimPrefix(options.RuntimeSocket, \"unix://\")\n\toptions.RelaySocket = strings.TrimPrefix(options.RelaySocket, \"unix://\")\n\n\tif m.relay, err = relay.NewRelay(options); err != nil {\n\t\treturn resmgrError(\"failed to create CRI relay: %v\", err)\n\t}\n\n\tif err = m.relay.Setup(); err != nil {\n\t\treturn resmgrError(\"failed to create CRI relay: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// setupControllers sets up the resource controllers.\nfunc (m *resmgr) setupControllers() error {\n\tvar err error\n\n\tif m.control, err = control.NewControl(); err != nil {\n\t\treturn resmgrError(\"failed to create resource controller: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// startControllers start the resource controllers.\nfunc (m *resmgr) startControllers() error {\n\tif err := m.control.StartStopControllers(m.cache, m.relay.Client()); err != nil {\n\t\treturn resmgrError(\"failed to start resource controllers: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// setupIntrospection prepares the resource manager for serving external introspection requests.\nfunc (m *resmgr) setupIntrospection() error {\n\tmux := instrumentation.GetHTTPMux()\n\n\ti, err := introspect.Setup(mux, m.policy.Introspect())\n\tif err != nil {\n\t\treturn resmgrError(\"failed to set up introspection service: %v\", err)\n\t}\n\tm.introspect = i\n\n\tif !opt.DisableUI {\n\t\tif err := visualizer.Setup(mux); err != nil {\n\t\t\tm.Error(\"failed to set up UI for visualization: %v\", err)\n\t\t}\n\t} else {\n\t\tm.Warn(\"built-in visualization UIs are disabled\")\n\t}\n\n\treturn nil\n}\n\n// startIntrospection starts serving the external introspection requests.\nfunc (m *resmgr) startIntrospection() {\n\tm.introspect.Start()\n\tm.updateIntrospection()\n}\n\n// stopInstrospection stops serving external introspection requests.\nfunc (m *resmgr) stopIntrospection() {\n\tm.introspect.Stop()\n}\n\n// updateIntrospection pushes updated data for external introspection·\nfunc (m *resmgr) updateIntrospection() {\n\tm.introspect.Set(m.policy.Introspect())\n}\n\n// registerPolicyMetricsCollector registers policy metrics collector·\nfunc (m *resmgr) registerPolicyMetricsCollector() error {\n\tpc := &policyCollector.PolicyCollector{}\n\tpc.SetPolicy(m.policy)\n\tif pc.HasPolicySpecificMetrics() {\n\t\treturn pc.RegisterPolicyMetricsCollector()\n\t}\n\tm.Info(\"%s policy has no policy-specific metrics.\", policy.ActivePolicy())\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/sockets/sockets.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sockets\n\nconst (\n\t// Containerd is the CRI socket containerd listens on.\n\tContainerd = \"/var/run/containerd/containerd.sock\"\n\t// ResourceManagerRelay is the CRI socket the resource manager listens on.\n\tResourceManagerRelay = \"/var/run/cri-resmgr/cri-resmgr.sock\"\n\t// ResourceManagerAgent is the socket the resource manager node agent listens on.\n\tResourceManagerAgent = \"/var/run/cri-resmgr/cri-resmgr-agent.sock\"\n\t// ResourceManagerConfig for resource manager configuration notifications.\n\tResourceManagerConfig = \"/var/run/cri-resmgr/cri-resmgr-config.sock\"\n\t// DirPermissions is the permissions to create the directory for sockets with.\n\tDirPermissions = 0711\n)\n"
  },
  {
    "path": "pkg/cri/resource-manager/test-api.go",
    "content": "// Copyright 2029 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build test\n// +build test\n\npackage resmgr\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n)\n\n// ResourceManagerTestAPI is a post-test verification helper interface.\ntype ResourceManagerTestAPI interface {\n\t// GetCache returns the Cache resource manager is running with.\n\tGetCache() cache.Cache\n}\n\nfunc (m *resmgr) GetCache() cache.Cache {\n\treturn m.cache\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/assets/css/style.css",
    "content": "body {\n    font-family: Arial;\n    color: white;\n}\n\na {\n    color: white;\n    text-decoration: underline;\n}\n\n.node {\n    cursor: pointer;\n}\n\n.node:hover {\n    stroke: #000;\n    stroke-width: 1.5px;\n}\n\n.node--leaf {\n    fill: white;\n}\n\n.label {\n    font: 11px \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n    text-anchor: middle;\n    text-shadow: 0 1px 0 #fff, 1px 0 0 #fff, -1px 0 0 #fff, 0 -1px 0 #fff;\n}\n\n.label,\n.inner--leaf,\n.node--root {\n    pointer-events: none;\n}\n\n.node--leaf:hover {\n    fill: gainsboro;\n}\n\n.node--leaf:active {\n    pointer-events: none\n}\n\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/assets/index.html",
    "content": "<!DOCTYPE html>\n<html>\n<head>\n    <link rel=\"stylesheet\" type=\"text/css\" href=\"css/style.css\">\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n    <script src=\"js/d3.v4.min.js\"></script>\n</head>\n<body>\n    <div id=\"graph\">\n          <svg width=\"800\" height=\"800\"></svg>\n    </div>\n    <script src=\"js/ui-json-adapter.js\"></script>\n    <script src=\"js/ui.js\"></script>\n</body>\n</html> \n\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/assets/js/ui-json-adapter.js",
    "content": "// CRI-RM introspection data to UI JSON data format adaptation.\n\"use strict\";\n\nfunction AdaptJSON(data) {\n    \"use strict\";\n    var root, nodes, containers\n\n    console.log(\"should translate introspection to d3obj: %o\", data)\n\n    root = null\n    nodes = new Object()\n    containers = new Object()\n\n    // create tree of pools\n    for (var name in data.Pools) {\n        var p = data.Pools[name]\n        var node = new Object()\n\n        console.log(\"got pool %o: %o\", name, p)\n\n        node.name     = p.Name\n        node.CPUs     = p.CPUs\n        node.Memory   = p.Memory\n        node.children = new Array()\n        if (p.Parent == \"\") {\n            root = node\n            console.log(\"root set to %o: %o\", p.parent, node)\n        }\n        nodes[name] = node\n    }\n    for (var name in data.Pools) {\n        var p = data.Pools[name]\n        var n = nodes[name]\n\n        if (n == null) {\n            console.log(\"failed to look up node %o\", name)\n        }\n        if (p.Children != null) {\n            for (var i = 0; i < p.Children.length; i++) {\n                var cname = p.Children[i]\n                n.children.push(nodes[cname])\n            }\n        }\n    }\n\n    // create lookup table of containers\n    for (var pid in data.Pods) {\n        var p = data.Pods[pid]\n\n        console.log(\"got pod %o\", pid)\n\n        for (var cid in p.Containers) {\n            var c = p.Containers[cid]\n\n            console.log(\"got container %o\", cid)\n\n            node = new Object()\n            node.name = p.Name + \":\" + c.Name\n            node.CPURequest = c.CPURequest\n            node.CPULimit = c.CPULimit\n            node.MemoryRequest = c.MemoryRequest\n            node.MemoryLimit = c.MemoryLimit\n            node.Hints = c.Hints\n            node.container = c\n            containers[cid] = node\n        }\n    }\n\n    // attach containers to pools\n    for (var cid in data.Assignments) {\n        var a = data.Assignments[cid]\n        var n = containers[cid]\n        var shared = \"\"\n        var exclusive = \"\"\n        var cpu = \"\"\n        var sep = \"\"\n\n        console.log(\"got assignment for container %o\", cid)\n\n        if (a.SharedCPUs != \"\") {\n            shared = \"shared:\"+a.SharedCPUs+\"(share:\"+a.CPUShare+\")\"\n        }\n        if (a.ExclusiveCPUs != \"\") {\n            exclusive = \"exclusive:\"+a.ExclusiveCPUs\n        }\n        if (exclusive != \"\") {\n            cpus = exclusive\n            sep = \" + \"\n        }\n        if (shared != \"\") {\n            cpu += sep + shared\n        }\n        n.CPUs         = cpu\n        n.Memory       = a.Memory\n        n.RDTClass     = a.RDTClass\n        n.BlockIOClass = a.BlockIOClass\n\n        p = nodes[a.Pool]\n        p.children.push(n)\n    }\n\n    console.log(\"translated object: %o\", root)\n\n    return root\n}\n\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/assets/js/ui.js",
    "content": "var svg = d3.select(\"svg\")\n   .attr(\"preserveAspectRatio\", \"xMinYMin meet\")\n   .attr(\"viewBox\", \"0 0 800 800\"),\n    margin = 20,\n    diameter = +svg.attr(\"width\"),\n    g = svg.append(\"g\").attr(\"transform\", \"translate(\" + diameter / 2 + \",\" + diameter / 2 + \")\");\n\nvar green = d3.color(\"green\");\n\nvar color = d3.scaleLinear()\n    .domain([-1, 5])\n    .range([\"hsl(152,80%,80%)\", \"hsl(228,30%,40%)\"])\n    .interpolate(d3.interpolateHcl);\n\nvar pack = d3.pack()\n    .size([diameter - margin, diameter - margin])\n    .padding(100);\n\ndrawBubbleGraph(\"/introspect\")\n\nfunction drawBubbleGraph(filename) {\n    console.log(\"redraw\")\n\n    g.selectAll(\"*\").remove()\n\n    d3.json(filename, function(error, introspectJSON) {\n        if (error) throw error;\n\n        var root = AdaptJSON(introspectJSON)\n        root = d3.hierarchy(root)\n            .sum(function(d) { return d.CPURequest; })\n            .sort(function(a, b) { console.log (b.value + \" - \" + a.value);return b.value - a.value; });\n\n\n        var focus = root,\n        nodes = pack(root).descendants(),\n        view;\n        console.log(nodes);\n        var circle = g.selectAll(\"circle\")\n            .data(nodes)\n            .enter().append(\"circle\")\n                .attr(\"class\", function(d) { console.log(\"dx: \" + d.x + \" dy: \" + d.y + \" dr: \" + d.r); console.log(d.data.name); d.parent ? d.children ? console.log(\"node\") : console.log(\"node leaf\") : console.log (\"node root\");  return d.parent ? d.children ? \"node\" : \"node node--leaf\" : \"node node--root\"; })\n                .on(\"click\", function(d) { if (focus !== d) zoom(d), d3.event.stopPropagation(); })\n                .on(\"mouseover\", function(d) {return d.children ? null : showData(d);})\n                .on(\"mouseout\", function(d) {return d.children ? null : clearData(d);})\n                .style(\"fill\", function(d) { return d.children ? color(d.depth) : null; })\n\n        let innercircle = g.selectAll(\"innercircle\")\n          .data(nodes)\n          .enter().append(\"circle\")\n          .attr(\"class\", function(d) { return d.parent ? d.children ? \"inner--node\" : \"inner--leaf\" : \"inner--root\"; })\n\n          let innerleaf = g.selectAll(\".inner--leaf\")\n              .attr(\"r\", function(d) {if (d.data.CPULimit || d.data.CPURequest) return (d.r * d.data.CPULimit / d.data.CPURequest);})\n              .style(\"fill-opacity\", 0.2)\n              .on(\"click\", function(d) { if (focus !== d) zoom(d), d3.event.stopPropagation(); })\n              .style(\"fill\", \"red\");\n\n        var text = g.selectAll(\"text\")\n            .data(nodes)\n            .enter().append(\"text\")\n                .attr(\"class\", \"label\")\n                .style(\"fill-opacity\", function(d) { return d.parent === root ? 1 : 0; })\n                .style(\"display\", function(d) { return d.parent === root ? \"inline\" : \"none\"; })\n                .text(function(d) { return d.data.name;});\n\n        var node = g.selectAll(\"circle,innerleaf,text\");\n\n    svg\n      .style(\"background\", color(-1))\n      .on(\"click\", function() { zoom(root); });\n\n    zoomTo([root.x, root.y, root.r * 2 + margin]);\n\n    function zoom(d) {\n        var focus0 = focus; focus = d;\n\n        var transition = d3.transition()\n            .duration(d3.event.altKey ? 7500 : 750)\n            .tween(\"zoom\", function(d) {\n              var i = d3.interpolateZoom(view, [focus.x, focus.y, focus.r * 2 + margin]);\n              return function(t) { zoomTo(i(t)); };\n            });\n\n        svg.transition().selectAll(\"text\")\n        .filter(function(d) { return d.parent === focus || this.style.display === \"inline\"; })\n            .style(\"fill-opacity\", function(d) { return d.parent === focus ? 1 : 0; })\n            .on(\"start\", function(d) { if (d.parent === focus) this.style.display = \"inline\"; })\n            .on(\"end\", function(d) { if (d.parent !== focus) this.style.display = \"none\"; });\n    }\n  function zoomTo(v) {\n    var k = diameter / v[2]; view = v;\n    node.attr(\"transform\", function(d) { return \"translate(\" + (d.x - v[0]) * k + \",\" + (d.y - v[1]) * k + \")\"; });\n    circle.attr(\"r\", function(d) { if (d.children) return d.r *k; if (d.data.CPULimit && d.data.CPURequest) return d.r * k; else return 20 * k ; })\n    circle.style(\"fill\", function(d) { if (d.children) return color(d.depth); if (!d.data.CPULimit || !d.data.CPURequest)return \"grey\"; else return color(d.depth);});\n    innerleaf.attr(\"r\", function(d) { if (d.data.CPULimit && d.data.CPURequest) {\n                                        if (d.data.CPULimit == d.data.CPURequest) return d.r  * k;\n                                        else return d.r * 2 *k; }});\n  }\n \n        let current_circle = undefined;\n        function clearData(d) {\n            console.log(\"CCLEAR DATA\");\n                svg.selectAll(\"#details-popup\").remove();\n        }\n\n        function showData(d) {\n            // cleanup previous selected circle\n            if(current_circle !== undefined){\n                svg.selectAll(\"#details-popup\").remove();\n            }\n            console.log(\"here I am\" + d.data.name);\n\n        // select the circle\n        current_circle = d3.select(this);\n        console.log(\"here\");\n        console.log(current_circle);\n\n        let textblock = svg.selectAll(\"#details-popup\")\n          .data([d])\n          .enter()\n          .append(\"g\")\n          .attr(\"id\", \"details-popup\")\n          .attr(\"font-size\", 14)\n          .attr(\"font-family\", \"sans-serif\")\n          .attr(\"text-anchor\", \"start\")\n          .attr(\"transform\", d => `translate(0, 20)`);\n\n        textblock.append(\"text\")\n          .text(\"Details:\")\n          .attr(\"font-weight\", \"bold\");\n        textblock.append(\"text\")\n          .text(d => \"Name: \" + d.data.name)\n          .attr(\"y\", \"16\");\n        textblock.append(\"text\")\n          .text(d => \"CPUs: \" + d.data.CPUs)\n          .attr(\"y\", \"32\");\n        textblock.append(\"text\")\n          .text(d => \"CPU Request: \" + d.data.CPURequest)\n          .attr(\"y\", \"48\");\n        textblock.append(\"text\")\n          .text(d => \"CPU Limit: \" + d.data.CPULimit)\n          .attr(\"y\", \"64\");\n        textblock.append(\"text\")\n          .text(d => \"Memory Request: \" + d.data.MemoryRequest)\n          .attr(\"y\", \"80\");\n        textblock.append(\"text\")\n          .text(d => \"Memory Limit: \" + d.data.MemoryLimit)\n          .attr(\"y\", \"96\");\n        }\n    });\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/assets.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build test\n// +build test\n\npackage bubbles\n\nimport (\n\t\"net/http\"\n)\n\n// Assets is our UI assets for 'bubbles' visualizer, to serve over HTTP.\nvar Assets = http.Dir(\"assets\")\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/assets_generate.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build ignore\n// +build ignore\n\npackage main\n\nimport (\n\t\"fmt\"\n\tvisualizer \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/visualizer/bubbles\"\n\t\"github.com/shurcooL/vfsgen\"\n\t\"log\"\n)\n\nconst (\n\tname = \"bubbles\"\n)\n\nfunc main() {\n\topts := vfsgen.Options{\n\t\tPackageName:  name,\n\t\tBuildTags:    \"!test\",\n\t\tVariableName: \"Assets\",\n\t\tFilename:     \"assets_gendata.go\",\n\t}\n\tif err := vfsgen.Generate(visualizer.Assets, opts); err != nil {\n\t\tlog.Fatalln(fmt.Sprintf(\"failed to generate assets for %s UI:\", name, err))\n\t}\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/bubbles/doc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage bubbles\n\nimport (\n\t// The blank import is to make govendor happy.\n\t_ \"github.com/shurcooL/vfsgen\"\n)\n\n//go:generate go run -tags=test assets_generate.go\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/builtins.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !dev\n// +build !dev\n\npackage visualizer\n\nimport (\n\t// Pull in builtin visualizer implementations.\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/visualizer/bubbles\"\n)\n\nfunc init() {\n\tvisualizers.register(\"bubbles\", bubbles.Assets)\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/flags.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage visualizer\n\nimport (\n\t\"flag\"\n)\n\n// externalDirs is a comma-separated list of directories to search for visualizers.\nvar externalDirs string\n\n// Register our command line options.\nfunc init() {\n\tflag.StringVar(&externalDirs, \"external-visualizers\", \"\",\n\t\t\"comma-separated list of directories to search for external visualizers.\")\n}\n"
  },
  {
    "path": "pkg/cri/resource-manager/visualizer/visualizer.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage visualizer\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strings\"\n\n\txhttp \"github.com/intel/cri-resource-manager/pkg/instrumentation/http\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// HTTP URI prefix to register all visualizer implementations under.\n\tvisualizerPrefix = \"/ui\"\n)\n\n// Our logger instance.\nvar log = logger.NewLogger(\"visualizer\")\n\n// visualizer captures our runtime state.\ntype visualizer struct {\n\tbuiltin map[string]http.FileSystem\n}\n\n// Visualizer singleton instance.\nvar visualizers = &visualizer{\n\tbuiltin: map[string]http.FileSystem{},\n}\n\n// Register registers an builtin visualizer implementation.\nfunc Register(name string, dir http.FileSystem) {\n\tvisualizers.register(name, dir)\n}\n\n// Setup sets up the given multiplexer to serve visualization implementations.\nfunc Setup(mux *xhttp.ServeMux) error {\n\tlog.Info(\"activating visualization interface...\")\n\n\tmux.Handle(\"/\", http.RedirectHandler(\"/ui/index.html\", http.StatusFound))\n\tmux.Handle(\"/ui\", http.RedirectHandler(\"/ui/index.html\", http.StatusFound))\n\tmux.Handle(\"/ui/builtin/\", http.FileServer(visualizers))\n\tmux.Handle(\"/ui/external/\", http.FileServer(visualizers))\n\tmux.HandleFunc(\"/ui/index.html\", visualizers.generateIndexHTML)\n\n\treturn nil\n}\n\n// Open is the http.Dir implementation for our visualizers.\nfunc (v *visualizer) Open(path string) (http.File, error) {\n\tlog.Debug(\"HTTP request %q\", path)\n\n\trelative, err := filepath.Rel(visualizerPrefix+\"/\", path)\n\tif err != nil {\n\t\treturn nil, visualizerError(\"failed to resolve path %q: %v\", err)\n\t}\n\n\tlog.Debug(\"%s => %s\", path, relative)\n\n\tsplit := strings.Split(relative, \"/\")\n\tif len(split) < 2 {\n\t\treturn nil, visualizerError(\"failed to resolve relative path %q\", relative)\n\t}\n\n\tkind, name := split[0], split[1]\n\tfs, err := v.getVisualizerFileSystem(kind, name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn fs.Open(filepath.Join(split[2:]...))\n}\n\n// getVisualizerFileSystem returns the http.FileSystem for the given visualizer.\nfunc (v *visualizer) getVisualizerFileSystem(kind, name string) (http.FileSystem, error) {\n\tswitch kind {\n\tcase \"builtin\":\n\t\tif dir, ok := v.builtin[name]; ok {\n\t\t\treturn dir, nil\n\t\t}\n\t\treturn nil, visualizerError(\"unknown builtin visualization UI %q\", name)\n\tcase \"external\":\n\t\texternal := v.discoverExternalUIs()\n\t\tif path, ok := external[name]; ok {\n\t\t\treturn http.FileSystem(http.Dir(path)), nil\n\t\t}\n\t\treturn nil, visualizerError(\"unkown external visualization UI %q\", name)\n\t}\n\treturn nil, visualizerError(\"unknown visualization UI type %q\", kind)\n}\n\n// Index page HTML header and footer.\nconst (\n\tuiPageHTMLHeader = `\n<html>\n  <head>\n    <title>CRI Resource Manager - Workload Placement Visualization</title>\n  </head>\n  <body>\n    <ul>\n`\n\tuiPageHTMLFooter = `\n    </ul>\n  </body>\n</html>\n`\n)\n\n// generateIndexHTML generates a HTML page to access all known visualization UIs.\nfunc (v *visualizer) generateIndexHTML(w http.ResponseWriter, _ *http.Request) {\n\tbuiltinUIs := []string{}\n\tfor name := range v.builtin {\n\t\tbuiltinUIs = append(builtinUIs, name)\n\t}\n\tsort.Strings(builtinUIs)\n\n\texternalUIs := []string{}\n\tfor name := range v.discoverExternalUIs() {\n\t\texternalUIs = append(externalUIs, name)\n\t}\n\tsort.Strings(externalUIs)\n\n\tfmt.Fprintf(w, \"%s\", uiPageHTMLHeader)\n\tif len(builtinUIs)+len(externalUIs) == 0 {\n\t\tfmt.Fprintf(w, \"No builtin or external visualization UIs found.\")\n\t} else {\n\t\tfor _, name := range builtinUIs {\n\t\t\tfmt.Fprintf(w, \"<li><a href=\\\"/ui/builtin/%s\\\">%s</a>\\n\", name, name)\n\t\t}\n\t\tfor _, name := range externalUIs {\n\t\t\tfmt.Fprintf(w, \"<li><a href=\\\"/ui/external/%s\\\">external %s</a>\\n\", name, name)\n\t\t}\n\t}\n\tfmt.Fprintf(w, \"%s\\r\\n\", uiPageHTMLFooter)\n}\n\n// register registers a builtin visualizer implementation.\nfunc (v *visualizer) register(name string, dir http.FileSystem) {\n\tif _, ok := v.builtin[name]; ok {\n\t\tlog.Error(\"builtin visualizer '%s' already registered\", name)\n\t\treturn\n\t}\n\tv.builtin[name] = dir\n\tlog.Info(\"registered %s builtin visualizer...\", name)\n}\n\n// discoverExternalUIs returns a map of external visualizer implementations.\nfunc (v *visualizer) discoverExternalUIs() map[string]string {\n\texternal := make(map[string]string)\n\tfor _, root := range strings.Split(externalDirs, \",\") {\n\t\tfilepath.Walk(root, func(path string, info os.FileInfo, err error) error {\n\t\t\tif err != nil || info.IsDir() || info.Name() != \"index.html\" {\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tdir, err := filepath.Abs(filepath.Dir(path))\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(\"failed to determine absolute directory for '%s': %v\", path, err)\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tname := v.uniqueExternalUIName(dir, external)\n\t\t\texternal[name] = dir\n\n\t\t\tlog.Debug(\"found external visualizer '%s' (%s)\", name, dir)\n\n\t\t\treturn nil\n\t\t})\n\t}\n\treturn external\n}\n\n// uniqueExternalUIName generates a unique name for the external visualizer.\nfunc (v *visualizer) uniqueExternalUIName(dir string, others map[string]string) string {\n\tbase := filepath.Base(dir)\n\tif base == \"assets\" {\n\t\tbase = filepath.Base(filepath.Dir(dir))\n\t}\n\tcnt := 0\n\tname := base\n\tfor {\n\t\tif cnt > 0 {\n\t\t\tname = base + fmt.Sprintf(\"-%d\", cnt)\n\t\t}\n\t\tif _, ok := others[name]; !ok {\n\t\t\treturn name\n\t\t}\n\t\tcnt++\n\t}\n}\n\n// visualizerError returns a formatted package-specific error.\nfunc visualizerError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"visualizer: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/server/server.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"os/user\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"google.golang.org/grpc\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/sockets\"\n\t\"github.com/intel/cri-resource-manager/pkg/dump\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation\"\n\t\"go.opencensus.io/trace\"\n)\n\n// Options contains the configurable options of our CRI server.\ntype Options struct {\n\t// Socket is the path of our gRPC servers unix-domain socket.\n\tSocket string\n\t// User is the user ID for our gRPC socket.\n\tUser int\n\t// Group is the group ID for our gRPC socket.\n\tGroup int\n\t// Mode is the permission mode bits for our gRPC socket.\n\tMode os.FileMode\n\t// QualifyReqFn produces return context for disambiguating a CRI request/reply.\n\tQualifyReqFn func(interface{}) string\n}\n\n// Handler is a CRI server generic request handler.\ntype Handler grpc.UnaryHandler\n\n// Interceptor is a hook that intercepts processing a request by a handler.\ntype Interceptor func(context.Context, string, interface{}, Handler) (interface{}, error)\n\n// Server is the interface we expose for controlling our CRI server.\ntype Server interface {\n\t// RegisterImageService registers the provided image service with the server.\n\tRegisterImageService(criv1.ImageServiceServer) error\n\t// RegistersRuntimeService registers the provided runtime service with the server.\n\tRegisterRuntimeService(criv1.RuntimeServiceServer) error\n\t// RegisterInterceptors registers the given interceptors with the server.\n\tRegisterInterceptors(map[string]Interceptor) error\n\t// Start starts the request processing loop (goroutine) of the server.\n\tStart() error\n\t// Stop stops the request processing loop (goroutine) of the server.\n\tStop()\n\t// Chmod changes the permissions of the server's socket.\n\tChmod(mode os.FileMode) error\n\t// Chown changes ownership of the server's socket.\n\tChown(uid, gid int) error\n}\n\n// server is the implementation of Server.\ntype server struct {\n\tlogger.Logger\n\tlistener     net.Listener                // socket our gRPC server listens on\n\tserver       *grpc.Server                // our gRPC server\n\toptions      Options                     // server options\n\tinterceptors map[string]Interceptor      // request intercepting hooks\n\truntime      *criv1.RuntimeServiceServer // CRI runtime service\n\timage        *criv1.ImageServiceServer   // CRI image service\n}\n\n// NewServer creates a new server instance.\nfunc NewServer(options Options) (Server, error) {\n\tif !filepath.IsAbs(options.Socket) {\n\t\treturn nil, serverError(\"invalid socket '%s', expecting absolute path\",\n\t\t\toptions.Socket)\n\t}\n\n\ts := &server{\n\t\tLogger:  logger.NewLogger(\"cri/server\"),\n\t\toptions: options,\n\t}\n\n\treturn s, nil\n}\n\n// RegisterImageService registers an image service with the server.\nfunc (s *server) RegisterImageService(service criv1.ImageServiceServer) error {\n\tif s.image != nil {\n\t\treturn serverError(\"can't register image service, already registered\")\n\t}\n\n\tif err := s.createGrpcServer(); err != nil {\n\t\treturn err\n\t}\n\n\tis := service\n\ts.image = &is\n\tcriv1.RegisterImageServiceServer(s.server, s)\n\n\treturn nil\n}\n\n// RegisterRuntimeService registers a runtime service with the server.\nfunc (s *server) RegisterRuntimeService(service criv1.RuntimeServiceServer) error {\n\tif s.runtime != nil {\n\t\treturn serverError(\"can't register runtime server, already registered\")\n\t}\n\n\tif err := s.createGrpcServer(); err != nil {\n\t\treturn err\n\t}\n\n\trs := service\n\ts.runtime = &rs\n\tcriv1.RegisterRuntimeServiceServer(s.server, s)\n\n\treturn nil\n}\n\n// RegisterInterceptors registers the given interveptors with the server.\nfunc (s *server) RegisterInterceptors(intercept map[string]Interceptor) error {\n\tif s.interceptors == nil {\n\t\ts.interceptors = make(map[string]Interceptor)\n\t}\n\n\tfor method, i := range intercept {\n\t\tif _, ok := s.interceptors[method]; ok {\n\t\t\treturn serverError(\"server already has a registered interceptor for '%s'\", method)\n\t\t}\n\t\ts.interceptors[method] = i\n\t}\n\n\treturn nil\n}\n\n// Start starts the servers request processing goroutine.\nfunc (s *server) Start() error {\n\ts.trainMessageDumper()\n\n\ts.Debug(\"starting server on socket %s...\", s.options.Socket)\n\tgo func() {\n\t\ts.server.Serve(s.listener)\n\t}()\n\n\ts.Debug(\"waiting for server to become ready...\")\n\tif err := utils.WaitForServer(s.options.Socket, time.Second); err != nil {\n\t\treturn serverError(\"starting CRI server failed: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// Stop serving CRI requests.\nfunc (s *server) Stop() {\n\ts.Debug(\"stopping server on socket %s...\", s.options.Socket)\n\ts.server.Stop()\n}\n\n// createGrpcServer creates a gRPC server instance on our socket.\nfunc (s *server) createGrpcServer() error {\n\tif s.server != nil {\n\t\treturn nil\n\t}\n\n\tif err := os.MkdirAll(filepath.Dir(s.options.Socket), sockets.DirPermissions); err != nil {\n\t\treturn serverError(\"failed to create directory for socket %s: %v\",\n\t\t\ts.options.Socket, err)\n\t}\n\n\tl, err := net.Listen(\"unix\", s.options.Socket)\n\tif err != nil {\n\t\tif ls, lsErr := utils.IsListeningSocket(s.options.Socket); ls || lsErr != nil {\n\t\t\treturn serverError(\"failed to create server: socket %q already exists\",\n\t\t\t\ts.options.Socket)\n\t\t}\n\t\ts.Warn(\"removing abandoned socket %q...\", s.options.Socket)\n\t\tos.Remove(s.options.Socket)\n\t\tl, err = net.Listen(\"unix\", s.options.Socket)\n\t\tif err != nil {\n\t\t\treturn serverError(\"failed to create server on socket %s: %v\",\n\t\t\t\ts.options.Socket, err)\n\t\t}\n\t}\n\n\ts.listener = l\n\n\tif s.options.User >= 0 {\n\t\tif err := s.Chown(s.options.User, s.options.Group); err != nil {\n\t\t\tl.Close()\n\t\t\ts.listener = nil\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif s.options.Mode != 0 {\n\t\tif err := s.Chmod(s.options.Mode); err != nil {\n\t\t\tl.Close()\n\t\t\ts.listener = nil\n\t\t\treturn err\n\t\t}\n\t}\n\n\ts.server = grpc.NewServer(instrumentation.InjectGrpcServerTrace()...)\n\n\treturn nil\n}\n\n// Chmod changes the permissions of the server's socket.\nfunc (s *server) Chmod(mode os.FileMode) error {\n\tif s.listener != nil {\n\t\tif err := os.Chmod(s.options.Socket, mode); err != nil {\n\t\t\treturn serverError(\"failed to change permissions of socket %q to %v: %v\",\n\t\t\t\ts.options.Socket, mode, err)\n\t\t}\n\t\ts.Info(\"changed permissions of socket %q to %v\", s.options.Socket, mode)\n\t}\n\n\ts.options.Mode = mode\n\n\treturn nil\n}\n\n// Chown changes ownership of the server's socket.\nfunc (s *server) Chown(uid, gid int) error {\n\tif s.listener != nil {\n\t\tuserName := strconv.FormatInt(int64(uid), 10)\n\t\tif u, err := user.LookupId(userName); u != nil && err == nil {\n\t\t\tuserName = u.Name\n\t\t}\n\t\tgroupName := strconv.FormatInt(int64(gid), 10)\n\t\tif g, err := user.LookupGroupId(groupName); g != nil && err == nil {\n\t\t\tgroupName = g.Name\n\t\t}\n\t\tif err := os.Chown(s.options.Socket, uid, gid); err != nil {\n\t\t\treturn serverError(\"failed to change ownership of socket %q to %s/%s: %v\",\n\t\t\t\ts.options.Socket, userName, groupName, err)\n\t\t}\n\t\ts.Info(\"changed ownership of socket %q to %s/%s\", s.options.Socket, userName, groupName)\n\t}\n\n\ts.options.User = uid\n\ts.options.Group = gid\n\n\treturn nil\n}\n\n// getInterceptor finds an interceptor for the given method.\nfunc (s *server) getInterceptor(method string) (Interceptor, string) {\n\tname := method[strings.LastIndex(method, \"/\")+1:]\n\n\tif fn, ok := s.interceptors[name]; ok {\n\t\treturn fn, name\n\t}\n\n\tif fn, ok := s.interceptors[\"*\"]; ok {\n\t\treturn fn, name\n\t}\n\n\treturn nil, name\n}\n\n// intercept processes requests with a registered interceptor or the default handler.\nfunc (s *server) intercept(ctx context.Context, req interface{},\n\tinfo *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {\n\t// Notes:\n\t//   We record timestamps at various phases of processing a request to later\n\t//   calculate local, CRI-server and total request processing latencies. We\n\t//   wrap the original handler to get the pre- and post-communication stamps\n\t//   with reasonable accuracy without having to get the stamps at the client.\n\t//\n\t//   One thing that we currently fail to measure separately is the latency of\n\t//   internally generated CRI requests (UpdateContainerResources). These are\n\t//   now accounted to the local processing latency of the triggering request.\n\n\tvar kind string\n\tvar start, send, recv, end time.Time\n\tvar sync bool\n\n\twrapHandler := func(ctx context.Context, req interface{}) (interface{}, error) {\n\t\tsend = time.Now()\n\t\trpl, err := handler(ctx, req)\n\t\trecv = time.Now()\n\t\treturn rpl, err\n\t}\n\n\tfn, name := s.getInterceptor(info.FullMethod)\n\tif fn != nil {\n\t\tkind = \"intercepted\"\n\t\tsync = true\n\t} else {\n\t\tkind = \"passthrough\"\n\t\tfn = func(c context.Context, n string, r interface{}, h Handler) (interface{}, error) {\n\t\t\trpl, err := h(c, r)\n\t\t\treturn rpl, err\n\t\t}\n\t}\n\n\tqualif := s.qualifier(req)\n\tdump.RequestMessage(kind, info.FullMethod, qualif, req, sync)\n\n\tif span := trace.FromContext(ctx); span != nil {\n\t\tspan.AddAttributes(trace.StringAttribute(\"kind\", kind))\n\t}\n\n\tstart = time.Now()\n\trpl, err := fn(ctx, name, req, wrapHandler)\n\tend = time.Now()\n\telapsed := end.Sub(start)\n\n\tif err != nil {\n\t\tdump.ReplyMessage(kind, info.FullMethod, qualif, err, elapsed, false)\n\t} else {\n\t\tdump.ReplyMessage(kind, info.FullMethod, qualif, rpl, elapsed, false)\n\t}\n\n\ts.collectStatistics(kind, name, start, send, recv, end)\n\tlogger.Flush()\n\n\treturn rpl, err\n}\n\n// collectStatistics collects (should collect) request processing statistics.\nfunc (s *server) collectStatistics(kind, name string, start, send, recv, end time.Time) {\n\tif kind == \"passthrough\" {\n\t\treturn\n\t}\n\n\tpre := send.Sub(start)\n\tserver := recv.Sub(send)\n\tpost := end.Sub(recv)\n\n\ts.Debug(\" * latency for %s: preprocess: %v, CRI server: %v, postprocess: %v, total: %v\",\n\t\tname, pre, server, post, pre+server+post)\n}\n\n// trainMessageDumper pre-trains the message dumper with our full set of service methods.\nfunc (s server) trainMessageDumper() {\n\tmethods := []string{}\n\tsvcinfo := s.server.GetServiceInfo()\n\tfor _, info := range svcinfo {\n\t\tfor _, m := range info.Methods {\n\t\t\tmethods = append(methods, m.Name)\n\t\t}\n\t}\n\tdump.Train(methods)\n}\n\n// qualifier pulls a qualifier for disambiguation from a CRI request message.\nfunc (s server) qualifier(msg interface{}) string {\n\tif fn := s.options.QualifyReqFn; fn != nil {\n\t\treturn fn(msg)\n\t}\n\treturn \"\"\n}\n\n// Return a formatter server error.\nfunc serverError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"cri/server: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/cri/server/services.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage server\n\nimport (\n\t\"context\"\n\n\t\"go.opencensus.io/trace\"\n\t\"google.golang.org/grpc\"\n\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n)\n\nconst (\n\tapiVersion = \"v1\"\n\n\timageService = \"ImageService\"\n\tlistImages   = \"ListImages\"\n\timageStatus  = \"ImageStatus\"\n\tpullImage    = \"PullImage\"\n\tremoveImage  = \"RemoveImage\"\n\timageFsInfo  = \"ImageFsInfo\"\n\n\truntimeService           = \"RuntimeService\"\n\tversion                  = \"Version\"\n\trunPodSandbox            = \"RunPodSandbox\"\n\tstopPodSandbox           = \"StopPodSandbox\"\n\tremovePodSandbox         = \"RemovePodSandbox\"\n\tpodSandboxStatus         = \"PodSandboxStatus\"\n\tlistPodSandbox           = \"ListPodSandbox\"\n\tcreateContainer          = \"CreateContainer\"\n\tstartContainer           = \"StartContainer\"\n\tstopContainer            = \"StopContainer\"\n\tremoveContainer          = \"RemoveContainer\"\n\tlistContainers           = \"ListContainers\"\n\tcontainerStatus          = \"ContainerStatus\"\n\tupdateContainerResources = \"UpdateContainerResources\"\n\treopenContainerLog       = \"ReopenContainerLog\"\n\texecSync                 = \"ExecSync\"\n\texec                     = \"Exec\"\n\tattach                   = \"Attach\"\n\tportForward              = \"PortForward\"\n\tcontainerStats           = \"ContainerStats\"\n\tlistContainerStats       = \"ListContainerStats\"\n\tpodSandboxStats          = \"PodSandboxStats\"\n\tlistPodSandboxStats      = \"ListPodSandboxStats\"\n\tupdateRuntimeConfig      = \"UpdateRuntimeConfig\"\n\tstatus                   = \"Status\"\n\tcheckpointContainer      = \"CheckpointContainer\"\n\tgetContainerEvents       = \"GetContainerEvents\"\n\tlistMetricDescriptors    = \"ListMetricDescriptors\"\n\tlistPodSandboxMetrics    = \"ListPodSandboxMetrics\"\n\truntimeConfig            = \"RuntimeConfig\"\n)\n\nfunc fqmn(service, method string) string {\n\treturn \"/runtime.\" + apiVersion + \".\" + service + \"/\" + method\n}\n\nfunc (s *server) interceptRequest(ctx context.Context, service, method string,\n\treq interface{}, handler grpc.UnaryHandler) (interface{}, error) {\n\n\tif span := trace.FromContext(ctx); span != nil {\n\t\tspan.AddAttributes(\n\t\t\ttrace.StringAttribute(\"service\", service),\n\t\t\ttrace.StringAttribute(\"method\", method))\n\t}\n\n\treturn s.intercept(ctx, req,\n\t\t&grpc.UnaryServerInfo{Server: s, FullMethod: fqmn(service, method)}, handler)\n}\n\nfunc (s *server) ListImages(ctx context.Context,\n\treq *criv1.ListImagesRequest) (*criv1.ListImagesResponse, error) {\n\trsp, err := s.interceptRequest(ctx, imageService, listImages, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.image).ListImages(ctx, req.(*criv1.ListImagesRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListImagesResponse), err\n}\n\nfunc (s *server) ImageStatus(ctx context.Context,\n\treq *criv1.ImageStatusRequest) (*criv1.ImageStatusResponse, error) {\n\trsp, err := s.interceptRequest(ctx, imageService, imageStatus, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.image).ImageStatus(ctx, req.(*criv1.ImageStatusRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ImageStatusResponse), err\n}\n\nfunc (s *server) PullImage(ctx context.Context,\n\treq *criv1.PullImageRequest) (*criv1.PullImageResponse, error) {\n\trsp, err := s.interceptRequest(ctx, imageService, pullImage, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.image).PullImage(ctx, req.(*criv1.PullImageRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.PullImageResponse), err\n}\n\nfunc (s *server) RemoveImage(ctx context.Context,\n\treq *criv1.RemoveImageRequest) (*criv1.RemoveImageResponse, error) {\n\trsp, err := s.interceptRequest(ctx, imageService, removeImage, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.image).RemoveImage(ctx, req.(*criv1.RemoveImageRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.RemoveImageResponse), err\n}\n\nfunc (s *server) ImageFsInfo(ctx context.Context,\n\treq *criv1.ImageFsInfoRequest) (*criv1.ImageFsInfoResponse, error) {\n\trsp, err := s.interceptRequest(ctx, imageService, imageFsInfo, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.image).ImageFsInfo(ctx, req.(*criv1.ImageFsInfoRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ImageFsInfoResponse), err\n}\n\nfunc (s *server) Version(ctx context.Context,\n\treq *criv1.VersionRequest) (*criv1.VersionResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, version, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).Version(ctx, req.(*criv1.VersionRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.VersionResponse), err\n}\n\nfunc (s *server) RunPodSandbox(ctx context.Context,\n\treq *criv1.RunPodSandboxRequest) (*criv1.RunPodSandboxResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, runPodSandbox, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).RunPodSandbox(ctx, req.(*criv1.RunPodSandboxRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.RunPodSandboxResponse), err\n}\n\nfunc (s *server) StopPodSandbox(ctx context.Context,\n\treq *criv1.StopPodSandboxRequest) (*criv1.StopPodSandboxResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, stopPodSandbox, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).StopPodSandbox(ctx, req.(*criv1.StopPodSandboxRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.StopPodSandboxResponse), err\n}\n\nfunc (s *server) RemovePodSandbox(ctx context.Context,\n\treq *criv1.RemovePodSandboxRequest) (*criv1.RemovePodSandboxResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, removePodSandbox, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).RemovePodSandbox(ctx, req.(*criv1.RemovePodSandboxRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.RemovePodSandboxResponse), err\n}\n\nfunc (s *server) PodSandboxStatus(ctx context.Context,\n\treq *criv1.PodSandboxStatusRequest) (*criv1.PodSandboxStatusResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, podSandboxStatus, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).PodSandboxStatus(ctx, req.(*criv1.PodSandboxStatusRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.PodSandboxStatusResponse), err\n}\n\nfunc (s *server) ListPodSandbox(ctx context.Context,\n\treq *criv1.ListPodSandboxRequest) (*criv1.ListPodSandboxResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, listPodSandbox, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ListPodSandbox(ctx, req.(*criv1.ListPodSandboxRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListPodSandboxResponse), err\n}\n\nfunc (s *server) CreateContainer(ctx context.Context,\n\treq *criv1.CreateContainerRequest) (*criv1.CreateContainerResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, createContainer, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).CreateContainer(ctx, req.(*criv1.CreateContainerRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.CreateContainerResponse), err\n}\n\nfunc (s *server) StartContainer(ctx context.Context,\n\treq *criv1.StartContainerRequest) (*criv1.StartContainerResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, startContainer, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).StartContainer(ctx, req.(*criv1.StartContainerRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.StartContainerResponse), err\n}\n\nfunc (s *server) StopContainer(ctx context.Context,\n\treq *criv1.StopContainerRequest) (*criv1.StopContainerResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, stopContainer, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).StopContainer(ctx, req.(*criv1.StopContainerRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.StopContainerResponse), err\n}\n\nfunc (s *server) RemoveContainer(ctx context.Context,\n\treq *criv1.RemoveContainerRequest) (*criv1.RemoveContainerResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, removeContainer, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).RemoveContainer(ctx, req.(*criv1.RemoveContainerRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.RemoveContainerResponse), err\n}\n\nfunc (s *server) ListContainers(ctx context.Context,\n\treq *criv1.ListContainersRequest) (*criv1.ListContainersResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, listContainers, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ListContainers(ctx, req.(*criv1.ListContainersRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListContainersResponse), err\n}\n\nfunc (s *server) ContainerStatus(ctx context.Context,\n\treq *criv1.ContainerStatusRequest) (*criv1.ContainerStatusResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, containerStatus, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ContainerStatus(ctx, req.(*criv1.ContainerStatusRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ContainerStatusResponse), err\n}\n\nfunc (s *server) UpdateContainerResources(ctx context.Context,\n\treq *criv1.UpdateContainerResourcesRequest) (*criv1.UpdateContainerResourcesResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, updateContainerResources, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).UpdateContainerResources(ctx,\n\t\t\t\treq.(*criv1.UpdateContainerResourcesRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.UpdateContainerResourcesResponse), err\n}\n\nfunc (s *server) ReopenContainerLog(ctx context.Context,\n\treq *criv1.ReopenContainerLogRequest) (*criv1.ReopenContainerLogResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, reopenContainerLog, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ReopenContainerLog(ctx, req.(*criv1.ReopenContainerLogRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ReopenContainerLogResponse), err\n}\n\nfunc (s *server) ExecSync(ctx context.Context,\n\treq *criv1.ExecSyncRequest) (*criv1.ExecSyncResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, execSync, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ExecSync(ctx, req.(*criv1.ExecSyncRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ExecSyncResponse), err\n}\n\nfunc (s *server) Exec(ctx context.Context,\n\treq *criv1.ExecRequest) (*criv1.ExecResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, exec, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).Exec(ctx, req.(*criv1.ExecRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ExecResponse), err\n}\n\nfunc (s *server) Attach(ctx context.Context,\n\treq *criv1.AttachRequest) (*criv1.AttachResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, attach, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).Attach(ctx, req.(*criv1.AttachRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.AttachResponse), err\n}\n\nfunc (s *server) PortForward(ctx context.Context,\n\treq *criv1.PortForwardRequest) (*criv1.PortForwardResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, portForward, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).PortForward(ctx, req.(*criv1.PortForwardRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.PortForwardResponse), err\n}\n\nfunc (s *server) ContainerStats(ctx context.Context,\n\treq *criv1.ContainerStatsRequest) (*criv1.ContainerStatsResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, containerStats, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ContainerStats(ctx, req.(*criv1.ContainerStatsRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ContainerStatsResponse), err\n}\n\nfunc (s *server) ListContainerStats(ctx context.Context,\n\treq *criv1.ListContainerStatsRequest) (*criv1.ListContainerStatsResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, listContainerStats, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ListContainerStats(ctx, req.(*criv1.ListContainerStatsRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListContainerStatsResponse), err\n}\n\nfunc (s *server) PodSandboxStats(ctx context.Context, req *criv1.PodSandboxStatsRequest) (*criv1.PodSandboxStatsResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, podSandboxStats, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).PodSandboxStats(ctx, req.(*criv1.PodSandboxStatsRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.PodSandboxStatsResponse), err\n}\n\nfunc (s *server) ListPodSandboxStats(ctx context.Context, req *criv1.ListPodSandboxStatsRequest) (*criv1.ListPodSandboxStatsResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, listPodSandboxStats, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ListPodSandboxStats(ctx, req.(*criv1.ListPodSandboxStatsRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListPodSandboxStatsResponse), err\n}\n\nfunc (s *server) UpdateRuntimeConfig(ctx context.Context,\n\treq *criv1.UpdateRuntimeConfigRequest) (*criv1.UpdateRuntimeConfigResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, updateRuntimeConfig, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).UpdateRuntimeConfig(ctx, req.(*criv1.UpdateRuntimeConfigRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.UpdateRuntimeConfigResponse), err\n}\n\nfunc (s *server) Status(ctx context.Context,\n\treq *criv1.StatusRequest) (*criv1.StatusResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, status, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).Status(ctx, req.(*criv1.StatusRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.StatusResponse), err\n}\n\nfunc (s *server) CheckpointContainer(ctx context.Context, req *criv1.CheckpointContainerRequest) (*criv1.CheckpointContainerResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, checkpointContainer, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).CheckpointContainer(ctx, req.(*criv1.CheckpointContainerRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.CheckpointContainerResponse), err\n}\n\nfunc (s *server) GetContainerEvents(req *criv1.GetEventsRequest, srv criv1.RuntimeService_GetContainerEventsServer) error {\n\t// TODO(klihub): interceptRequest is a unary interceptor. It can't handle streaming\n\t// requests so for now we short-circuit the call to the server here.\n\treturn (*s.runtime).GetContainerEvents(req, srv)\n}\n\nfunc (s *server) ListMetricDescriptors(ctx context.Context, req *criv1.ListMetricDescriptorsRequest) (*criv1.ListMetricDescriptorsResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, listMetricDescriptors, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ListMetricDescriptors(ctx, req.(*criv1.ListMetricDescriptorsRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListMetricDescriptorsResponse), err\n}\n\nfunc (s *server) ListPodSandboxMetrics(ctx context.Context, req *criv1.ListPodSandboxMetricsRequest) (*criv1.ListPodSandboxMetricsResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, listPodSandboxMetrics, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).ListPodSandboxMetrics(ctx, req.(*criv1.ListPodSandboxMetricsRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.ListPodSandboxMetricsResponse), err\n}\n\nfunc (s *server) RuntimeConfig(ctx context.Context, req *criv1.RuntimeConfigRequest) (*criv1.RuntimeConfigResponse, error) {\n\trsp, err := s.interceptRequest(ctx, runtimeService, runtimeConfig, req,\n\t\tfunc(ctx context.Context, req interface{}) (interface{}, error) {\n\t\t\treturn (*s.runtime).RuntimeConfig(ctx, req.(*criv1.RuntimeConfigRequest))\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rsp.(*criv1.RuntimeConfigResponse), err\n}\n"
  },
  {
    "path": "pkg/dump/doc.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dump\n\n//\n// This package implements the dumping of (gRPC) methods calls where\n// each method is called with a single request struct and returns a\n// single reply struct or an error. Configuring what to dump happens\n// by specifying a comma-separated dump request on the command line.\n//\n// A dump request is a comma-separated list of dump specs:\n//     <spec>[,<spec>,...,<spec>], where each spec is of the form\n//     <[target:]request>\n// A request is either a requests name (gRPC method name without\n// the leading path), or a regexp for matching requests.\n// The dump targets are: 'off', 'name', 'full', 'count' by default.\n//\n\nvar configHelp = `\nDump CRI gRPC method calls as YAML.\n\nThis package implements configurable message dumping of CRI gRPC\nmethod calls. Both requests and the resulting replies or errors\ncan be dumped. Messages can be both logged and dumped to a given\nfile.\n\nConfiguring what to dumps happens using a dump configuration string\nof the following format:\n\n  level1:pattern1[,level2:pattern2,...][,debug]\n\nEach level specifies a level of detail for method calls with names\nmatching the corresponding pattern. A pattern can be a method call\nname to match just a single method, or it can be regexp to match\nseveral methods. For regexps all the patterns are evaluated in order\nof appearance with the last one staying in effect. Exact method name\npatterns terminate the evaluation without any regexp processing.\n\nThe possible levels of duping detail are:\n\n  off: suppress dumping of matching requests and replies\n  short: short dump of requests and potential error replies\n  full: full dump of both request and reply content as YAML\n\nAdditionally including 'debug' in the configuration string will\ncause messages to be logged as debug messages with the 'message'\nlog source. Note that debugging for this source needs to be\nexplicitly enabled, otherwise messages are suppressed.\n\nIf a dump file is specified messages will be dumped additionally\nto the dump file as well.\n\nHere is a sample configuration fragment to suppress all .*List.*\ncalls, produce short dumps of all .*Stop.* calls, and full dumps\nof everything else, dumps also going to the file '/tmp/cri-dump.log'\n\n  dump:\n    config: full:.*,short:.*Stop.*,off:.*List.*\n    file: /tmp/cri-dump.log\n`\n"
  },
  {
    "path": "pkg/dump/dump.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dump\n\n//\n// This package implements the dumping of (gRPC) methods calls where\n// each method is called with a single request struct and returns a\n// single reply struct or an error. Configuring what to dump happens\n// by specifying a comma-separated dump request on the command line.\n//\n// A dump request is a comma-separated list of dump specs:\n//     <spec>[,<spec>,...,<spec>], where each spec is of the form\n//     <[target:]request>\n// A request is either a requests name (gRPC method name without\n// the leading path), or a regexp for matching requests.\n// The dump targets are: 'off', 'name', 'full', 'count' by default.\n//\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sigs.k8s.io/yaml\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// stampLayout is the timestamp format used in dump files.\n\tstampLayout = \"2006-Jan-02 15:04:05.000\"\n\t// stampLen is the length we adjust our printed latencies to.\n\tstampLen = len(stampLayout)\n)\n\n// dumper encapsulates the runtime state of our message dumper.\ntype dumper struct {\n\tsync.RWMutex                  // protect concurrent dumping/reconfiguration\n\trules        ruleset          // dumping rules\n\tdetails      map[string]level // corresponding dump details per method\n\tdisabled     bool             // dumping globally disabled\n\tdebug        bool             // dump as debug messages\n\tpath         string           // extra dump file path\n\tfile         *os.File         // extra dump file\n\tmethods      []string         // training set for config\n\tq            chan *dumpreq\n}\n\n// dumpreq is a request to dump a (CRI) request or a reply\ntype dumpreq struct {\n\tdir       direction\n\tkind      string\n\tmethod    string\n\tqualifier string\n\tmsg       interface{}\n\tlatency   time.Duration\n\tsync      chan struct{}\n}\n\n// direction is a message direction, a request or a reply\ntype direction int\n\nconst (\n\trequest = iota\n\treply\n\tnop\n)\n\n// Our global dumper instance.\nvar dump = newDumper()\n\n// Our logger instances, one for generic logging and another for message dumps.\nvar log = logger.NewLogger(\"dump\")\nvar message = logger.NewLogger(\"message\")\n\n// Train trains the message dumper for the given set of methods.\nfunc Train(methods []string) {\n\tdump.Lock()\n\tdefer dump.Unlock()\n\tdump.train(methods)\n}\n\n// RequestMessage dumps a CRI request.\nfunc RequestMessage(kind, name, qualifier string, req interface{}, sync bool) {\n\tif !dump.disabled {\n\t\tvar ch chan struct{}\n\t\tif sync {\n\t\t\tch = make(chan struct{})\n\t\t}\n\t\tdump.q <- &dumpreq{\n\t\t\tdir:       request,\n\t\t\tkind:      kind,\n\t\t\tmethod:    name,\n\t\t\tqualifier: qualifier,\n\t\t\tmsg:       req,\n\t\t\tsync:      ch,\n\t\t}\n\t\tif ch != nil {\n\t\t\t_ = <-ch\n\t\t}\n\t}\n}\n\n// ReplyMessage dumps a CRI reply.\nfunc ReplyMessage(kind, name, qualifier string, rpl interface{}, latency time.Duration, sync bool) {\n\tif !dump.disabled {\n\t\tvar ch chan struct{}\n\t\tif sync {\n\t\t\tch = make(chan struct{})\n\t\t}\n\t\tdump.q <- &dumpreq{\n\t\t\tdir:       reply,\n\t\t\tkind:      kind,\n\t\t\tmethod:    name,\n\t\t\tqualifier: qualifier,\n\t\t\tmsg:       rpl,\n\t\t\tlatency:   latency,\n\t\t\tsync:      ch,\n\t\t}\n\t\tif ch != nil {\n\t\t\t_ = <-ch\n\t\t}\n\t}\n}\n\n// Sync returns once the last message currently being dumped is finished.\nfunc Sync() {\n\tif !dump.disabled {\n\t\tdump.sync()\n\t}\n}\n\n// newDumper creates a dumper instance.\nfunc newDumper() *dumper {\n\td := &dumper{q: make(chan *dumpreq, 16)}\n\td.run()\n\treturn d\n}\n\n// run runs the dumping goroutine of the dumper.\nfunc (d *dumper) run() {\n\tgo func() {\n\t\tfor req := range d.q {\n\t\t\tif req.dir != nop {\n\t\t\t\tmethod := methodName(req.method)\n\t\t\t\td.RLock()\n\t\t\t\tdetail, ok := d.details[method]\n\t\t\t\tif !ok {\n\t\t\t\t\tdetail = d.rules.detailOf(method)\n\t\t\t\t}\n\t\t\t\td.RUnlock()\n\t\t\t\tswitch detail {\n\t\t\t\tcase Name:\n\t\t\t\t\td.name(req.dir, req.kind, method, req.qualifier, req.msg, req.latency)\n\t\t\t\tcase Full:\n\t\t\t\t\td.full(req.dir, req.kind, method, req.qualifier, req.msg, req.latency)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif req.sync != nil {\n\t\t\t\tclose(req.sync)\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// sync waits until all the persent messages in the queue are dumped.\nfunc (d *dumper) sync() {\n\tch := make(chan struct{})\n\tdump.q <- &dumpreq{dir: nop, sync: ch}\n\t_ = <-ch\n}\n\n// configure (re)configures dumper\nfunc (d *dumper) configure(o *options) {\n\td.Lock()\n\tdefer d.Unlock()\n\n\td.debug = o.Debug\n\td.rules = o.rules.duplicate()\n\n\tif d.path != o.File || d.disabled != o.Disabled {\n\t\tif d.file != nil {\n\t\t\tlog.Info(\"closing old message dump file %q...\", d.path)\n\t\t\td.file.Close()\n\t\t\td.file = nil\n\t\t}\n\t\td.disabled = o.Disabled\n\n\t\tif d.disabled {\n\t\t\treturn\n\t\t}\n\n\t\td.path = o.File\n\t\tif d.path != \"\" {\n\t\t\tvar err error\n\t\t\tlog.Info(\"opening new message dump file %q...\", d.path)\n\t\t\td.file, err = os.Create(d.path)\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(\"failed to open file %q: %v\", d.path, err)\n\t\t\t}\n\t\t}\n\t}\n\n\td.train(nil)\n}\n\n// train trains the dumper with the given set of messages.\nfunc (d *dumper) train(names []string) {\n\tif names != nil {\n\t\td.methods = make([]string, len(names), len(names))\n\t} else {\n\t\tnames = d.methods\n\t}\n\td.details = make(map[string]level)\n\tfor idx, name := range names {\n\t\tmethod := methodName(name)\n\t\tdetail := d.rules.detailOf(method)\n\t\tlog.Info(\"%s: %v\", method, detail)\n\t\td.methods[idx] = method\n\t\td.details[method] = detail\n\t}\n}\n\n// name does a name-only dump of the given message.\nfunc (d *dumper) name(dir direction, kind, method, qualifier string, msg interface{}, latency time.Duration) {\n\tvar hdr string\n\n\tswitch dir {\n\tcase request:\n\t\treturn\n\tcase reply:\n\t\tif qualifier != \"\" {\n\t\t\thdr = qualifier + \" \" + method + \" \" + dir.arrow() + \" \"\n\t\t} else {\n\t\t\thdr = method + \" \" + dir.arrow() + \" \"\n\t\t}\n\t\tif err, ok := msg.(error); ok {\n\t\t\td.warn(dir, latency, hdr+\"(%s) FAILED: %v\", kind, err)\n\t\t} else {\n\t\t\td.line(dir, latency, hdr+\"(%s) REQUEST\", kind)\n\t\t}\n\t}\n}\n\n// full does a full dump of the given message.\nfunc (d *dumper) full(dir direction, kind, method, qualifier string, msg interface{}, latency time.Duration) {\n\tvar hdr string\n\n\tif qualifier != \"\" {\n\t\thdr = qualifier + \" \" + method + \" \" + dir.arrow() + \" \"\n\t} else {\n\t\thdr = method + \" \" + dir.arrow() + \" \"\n\t}\n\n\tswitch dir {\n\tcase request:\n\t\traw, _ := yaml.Marshal(msg)\n\t\tstr := strings.TrimRight(string(raw), \"\\n\")\n\t\tif strings.LastIndexByte(str, '\\n') > 0 {\n\t\t\td.line(dir, latency, hdr+\"(%s) REQUEST\", kind)\n\t\t\td.block(dir, latency, hdr+\"    \", str)\n\t\t} else {\n\t\t\td.line(dir, latency, hdr+\"(%s) REQUEST %s\", kind, str)\n\t\t}\n\n\tcase reply:\n\t\tif err, ok := msg.(error); ok {\n\t\t\td.warn(dir, latency, hdr+\"(%s) FAILED\", kind)\n\t\t\td.warn(dir, latency, hdr+\"    %v\", err)\n\t\t} else {\n\t\t\traw, _ := yaml.Marshal(msg)\n\t\t\tstr := strings.TrimRight(string(raw), \"\\n\")\n\t\t\tif strings.LastIndexByte(str, '\\n') > 0 {\n\t\t\t\td.line(dir, latency, hdr+\"(%s) REPLY\", kind)\n\t\t\t\td.block(dir, latency, hdr+\"    \", str)\n\t\t\t} else {\n\t\t\t\td.line(dir, latency, hdr+\"(%s) REPLY %s\", kind, str)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// line dumps a single line.\nfunc (d *dumper) line(dir direction, latency time.Duration, format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif !d.debug {\n\t\tmessage.Info(\"%s\", msg)\n\t} else {\n\t\tmessage.Debug(\"%s\", msg)\n\t}\n\tif d.file != nil {\n\t\td.tofile(dir, latency, \"%s\", msg)\n\t}\n}\n\n// block dumps a block of lines.\nfunc (d *dumper) block(dir direction, latency time.Duration, prefix, msg string) {\n\tif !d.debug {\n\t\tmessage.InfoBlock(prefix, msg)\n\t} else {\n\t\tmessage.DebugBlock(prefix, msg)\n\t}\n\tif d.file != nil {\n\t\tfor _, line := range strings.Split(msg, \"\\n\") {\n\t\t\td.tofile(dir, latency, \"%s%s\", prefix, line)\n\t\t}\n\t}\n}\n\n// warn dumps a single line as a warning.\nfunc (d *dumper) warn(dir direction, latency time.Duration, format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tmessage.Warn(\"%s\", msg)\n\tif d.file != nil {\n\t\td.tofile(dir, latency, \"%s\", msg)\n\t}\n}\n\n// tofile dumps a single line to a file.\nfunc (d *dumper) tofile(dir direction, latency time.Duration, format string, args ...interface{}) {\n\tfmt.Fprintf(d.file, \"[\"+stamp(dir, latency)+\"] \"+format+\"\\n\", args...)\n}\n\n// stamp produces a stamp from a direction and a latency.\nfunc stamp(dir direction, latency time.Duration) string {\n\tswitch dir {\n\tcase request:\n\t\treturn time.Now().Format(stampLayout)\n\tcase reply:\n\t\treturn fmt.Sprintf(\"%*s\", stampLen, fmt.Sprintf(\"+%f\", latency.Seconds()))\n\t}\n\treturn \"\"\n}\n\n// String returns a string representing the direction.\nfunc (d direction) String() string {\n\tswitch d {\n\tcase request:\n\t\treturn \"request\"\n\tcase reply:\n\t\treturn \"reply\"\n\t}\n\treturn \"unknown\"\n}\n\n// arrow returns an 'ASCII arrow' for the direction.\nfunc (d direction) arrow() string {\n\tswitch d {\n\tcase request:\n\t\treturn \"=>\"\n\tcase reply:\n\t\treturn \"<=\"\n\t}\n\treturn \"<=???=>\"\n}\n\n// methodName returns the basename of a method.\nfunc methodName(method string) string {\n\treturn method[strings.LastIndex(method, \"/\")+1:]\n}\n\n// dumpError produces a formatted package-specific error.\nfunc dumpError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"dump: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/dump/dump_test.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dump\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\n// TestConfigParsing test parsing of dump configuration strings.\nfunc TestConfigParsing(t *testing.T) {\n\ttcases := []string{\n\t\tDefaultConfig,\n\t\t\"off:.*\",\n\t\t\"full:.*\",\n\t\t\"name:.*\",\n\t\t\"off:.*,full:CreateContainer,StartContainer,StopContainer,RemoveContainer\",\n\t\t\"off:.*,full:.*((PodSandbox)|(Container)),off:.*((Status)|(List)).*\",\n\t}\n\n\tfor _, cfg := range tcases {\n\t\tt.Run(\"parse config \"+cfg, func(t *testing.T) {\n\t\t\tr := ruleset{}\n\t\t\tif err := r.parse(cfg); err != nil {\n\t\t\t\tt.Errorf(\"failed to parse dump config string '%s': %v\", cfg, err)\n\t\t\t}\n\t\t\tif chk := r.String(); chk != cfg {\n\t\t\t\tswitch {\n\t\t\t\tcase strings.Replace(cfg, \"short:\", \"name:\", 1) == chk:\n\t\t\t\tcase strings.Replace(cfg, \"suppress:\", \"off:\", 1) == chk:\n\t\t\t\tcase strings.Replace(cfg, \"verbose:\", \"full:\", 1) == chk:\n\t\t\t\tdefault:\n\t\t\t\t\tt.Errorf(\"expected %s, got %s\", cfg, chk)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// TestFiltering test message filtering, and a bit of formatting.\nfunc fooTestFiltering(t *testing.T) {\n\tmessages := []interface{}{\n\t\tmkmsg(&Type1Message1{}),\n\t\tmkmsg(&Type1Message2{}),\n\t\tmkmsg(&Type1Message3{}),\n\t\tmkmsg(&Type1Whatever{}),\n\t\tmkmsg(&Type2Message1{}),\n\t\tmkmsg(&Type2Message2{}),\n\t\tmkmsg(&Type2Message3{}),\n\t\tmkmsg(&Type2Whatever{}),\n\t\tmkmsg(&Type3Message1{}),\n\t\tmkmsg(&Type3Message2{}),\n\t\tmkmsg(&Type3Message3{}),\n\t\tmkmsg(&Type3Whatever{}),\n\t}\n\n\ttcases := []filterTest{\n\t\t{\n\t\t\tmessages: messages,\n\t\t\tconfig:   \"off:.*\",\n\t\t},\n\t\t{\n\t\t\tmessages: messages,\n\t\t\tconfig:   \"name:Type1.*\",\n\t\t\tdetails: map[string]level{\n\t\t\t\tmsgmethod(&Type1Message1{}): Name,\n\t\t\t\tmsgmethod(&Type1Message2{}): Name,\n\t\t\t\tmsgmethod(&Type1Message3{}): Name,\n\t\t\t\tmsgmethod(&Type1Whatever{}): Name,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmessages: messages,\n\t\t\tconfig:   \"full:.*Whatever.*\",\n\t\t\tdetails: map[string]level{\n\t\t\t\tmsgmethod(&Type1Whatever{}): Full,\n\t\t\t\tmsgmethod(&Type2Whatever{}): Full,\n\t\t\t\tmsgmethod(&Type3Whatever{}): Full,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmessages: messages,\n\t\t\tconfig:   \"full:.*Whatever.*,off:Type1.*\",\n\t\t\tdetails: map[string]level{\n\t\t\t\tmsgmethod(&Type2Whatever{}): Full,\n\t\t\t\tmsgmethod(&Type3Whatever{}): Full,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tmessages: messages,\n\t\t\tconfig:   \"full:.*Message.*,off:Type2.*,name:Type2Whatever\",\n\t\t\tdetails: map[string]level{\n\t\t\t\tmsgmethod(&Type1Message1{}): Full,\n\t\t\t\tmsgmethod(&Type1Message2{}): Full,\n\t\t\t\tmsgmethod(&Type1Message3{}): Full,\n\t\t\t\tmsgmethod(&Type2Whatever{}): Name,\n\t\t\t\tmsgmethod(&Type3Message1{}): Full,\n\t\t\t\tmsgmethod(&Type3Message2{}): Full,\n\t\t\t\tmsgmethod(&Type3Message3{}): Full,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range tcases {\n\t\tt.Run(\"filter with config \"+tc.config, func(t *testing.T) {\n\t\t\ttc.run(t)\n\t\t})\n\t}\n}\n\ntype filterTest struct {\n\tmessages []interface{}\n\tconfig   string\n\tdetails  map[string]level\n}\n\nconst (\n\t// test log marker to identify logged messages\n\tmarker = \"<testmsg>\"\n)\n\nfunc (ft *filterTest) setup(train bool) *testlog {\n\t// override message logger\n\tlogger := &testlog{}\n\tmessage = logger\n\n\t// create training set/reset messages\n\tmethods := []string{}\n\tif train {\n\t\tfor _, msg := range ft.messages {\n\t\t\tmethods = append(methods, msgname(msg))\n\t\t}\n\t}\n\tTrain(methods)\n\n\t// trigger reconfiguration\n\topt.Config = ft.config\n\topt.configNotify(config.UpdateEvent, config.ConfigFile)\n\n\treturn logger\n}\n\nfunc (ft *filterTest) dumpMessages(logger *testlog) []string {\n\t// dump all test messages and a fake reply for each\n\tfor _, msg := range ft.messages {\n\t\tRequestMessage(marker, msgname(msg), \"\", msg, false)\n\t\tReplyMessage(marker, msgname(msg), \"\", Reply, time.Duration(0), false)\n\t}\n\tdump.sync()\n\n\treturn logger.info\n}\n\nfunc (ft *filterTest) parseLogs(t *testing.T, logged []string) (map[string]int, map[string]int) {\n\t// count logged entries and lines per message\n\tlines := map[string]int{}\n\tentries := map[string]int{}\n\tfor _, entry := range logged {\n\t\tentry = strings.Trim(entry, \" \")\n\t\tsplit := strings.Split(entry, \" \")\n\t\tmethod := \"\"\n\t\tswitch {\n\t\t// log line: (marker) {REQUEST|REPLY} method\n\t\tcase len(split) > 1 && split[0] == \"(\"+marker+\")\":\n\t\t\tmethod = split[2]\n\t\t\tentries[method] = entries[method] + 1\n\t\tcase len(split) > 1:\n\t\t\t// log line continuation: method {=>|<=} content...\n\t\t\tmethod = split[0]\n\t\t}\n\t\tif method == \"\" {\n\t\t\tt.Errorf(\"failed to parse log entry '%s' for config '%s'\", entry, ft.config)\n\t\t}\n\n\t\tdetail, ok := ft.details[method]\n\t\tif !ok || detail == Off {\n\t\t\tt.Errorf(\"message '%s' should have been filtered for config '%s'\",\n\t\t\t\tmethod, ft.config)\n\t\t}\n\t}\n\n\treturn lines, entries\n}\n\nfunc (ft *filterTest) checkResult(t *testing.T, entries map[string]int, lines map[string]int) {\n\t// check correctness of logged entries and lines per method\n\tfor method, lineCnt := range lines {\n\t\tlogcnt := entries[method]\n\t\texpected := 0\n\t\tswitch ft.details[method] {\n\t\tcase Full:\n\t\t\texpected = logcnt/2*(1+LinesPerRequest) + logcnt/2*(1+LinesPerReply)\n\t\tcase Name:\n\t\t\texpected = logcnt\n\t\t}\n\t\tif lineCnt != expected {\n\t\t\tt.Errorf(\"message '%s' expected %d logged lines, got %d for config '%s'\",\n\t\t\t\tmethod, expected, lineCnt, ft.config)\n\t\t}\n\t}\n}\n\nfunc (ft *filterTest) run(t *testing.T) {\n\tfor _, train := range []bool{false, true} {\n\t\tlogger := ft.setup(train)\n\t\tlogged := ft.dumpMessages(logger)\n\t\tlines, entries := ft.parseLogs(t, logged)\n\n\t\tft.checkResult(t, entries, lines)\n\t}\n}\n\n//\n// a few message types for testing\n//\n\ntype Message struct {\n\tBody []string\n}\n\ntype Type1Message1 Message\ntype Type1Message2 Message\ntype Type1Message3 Message\ntype Type1Whatever Message\ntype Type2Message1 Message\ntype Type2Message2 Message\ntype Type2Message3 Message\ntype Type2Whatever Message\ntype Type3Message1 Message\ntype Type3Message2 Message\ntype Type3Message3 Message\ntype Type3Whatever Message\n\nconst (\n\tLinesPerRequest = 6\n\tLinesPerReply   = 2\n)\n\nvar (\n\tReply  = []string{\"reply\", \"OK\"}\n\tmsgCnt int\n)\n\nfunc mkmsg(o interface{}) interface{} {\n\tmsgCnt++\n\tbody := []string{\n\t\t\"this\",\n\t\t\"is\",\n\t\t\"message\",\n\t\tfmt.Sprintf(\"#%d\", msgCnt),\n\t\tfmt.Sprintf(\"of type (%T)\", o),\n\t}\n\n\tswitch o.(type) {\n\tcase *Type1Message1:\n\t\tm := o.(*Type1Message1)\n\t\tm.Body = body\n\tcase *Type1Message2:\n\t\tm := o.(*Type1Message2)\n\t\tm.Body = body\n\tcase *Type1Message3:\n\t\tm := o.(*Type1Message3)\n\t\tm.Body = body\n\tcase *Type1Whatever:\n\t\tm := o.(*Type1Whatever)\n\t\tm.Body = body\n\n\tcase *Type2Message1:\n\t\tm := o.(*Type2Message1)\n\t\tm.Body = body\n\tcase *Type2Message2:\n\t\tm := o.(*Type2Message2)\n\t\tm.Body = body\n\tcase *Type2Message3:\n\t\tm := o.(*Type2Message3)\n\t\tm.Body = body\n\tcase *Type2Whatever:\n\t\tm := o.(*Type2Whatever)\n\t\tm.Body = body\n\n\tcase *Type3Message1:\n\t\tm := o.(*Type3Message1)\n\t\tm.Body = body\n\tcase *Type3Message2:\n\t\tm := o.(*Type3Message2)\n\t\tm.Body = body\n\tcase *Type3Message3:\n\t\tm := o.(*Type3Message3)\n\t\tm.Body = body\n\tcase *Type3Whatever:\n\t\tm := o.(*Type3Whatever)\n\t\tm.Body = body\n\t}\n\n\treturn o\n}\n\nfunc msgname(o interface{}) string {\n\treturn strings.ReplaceAll(fmt.Sprintf(\"%T\", o), \".\", \"/\")\n}\n\nfunc msgmethod(o interface{}) string {\n\treturn methodName(msgname(o))\n}\n\n//\n// test logger to override and check dumping/logging for test.\n//\n\ntype testlog struct {\n\tsync.Mutex\n\tinfo  []string\n\twarn  []string\n\terr   []string\n\tdebug []string\n}\n\nfunc (t *testlog) reset() {\n\tt.Lock()\n\tdefer t.Unlock()\n\tt.info = nil\n\tt.warn = nil\n\tt.err = nil\n\tt.debug = nil\n}\n\nfunc (t *testlog) log(save *[]string, prefix, format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\t*save = append(*save, msg)\n\tfmt.Println(\"<dump-test> \" + prefix + \" \" + msg)\n}\n\nfunc (t *testlog) Info(format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tt.log(&t.info, \"I:\", format, args...)\n}\n\nfunc (t *testlog) Warn(format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tt.log(&t.warn, \"W:\", format, args...)\n}\n\nfunc (t *testlog) Error(format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tt.log(&t.err, \"E:\", format, args...)\n}\n\nfunc (t *testlog) Debug(format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tt.log(&t.debug, \"D:\", format, args...)\n}\n\nfunc (t *testlog) Fatal(format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tfmt.Printf(\"<dump-test> Fatal error: %s\\n\", msg)\n\tos.Exit(1)\n}\n\nfunc (*testlog) Panic(format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tfmt.Printf(\"<dump-test> Panic: %s\\n\", msg)\n\tpanic(msg)\n}\n\nfunc (t *testlog) Infof(format string, args ...interface{}) {\n\tt.Info(format, args...)\n}\n\nfunc (t *testlog) Warnf(format string, args ...interface{}) {\n\tt.Warn(format, args...)\n}\n\nfunc (t *testlog) Errorf(format string, args ...interface{}) {\n\tt.Error(format, args...)\n}\n\nfunc (t *testlog) Debugf(format string, args ...interface{}) {\n\tt.Debug(format, args...)\n}\n\nfunc (t *testlog) Fatalf(format string, args ...interface{}) {\n\tt.Fatal(format, args...)\n}\n\nfunc (t *testlog) Panicf(format string, args ...interface{}) {\n\tt.Panic(format, args...)\n}\n\nfunc (*testlog) Block(fn func(string, ...interface{}), prfx string, frmt string, a ...interface{}) {\n\tfor _, line := range strings.Split(fmt.Sprintf(frmt, a...), \"\\n\") {\n\t\tfn(\"%s%s\", prfx, line)\n\t}\n}\n\nfunc (t *testlog) InfoBlock(prefix string, format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tfor _, line := range strings.Split(fmt.Sprintf(format, args...), \"\\n\") {\n\t\tt.log(&t.info, \"I:\", \"%s%s\", prefix, line)\n\t}\n}\n\nfunc (t *testlog) WarnBlock(prefix string, format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tfor _, line := range strings.Split(fmt.Sprintf(format, args...), \"\\n\") {\n\t\tt.log(&t.info, \"W:\", \"%s%s\", prefix, line)\n\t}\n}\n\nfunc (t *testlog) ErrorBlock(prefix string, format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tfor _, line := range strings.Split(fmt.Sprintf(format, args...), \"\\n\") {\n\t\tt.log(&t.err, \"E:\", \"%s%s\", prefix, line)\n\t}\n}\n\nfunc (t *testlog) DebugBlock(prefix string, format string, args ...interface{}) {\n\tt.Lock()\n\tdefer t.Unlock()\n\tfor _, line := range strings.Split(fmt.Sprintf(format, args...), \"\\n\") {\n\t\tt.log(&t.debug, \"I:\", \"%s%s\", prefix, line)\n\t}\n}\n\nfunc (*testlog) EnableDebug() bool  { return true }\nfunc (*testlog) DebugEnabled() bool { return true }\nfunc (*testlog) Stop()              {}\nfunc (*testlog) Source() string     { return \"\" }\n"
  },
  {
    "path": "pkg/dump/flags.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage dump\n\n//\n// This package implements the dumping of (gRPC) methods calls where\n// each method is called with a single request struct and returns a\n// single reply struct or an error. Configuring what to dump happens\n// by specifying a comma-separated dump request on the command line.\n//\n// A dump request is a comma-separated list of dump specs:\n//     <spec>[,<spec>,...,<spec>], where each spec is of the form\n//     <[target:]request>\n// A request is either a requests name (gRPC method name without\n// the leading path), or a regexp for matching requests.\n// The dump targets are: 'off', 'name', 'full', 'count' by default.\n//\n\nimport (\n\t\"fmt\"\n\tre \"regexp\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n)\n\nconst (\n\t// DefaultConfig is the default dump configuration.\n\tDefaultConfig = \"off:.*,short:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\"\n)\n\n// Dumping options configurable via the command line or pkg/config.\ntype options struct {\n\tDebug    bool    // log messages as debug messages\n\tDisabled bool    // whether dumping is globally disabled\n\tFile     string  // file to also dump to, if set\n\tConfig   string  // dumping configuration\n\trules    ruleset // corresponding dumping rules\n}\n\n// ruleset is an ordered set of dumping rules.\ntype ruleset []*rule\n\n// rule is a single dumping rule, declaring verbosity of a single or a set of methods.\ntype rule struct {\n\tmethod string     // method, '*' wildcard, or regexp matching a set of methods\n\tregexp *re.Regexp // compiled regexp, if applicable\n\tdetail level      // dumping verbosity\n}\n\n// level describes the level of detail to dump.\ntype level int\n\nconst (\n\t// Off suppresses dumping of matching methods\n\tOff level = iota\n\t// Name dumps only success/failure status of matching methods.\n\tName\n\t// Full dumps matching methods with full level of detail.\n\tFull\n)\n\n// Our runtime configuration.\nvar opt = defaultOptions().(*options)\n\n// parse parses the given string into a ruleset.\nfunc (set *ruleset) parse(value string) error {\n\tprev := Full\n\tfor _, spec := range strings.Split(value, \",\") {\n\t\tr := &rule{}\n\t\tsplit := strings.SplitN(spec, \":\", 2)\n\t\tswitch len(split) {\n\t\tcase 1:\n\t\t\tr.detail = prev\n\t\t\tr.method = split[0]\n\t\tcase 2:\n\t\t\tswitch strings.ToLower(split[0]) {\n\t\t\tcase \"off\", \"suppress\":\n\t\t\t\tr.detail = Off\n\t\t\tcase \"name\", \"short\":\n\t\t\t\tr.detail = Name\n\t\t\tcase \"full\", \"verbose\":\n\t\t\t\tr.detail = Full\n\t\t\tdefault:\n\t\t\t\treturn dumpError(\"invalid dump level '%s'\", split[0])\n\t\t\t}\n\t\t\tr.method = split[1]\n\t\t\tprev = r.detail\n\t\t}\n\n\t\tif strings.ContainsAny(r.method, \".*?+()[]|\") && r.method != \"*\" {\n\t\t\tregexp, err := re.Compile(r.method)\n\t\t\tif err != nil {\n\t\t\t\treturn dumpError(\"invalid dump method regexp '%s': %v\", r.method, err)\n\t\t\t}\n\t\t\tr.regexp = regexp\n\t\t}\n\t\t*set = append(*set, r)\n\t}\n\n\treturn nil\n}\n\n// String returns the ruleset as a string.\nfunc (set *ruleset) String() string {\n\tif set == nil || *set == nil {\n\t\treturn \"\"\n\t}\n\tprev := Off\n\tvalue, sep := \"\", \"\"\n\tfor idx, r := range *set {\n\t\tdetail := \"\"\n\t\tif idx == 0 || r.detail != prev {\n\t\t\tdetail = r.detail.String() + \":\"\n\t\t}\n\t\tvalue += sep + detail + r.method\n\t\tsep = \",\"\n\t\tprev = r.detail\n\t}\n\treturn value\n}\n\n// detailOf returns the level of detail for dumping the given method.\nfunc (set *ruleset) detailOf(method string) level {\n\tlog.Debug(\"%s: checking level of detail...\", method)\n\tif set == nil {\n\t\treturn Off\n\t}\n\tdetail := Off\n\tfor _, r := range *set {\n\t\tlog.Debug(\"  - checking rule '%s'...\", r.method)\n\t\tswitch {\n\t\tcase r.method == method:\n\t\t\tlog.Debug(\"    => exact match: %v\", r.detail)\n\t\t\treturn r.detail\n\t\tcase r.method == \"*\":\n\t\t\tlog.Debug(\"    => wildcard match: %v\", r.detail)\n\t\t\tdetail = r.detail\n\t\tcase r.regexp != nil && r.regexp.MatchString(method):\n\t\t\tlog.Debug(\"    => regexp match (%s): %v\", r.method, r.detail)\n\t\t\tdetail = r.detail\n\t\t}\n\t}\n\treturn detail\n}\n\n// copy creates a (shallow) copy of the ruleset.\nfunc (set *ruleset) duplicate() ruleset {\n\tif set == nil || *set == nil {\n\t\treturn nil\n\t}\n\tcp := make([]*rule, len(*set))\n\tcopy(cp, *set)\n\treturn cp\n}\n\n// String returns the level of detail as a string.\nfunc (detail level) String() string {\n\tswitch detail {\n\tcase Off:\n\t\treturn \"off\"\n\tcase Name:\n\t\treturn \"name\"\n\tcase Full:\n\t\treturn \"full\"\n\t}\n\treturn fmt.Sprintf(\"<invalid dump level of detail %d>\", detail)\n}\n\n// defaultOptions returns a new options instance, initialized to defaults.\nfunc defaultOptions() interface{} {\n\to := &options{Config: DefaultConfig}\n\to.rules.parse(DefaultConfig)\n\treturn o\n}\n\n// configNotify updates our runtime configuration.\nfunc (o *options) configNotify(event config.Event, _ config.Source) error {\n\tlog.Info(\"message dumper configuration %v\", event)\n\tlog.Info(\" * config: %s\", o.Config)\n\n\trules := ruleset{}\n\tif err := rules.parse(o.Config); err != nil {\n\t\treturn err\n\t}\n\n\to.rules = rules\n\n\tlog.Info(\" * parsed: %s\", o.rules.String())\n\tlog.Info(\" * dump file: %v\", opt.File)\n\tlog.Info(\" * log with debug: %v\", opt.Debug)\n\n\tdump.configure(o)\n\n\treturn nil\n}\n\n// Register us for command line parsing and configuration handling.\nfunc init() {\n\topt.rules.parse(opt.Config)\n\tconfig.Register(\"dump\", configHelp, opt, defaultOptions,\n\t\tconfig.WithNotify(opt.configNotify))\n}\n"
  },
  {
    "path": "pkg/instrumentation/flags.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"go.opencensus.io/trace\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n)\n\n// Sampling defines how often trace samples are taken.\ntype Sampling float64\n\nconst (\n\t// Disabled is the trace configuration for disabling tracing.\n\tDisabled Sampling = 0.0\n\t// Production is a trace configuration for production use.\n\tProduction Sampling = 0.1\n\t// Testing is a trace configuration for testing.\n\tTesting Sampling = 1.0\n\n\t// defaultSampling is the default sampling frequency.\n\tdefaultSampling = \"0\"\n\t// defaultReportPeriod is the default report period\n\tdefaultReportPeriod = \"15s\"\n\t// defaultJaegerCollector is the default Jaeger collector endpoint.\n\tdefaultJaegerCollector = \"\"\n\t// defaultJaegerAgent is the default Jaeger agent endpoint.\n\tdefaultJaegerAgent = \"\"\n\t// defaultHTTPEndpoint is the default HTTP endpoint serving Prometheus /metrics.\n\tdefaultHTTPEndpoint = \"\"\n\t// defaultPrometheusExport is the default state for Prometheus exporting.\n\tdefaultPrometheusExport = \"false\"\n)\n\n// options encapsulates our configurable instrumentation parameters.\ntype options optstruct\n\ntype optstruct struct {\n\t// Sampling is the sampling frequency for traces.\n\tSampling Sampling\n\t// ReportPeriod is the OpenCensus view reporting period.\n\tReportPeriod time.Duration\n\t// jaegerCollector is the URL to the Jaeger HTTP Thrift collector.\n\tJaegerCollector string\n\t// jaegerAgent, if set, defines the address of a Jaeger agent to send spans to.\n\tJaegerAgent string\n\t// HTTPEndpoint is our HTTP endpoint, used among others to export Prometheus /metrics.\n\tHTTPEndpoint string\n\t// PrometheusExport defines whether we export /metrics to/for Prometheus.\n\tPrometheusExport bool `json:\"PrometheusExport\"`\n}\n\n// UnmarshalJSON is a resetting JSON unmarshaller for options.\nfunc (o *options) UnmarshalJSON(raw []byte) error {\n\tostruct := optstruct{}\n\tif err := json.Unmarshal(raw, &ostruct); err != nil {\n\t\treturn instrumentationError(\"failed to unmashal options: %v\", err)\n\t}\n\t*o = options(ostruct)\n\treturn nil\n}\n\n// Our instrumentation options.\nvar opt = defaultOptions().(*options)\n\n// MarshalJSON is the JSON marshaller for Sampling values.\nfunc (s Sampling) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(s.String())\n}\n\n// UnmarshalJSON is the JSON unmarshaller for Sampling values.\nfunc (s *Sampling) UnmarshalJSON(raw []byte) error {\n\tvar obj interface{}\n\tif err := json.Unmarshal(raw, &obj); err != nil {\n\t\treturn instrumentationError(\"failed to unmarshal Sampling value: %v\", err)\n\t}\n\tswitch v := obj.(type) {\n\tcase string:\n\t\tif err := s.Parse(v); err != nil {\n\t\t\treturn err\n\t\t}\n\tcase float64:\n\t\t*s = Sampling(v)\n\tdefault:\n\t\treturn instrumentationError(\"invalid Sampling value of type %T: %v\", obj, obj)\n\t}\n\treturn nil\n}\n\n// Parse parses the given string to a Sampling value.\nfunc (s *Sampling) Parse(value string) error {\n\tswitch strings.ToLower(value) {\n\tcase \"disabled\":\n\t\t*s = Disabled\n\tcase \"testing\":\n\t\t*s = Testing\n\tcase \"production\":\n\t\t*s = Production\n\tdefault:\n\t\tf, err := strconv.ParseFloat(value, 64)\n\t\tif err != nil {\n\t\t\treturn instrumentationError(\"invalid Sampling value '%s': %v\", value, err)\n\t\t}\n\t\t*s = Sampling(f)\n\t}\n\treturn nil\n}\n\n// String returns the Sampling value as a string.\nfunc (s Sampling) String() string {\n\tswitch s {\n\tcase Disabled:\n\t\treturn \"disabled\"\n\tcase Production:\n\t\treturn \"production\"\n\tcase Testing:\n\t\treturn \"testing\"\n\t}\n\treturn strconv.FormatFloat(float64(s), 'f', -1, 64)\n}\n\n// Sampler returns a trace.Sampler corresponding to the Sampling value.\nfunc (s Sampling) Sampler() trace.Sampler {\n\tif s == Disabled {\n\t\treturn trace.NeverSample()\n\t}\n\treturn trace.ProbabilitySampler(float64(s))\n}\n\n// parseEnv parses the environment for default values.\nfunc parseEnv(name, defval string, parsefn func(string) error) {\n\tif envval := os.Getenv(name); envval != \"\" {\n\t\terr := parsefn(envval)\n\t\tif err == nil {\n\t\t\treturn\n\t\t}\n\t\tlog.Error(\"invalid environment %s=%q: %v, using default %q\", name, envval, err, defval)\n\t}\n\tif err := parsefn(defval); err != nil {\n\t\tlog.Error(\"invalid default %s=%q: %v\", name, defval, err)\n\t}\n}\n\n// defaultOptions returns a new options instance, all initialized to defaults.\nfunc defaultOptions() interface{} {\n\to := &options{}\n\n\ttype param struct {\n\t\tdefval  string\n\t\tparsefn func(string) error\n\t}\n\n\tparams := map[string]param{\n\t\t\"JAEGER_COLLECTOR\": {\n\t\t\tdefaultJaegerCollector,\n\t\t\tfunc(v string) error { o.JaegerCollector = v; return nil },\n\t\t},\n\t\t\"JAEGER_AGENT\": {\n\t\t\tdefaultJaegerAgent,\n\t\t\tfunc(v string) error { o.JaegerAgent = v; return nil },\n\t\t},\n\t\t\"HTTP_ENDPOINT\": {\n\t\t\tdefaultHTTPEndpoint,\n\t\t\tfunc(v string) error { o.HTTPEndpoint = v; return nil },\n\t\t},\n\t\t\"PROMETHEUS_EXPORT\": {\n\t\t\tdefaultPrometheusExport,\n\t\t\tfunc(v string) error {\n\t\t\t\tenabled, err := utils.ParseEnabled(v)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\to.PrometheusExport = enabled\n\t\t\t\treturn nil\n\t\t\t},\n\t\t},\n\t\t\"SAMPLING_FREQUENCY\": {\n\t\t\tdefaultSampling,\n\t\t\tfunc(v string) error { return o.Sampling.Parse(v) },\n\t\t},\n\t\t\"REPORT_PERIOD\": {\n\t\t\tdefaultReportPeriod,\n\t\t\tfunc(v string) error {\n\t\t\t\td, err := time.ParseDuration(v)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\to.ReportPeriod = d\n\t\t\t\treturn nil\n\t\t\t},\n\t\t},\n\t}\n\n\tfor envvar, p := range params {\n\t\tparseEnv(envvar, p.defval, p.parsefn)\n\t}\n\n\treturn o\n}\n\n// configNotify is our configuration udpate notification handler.\nfunc configNotify(_ config.Event, _ config.Source) error {\n\tlog.Info(\"instrumentation configuration is now %v\", opt)\n\n\tlog.Info(\"reconfiguring...\")\n\tif err := svc.reconfigure(); err != nil {\n\t\tlog.Error(\"failed to restart instrumentation: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// Register us for for configuration handling.\nfunc init() {\n\tconfig.Register(\"instrumentation\", \"Instrumentation for traces and metrics.\",\n\t\topt, defaultOptions, config.WithNotify(configNotify))\n}\n"
  },
  {
    "path": "pkg/instrumentation/grpc.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"google.golang.org/grpc\"\n\n\t\"go.opencensus.io/plugin/ocgrpc\"\n\t\"go.opencensus.io/stats/view\"\n)\n\n// InjectGrpcClientTrace injects gRPC dial options for instrumentation if necessary.\nfunc InjectGrpcClientTrace(opts ...grpc.DialOption) []grpc.DialOption {\n\textra := grpc.WithStatsHandler(&ocgrpc.ClientHandler{})\n\n\tif len(opts) > 0 {\n\t\topts = append(opts, extra)\n\t} else {\n\t\topts = []grpc.DialOption{extra}\n\t}\n\n\treturn opts\n}\n\n// InjectGrpcServerTrace injects gRPC server options for instrumentation if necessary.\nfunc InjectGrpcServerTrace(opts ...grpc.ServerOption) []grpc.ServerOption {\n\textra := grpc.StatsHandler(&ocgrpc.ServerHandler{})\n\n\tif len(opts) > 0 {\n\t\topts = append(opts, extra)\n\t} else {\n\t\topts = []grpc.ServerOption{extra}\n\t}\n\n\treturn opts\n}\n\n// registerGrpcViews registers default client and server trace views for gRPC.\nfunc registerGrpcViews() error {\n\tlog.Debug(\"registering gRPC trace views...\")\n\n\tif err := view.Register(ocgrpc.DefaultClientViews...); err != nil {\n\t\treturn instrumentationError(\"failed to register default gRPC client views: %v\", err)\n\t}\n\tif err := view.Register(ocgrpc.DefaultServerViews...); err != nil {\n\t\treturn instrumentationError(\"failed to register default gRPC server views: %v\", err)\n\t}\n\n\treturn nil\n}\n\n// unregisterGrpcViews unregisters default client and server trace views for gRPC.\nfunc unregisterGrpcViews() {\n\tview.Unregister(ocgrpc.DefaultClientViews...)\n\tview.Unregister(ocgrpc.DefaultServerViews...)\n}\n"
  },
  {
    "path": "pkg/instrumentation/http/http.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage http\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\t\"sync\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// httpServer is used in log messages.\n\thttpServer = \"HTTP server\"\n)\n\n// Our logger instance.\nvar log = logger.NewLogger(\"http\")\n\n// ServeMux is our HTTP request multiplexer with removable handlers.\ntype ServeMux struct {\n\tsync.RWMutex\n\thandlers map[string]http.Handler\n\tmux      *http.ServeMux\n}\n\n// NewServeMux create a new HTTP request multiplexer.\nfunc NewServeMux() *ServeMux {\n\treturn &ServeMux{\n\t\thandlers: make(map[string]http.Handler),\n\t\tmux:      http.NewServeMux(),\n\t}\n}\n\n// Handle registers a handler for the given pattern.\nfunc (mux *ServeMux) Handle(pattern string, handler http.Handler) {\n\tmux.Lock()\n\tdefer mux.Unlock()\n\n\tlog.Debug(\"registering handler for %q...\", pattern)\n\n\tif _, ok := mux.handlers[pattern]; ok {\n\t\tlog.Error(\"can't register duplicate HTTP handler for %q\", pattern)\n\t\treturn\n\t}\n\n\tmux.handlers[pattern] = handler\n\tmux.mux.Handle(pattern, handler)\n}\n\n// HandleFunc registers a handler function for the given pattern.\nfunc (mux *ServeMux) HandleFunc(pattern string, fn func(http.ResponseWriter, *http.Request)) {\n\tmux.Lock()\n\tdefer mux.Unlock()\n\n\tlog.Debug(\"registering handler function for %q...\", pattern)\n\n\tif _, ok := mux.handlers[pattern]; ok {\n\t\tlog.Error(\"can't register duplicate HTTP handler function for '%s'\", pattern)\n\t\treturn\n\t}\n\n\thandler := http.HandlerFunc(fn)\n\n\tmux.handlers[pattern] = handler\n\tmux.mux.Handle(pattern, handler)\n}\n\n// Unregister unregister any handlers for the given pattern.\nfunc (mux *ServeMux) Unregister(pattern string) (http.Handler, bool) {\n\tmux.Lock()\n\tdefer mux.Unlock()\n\n\th, ok := mux.handlers[pattern]\n\tif !ok {\n\t\treturn nil, false\n\t}\n\n\tlog.Debug(\"unregistering handler for %q...\", pattern)\n\n\tdelete(mux.handlers, pattern)\n\tmux.mux = http.NewServeMux()\n\tfor pattern, handler := range mux.handlers {\n\t\tmux.mux.Handle(pattern, handler)\n\t}\n\n\treturn h, true\n}\n\n// ServeHTTP serves a HTTP request.\nfunc (mux *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tmux.RLock()\n\tdefer mux.RUnlock()\n\tlog.Debug(\"serving %s...\", r.URL)\n\tmux.mux.ServeHTTP(w, r)\n}\n\n// Server is our HTTP server, with support for unregistering handlers.\ntype Server struct {\n\tsync.RWMutex\n\tserver *http.Server\n\tmux    *ServeMux\n}\n\n// NewServer creates a new server instance.\nfunc NewServer() *Server {\n\treturn &Server{\n\t\tmux: NewServeMux(),\n\t}\n}\n\n// GetMux returns the mux for this server.\nfunc (s *Server) GetMux() *ServeMux {\n\treturn s.mux\n}\n\n// GetAddress returns the current server HTTP endpoint/address.\nfunc (s *Server) GetAddress() string {\n\tif s.server == nil {\n\t\treturn \"\"\n\t}\n\treturn s.server.Addr\n}\n\n// Start sets up the server to listen and serve on the given address.\nfunc (s *Server) Start(addr string) error {\n\tif addr == \"\" {\n\t\tlog.Info(\"%s is disabled\", httpServer)\n\t\treturn nil\n\t}\n\n\tlog.Info(\"starting %s...\", httpServer)\n\n\ts.Lock()\n\tdefer s.Unlock()\n\n\ts.server = &http.Server{Addr: addr, Handler: s}\n\tln, err := net.Listen(\"tcp\", s.server.Addr)\n\tif err != nil {\n\t\treturn httpError(\"can't listen on HTTP TCP address '%s': %v\",\n\t\t\ts.server.Addr, err)\n\t}\n\n\t// update address if port was autobound\n\tif ln.Addr().String() != s.server.Addr {\n\t\ts.server.Addr = ln.Addr().String()\n\t}\n\n\tgo s.server.Serve(ln)\n\n\treturn nil\n}\n\n// Stop Close()'s the server immediately.\nfunc (s *Server) Stop() {\n\tlog.Info(\"stopping %s...\", httpServer)\n\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tif s.server == nil {\n\t\treturn\n\t}\n\n\ts.server.Close()\n\ts.server = nil\n}\n\n// Shutdown shuts down the server gracefully.\nfunc (s *Server) Shutdown(wait bool) {\n\tvar sync chan struct{}\n\n\tlog.Info(\"shutting down %s...\", httpServer)\n\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tif s.server == nil {\n\t\treturn\n\t}\n\n\tif wait {\n\t\tsync = make(chan struct{})\n\t\ts.server.RegisterOnShutdown(func() {\n\t\t\tclose(sync)\n\t\t})\n\t}\n\ts.server.Shutdown(context.Background())\n\t_ = <-sync\n\n\ts.server = nil\n}\n\n// Reconfigure reconfigures the server.\nfunc (s *Server) Reconfigure(addr string) error {\n\tlog.Info(\"reconfiguring %s...\", httpServer)\n\n\tif s.GetAddress() != addr {\n\t\treturn s.Restart(addr)\n\t}\n\treturn nil\n}\n\n// Restart restarts it on the given address.\nfunc (s *Server) Restart(addr string) error {\n\tlog.Info(\"restarting %s...\", httpServer)\n\n\ts.Stop()\n\treturn s.Start(addr)\n}\n\n// ServeHTTP servers the given HTTP request.\nfunc (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\ts.RLock()\n\tdefer s.RUnlock()\n\ts.mux.ServeHTTP(w, r)\n}\n\n// httpError returns a formatted instrumentation/http-specific error.\nfunc httpError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"instrumentation/http: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/instrumentation/http/http_test.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage http\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"testing\"\n)\n\nfunc TestStartStop(t *testing.T) {\n\tsrv := NewServer()\n\n\tif err := srv.Start(\":0\"); err != nil {\n\t\tt.Errorf(\"failed to start HTTP server: %v\", err)\n\t}\n\n\tsrv.Stop()\n\n\tif err := srv.Start(\":0\"); err != nil {\n\t\tt.Errorf(\"failed to start HTTP server: %v\", err)\n\t}\n\n\tif err := srv.Restart(\":0\"); err != nil {\n\t\tt.Errorf(\"failed to restart HTTP server on different port: %v\", err)\n\t}\n\n\tif err := srv.Reconfigure(srv.GetAddress()); err != nil {\n\t\tt.Errorf(\"failed to reconfigure HTTP server on same port: %v\", err)\n\t}\n\tif err := srv.Reconfigure(\":0\"); err != nil {\n\t\tt.Errorf(\"failed to reconfigure HTTP server on different port: %v\", err)\n\t}\n\n\tsrv.Stop()\n}\n\ntype urlTest struct {\n\tpattern  string\n\tresponse string\n\tfallback string\n}\n\nfunc checkURL(t *testing.T, srv *Server, path, response string, status int) {\n\turl := \"http://\" + srv.GetAddress() + path\n\n\tres, err := http.Get(url)\n\tif err != nil {\n\t\tt.Errorf(\"http.Get(%s) failed: %v\", url, err)\n\t}\n\n\tif res.StatusCode != status {\n\t\tt.Errorf(\"http.Get(%s) status %d, expected %d\", url, res.StatusCode, status)\n\t}\n\n\ttxt, err := io.ReadAll(res.Body)\n\tif err != nil {\n\t\tt.Errorf(\"http.Get(%s) failed to read response: %v\", url, err)\n\t}\n\n\tif string(txt) != response {\n\t\tt.Errorf(\"http.Get(%s) unexpected response: %v, expected: %v\", url, txt, response)\n\t}\n}\n\ntype testHandler struct {\n\tresponse string\n}\n\nfunc (h *testHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {\n\t_, _ = w.Write([]byte(h.response))\n}\n\nfunc TestPatternsp(t *testing.T) {\n\tsrv := NewServer()\n\tmux := srv.GetMux()\n\n\tif err := srv.Start(\":0\"); err != nil {\n\t\tt.Errorf(\"failed to start HTTP server: %v\", err)\n\t}\n\n\trh := &testHandler{\"/\"}\n\tah := &testHandler{\"a\"}\n\tbh := &testHandler{\"b\"}\n\tch := &testHandler{\"c\"}\n\n\tmux.Handle(\"/a\", ah)\n\tcheckURL(t, srv, \"/a\", \"a\", 200)\n\n\tmux.Handle(\"/b\", bh)\n\tcheckURL(t, srv, \"/b\", \"b\", 200)\n\n\tmux.Handle(\"/\", rh)\n\tcheckURL(t, srv, \"/b\", \"b\", 200)\n\n\tmux.Unregister(\"/b\")\n\tcheckURL(t, srv, \"/b\", \"/\", 200)\n\n\tmux.Handle(\"/b\", ch)\n\tcheckURL(t, srv, \"/b\", \"c\", 200)\n\n\tmux.Unregister(\"/a\")\n\tcheckURL(t, srv, \"/a\", \"/\", 200)\n\n\tsrv.Stop()\n}\n"
  },
  {
    "path": "pkg/instrumentation/instrumentation.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation/http\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\t// ServiceName is our service name in external tracing and metrics services.\n\tServiceName = \"CRI-RM\"\n)\n\n// Our logger instance.\nvar log = logger.NewLogger(\"instrumentation\")\n\n// Our instrumentation service instance.\nvar svc = newService()\n\n// GetHTTPMux returns our HTTP request mux for external services.\nfunc GetHTTPMux() *http.ServeMux {\n\tif svc == nil {\n\t\treturn nil\n\t}\n\treturn svc.http.GetMux()\n}\n\n// TracingEnabled returns true if the Jaeger tracing sampler is not disabled.\nfunc TracingEnabled() bool {\n\tif svc == nil {\n\t\treturn false\n\t}\n\treturn svc.TracingEnabled()\n}\n\n// Start our internal instrumentation services.\nfunc Start() error {\n\tif svc == nil {\n\t\treturn instrumentationError(\"cannot start, no instrumentation service instance\")\n\t}\n\treturn svc.Start()\n}\n\n// Stop stops our internal instrumentation services.\nfunc Stop() {\n\tif svc != nil {\n\t\tsvc.Stop()\n\t}\n}\n\n// Restart restarts our internal instrumentation services.\nfunc Restart() error {\n\tif svc == nil {\n\t\treturn instrumentationError(\"cannot restart, no instrumentation service instance\")\n\t}\n\treturn svc.Restart()\n}\n\n// instrumentationError produces a formatted instrumentation-specific error.\nfunc instrumentationError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"instrumentation: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/instrumentation/instrumentation_test.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestSamplingIdempotency(t *testing.T) {\n\ttcases := []Sampling{\n\t\tDisabled,\n\t\tTesting,\n\t\tProduction,\n\t\t0.2, 0.25, 0.5, 0.75, 0.8,\n\t}\n\tfor _, tc := range tcases {\n\t\tvar chk Sampling\n\t\tif err := chk.Parse(tc.String()); err != nil {\n\t\t\tt.Errorf(\"failed to parse Sampling.String() %q: %v\", tc, err)\n\t\t}\n\t\tif chk != tc {\n\t\t\tt.Errorf(\"expected sampling value for %q: %v, got: %v\", tc, tc, chk)\n\t\t}\n\t}\n}\n\nfunc TestPrometheusConfiguration(t *testing.T) {\n\tlog.EnableDebug()\n\n\tif opt.HTTPEndpoint == \"\" {\n\t\topt.HTTPEndpoint = \":0\"\n\t}\n\n\ts := newService()\n\ts.Start()\n\n\taddress := s.http.GetAddress()\n\tif strings.HasSuffix(opt.HTTPEndpoint, \":0\") {\n\t\topt.HTTPEndpoint = address\n\t}\n\n\tcheckPrometheus(t, address, !opt.PrometheusExport)\n\n\topt.PrometheusExport = !opt.PrometheusExport\n\ts.reconfigure()\n\tcheckPrometheus(t, address, !opt.PrometheusExport)\n\n\topt.PrometheusExport = !opt.PrometheusExport\n\ts.reconfigure()\n\tcheckPrometheus(t, address, !opt.PrometheusExport)\n\n\topt.PrometheusExport = !opt.PrometheusExport\n\ts.reconfigure()\n\tcheckPrometheus(t, address, !opt.PrometheusExport)\n\n\ts.http.Shutdown(true)\n\ts.Stop()\n}\n\nfunc checkPrometheus(t *testing.T, server string, shouldFail bool) {\n\trpl, err := http.Get(\"http://\" + server + \"/metrics\")\n\n\tswitch shouldFail {\n\tcase false:\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Prometheus HTTP GET failed: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\tif rpl.StatusCode != 200 {\n\t\t\tt.Errorf(\"Prometheus HTTP GET failed: %s\", rpl.Status)\n\t\t\treturn\n\t\t}\n\n\t\t_, err = io.ReadAll(rpl.Body)\n\t\trpl.Body.Close()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"failed to read Prometheus response: %v\", err)\n\t\t}\n\t\treturn\n\n\tcase true:\n\t\tif err == nil && rpl.StatusCode == 200 {\n\t\t\tt.Errorf(\"Prometheus HTTP GET should have failed, but it didn't.\")\n\t\t\treturn\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/instrumentation/jaeger.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"contrib.go.opencensus.io/exporter/jaeger\"\n\t\"go.opencensus.io/trace\"\n)\n\nconst (\n\t// jaegerExporter is used in log messages.\n\tjaegerExporter = \"Jaeger trace exporter\"\n)\n\n// tracing encapsulates the state of our Jaeger exporter.\ntype tracing struct {\n\texporter  *jaeger.Exporter\n\tagent     string\n\tcollector string\n\tsampling  Sampling\n}\n\n// start starts our Jaeger exporter.\nfunc (t *tracing) start(agent, collector string, sampling Sampling) error {\n\tif agent == \"\" && collector == \"\" {\n\t\tlog.Info(\"%s is disabled\", jaegerExporter)\n\t\treturn nil\n\t}\n\n\tlog.Info(\"creating %s...\", jaegerExporter)\n\n\tcfg := jaeger.Options{\n\t\tServiceName:       ServiceName,\n\t\tCollectorEndpoint: collector,\n\t\tAgentEndpoint:     agent,\n\n\t\tProcess: jaeger.Process{ServiceName: ServiceName},\n\t\tOnError: func(err error) { log.Error(\"jaeger error: %v\", err) },\n\t}\n\n\texp, err := jaeger.NewExporter(cfg)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to create %s: %v\", jaegerExporter, err)\n\t}\n\n\tt.exporter = exp\n\tt.agent = agent\n\tt.collector = collector\n\tt.sampling = sampling\n\n\ttrace.RegisterExporter(t.exporter)\n\ttrace.ApplyConfig(trace.Config{DefaultSampler: t.sampling.Sampler()})\n\n\treturn nil\n}\n\n// stop stops our Jaeger exporter.\nfunc (t *tracing) stop() {\n\tif t.exporter == nil {\n\t\treturn\n\t}\n\n\tlog.Info(\"stopping Jaeger trace exporter...\")\n\n\ttrace.UnregisterExporter(t.exporter)\n\t*t = tracing{}\n}\n\n// reconfigure reconfigures our Jaeger exporter.\nfunc (t *tracing) reconfigure(agent, collector string, sampling Sampling) error {\n\tlog.Info(\"reconfiguring %s...\", jaegerExporter)\n\n\tif agent == \"\" && collector == \"\" {\n\t\tt.stop()\n\t\treturn nil\n\t}\n\n\tif t.agent != agent || t.collector != collector {\n\t\tt.stop()\n\t}\n\n\tif t.exporter != nil {\n\t\tt.sampling = sampling\n\t\ttrace.ApplyConfig(trace.Config{DefaultSampler: t.sampling.Sampler()})\n\t\treturn nil\n\t}\n\n\treturn t.start(agent, collector, sampling)\n}\n"
  },
  {
    "path": "pkg/instrumentation/prometheus.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"contrib.go.opencensus.io/exporter/prometheus\"\n\tpclient \"github.com/prometheus/client_golang/prometheus\"\n\tmodel \"github.com/prometheus/client_model/go\"\n\t\"go.opencensus.io/stats/view\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation/http\"\n)\n\nconst (\n\t// PrometheusMetricsPath is the URL path for exposing metrics to Prometheus.\n\tPrometheusMetricsPath = \"/metrics\"\n\t// prometheusExporter is used in log messages.\n\tprometheusExporter = \"Prometheus metrics exporter\"\n)\n\n// metrics encapsulates the state of our Prometheus exporter.\ntype metrics struct {\n\texporter *prometheus.Exporter\n\tmux      *http.ServeMux\n\tperiod   time.Duration\n}\n\n// start starts our Prometheus exporter.\nfunc (m *metrics) start(mux *http.ServeMux, period time.Duration, enable bool) error {\n\tif !enable {\n\t\tlog.Info(\"%s is disabled\", prometheusExporter)\n\t\treturn nil\n\t}\n\n\tlog.Info(\"starting %s...\", prometheusExporter)\n\n\tcfg := prometheus.Options{\n\t\tNamespace: prometheusNamespace(ServiceName),\n\t\tGatherer:  pclient.Gatherers{dynamicGatherers},\n\t\tOnError:   func(err error) { log.Error(\"prometheus error: %v\", err) },\n\t}\n\n\texp, err := prometheus.NewExporter(cfg)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to create %s: %v\", prometheusExporter, err)\n\t}\n\n\tm.exporter = exp\n\tm.mux = mux\n\tm.period = period\n\n\tm.mux.Handle(PrometheusMetricsPath, m.exporter)\n\tview.RegisterExporter(m.exporter)\n\tview.SetReportingPeriod(m.period)\n\n\treturn nil\n}\n\n// stop stops our Prometheus exporter.\nfunc (m *metrics) stop() {\n\tif m.exporter == nil {\n\t\treturn\n\t}\n\n\tlog.Info(\"stopping %s...\", prometheusExporter)\n\n\tview.UnregisterExporter(m.exporter)\n\tm.mux.Unregister(PrometheusMetricsPath)\n\n\t*m = metrics{}\n}\n\n// reconfigure reconfigures our Prometheus exporter.\nfunc (m *metrics) reconfigure(mux *http.ServeMux, period time.Duration, enable bool) error {\n\tlog.Info(\"reconfiguring %s...\", prometheusExporter)\n\n\tif !enable {\n\t\tm.stop()\n\t\treturn nil\n\t}\n\n\tif m.exporter != nil {\n\t\tm.period = period\n\t\tview.SetReportingPeriod(m.period)\n\t\treturn nil\n\t}\n\n\treturn m.start(mux, period, enable)\n}\n\n// mutate service name into a valid Prometheus namespace name.\nfunc prometheusNamespace(service string) string {\n\treturn strings.ReplaceAll(strings.ToLower(service), \"-\", \"_\")\n}\n\n// gatherers is a trivial wrapper around prometheus Gatherers.\ntype gatherers struct {\n\tsync.RWMutex\n\tgatherers pclient.Gatherers\n}\n\n// Our dynamically registered Prometheus gatherers.\nvar dynamicGatherers = &gatherers{gatherers: pclient.Gatherers{}}\n\n// Register registers a new gatherer.\nfunc (g *gatherers) Register(gatherer pclient.Gatherer) {\n\tg.Lock()\n\tdefer g.Unlock()\n\tg.gatherers = append(g.gatherers, gatherer)\n}\n\n// Gather implements the pclient.Gatherer interface.\nfunc (g *gatherers) Gather() ([]*model.MetricFamily, error) {\n\tg.RLock()\n\tdefer g.RUnlock()\n\treturn g.gatherers.Gather()\n}\n\n// RegisterGatherer registers a new prometheus Gatherer.\nfunc RegisterGatherer(g pclient.Gatherer) {\n\tdynamicGatherers.Register(g)\n}\n"
  },
  {
    "path": "pkg/instrumentation/service.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage instrumentation\n\nimport (\n\t\"sync\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/instrumentation/http\"\n)\n\n// service is the state of our instrumentation services: HTTP endpoint, trace/metrics exporters.\ntype service struct {\n\tsync.RWMutex              // we're RW-lockable\n\thttp         *http.Server // HTTP server\n\ttracing      *tracing     // tracing data exporter\n\tmetrics      *metrics     // metrics data exporter\n}\n\n// newService creates an instance of our instrumentation services.\nfunc newService() *service {\n\treturn &service{\n\t\thttp:    http.NewServer(),\n\t\ttracing: &tracing{},\n\t\tmetrics: &metrics{},\n\t}\n}\n\n// Start starts instrumentation services.\nfunc (s *service) Start() error {\n\tlog.Info(\"starting instrumentation services...\")\n\n\ts.Lock()\n\tdefer s.Unlock()\n\n\terr := s.http.Start(opt.HTTPEndpoint)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to start HTTP server: %v\", err)\n\t}\n\terr = s.tracing.start(opt.JaegerAgent, opt.JaegerCollector, opt.Sampling)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to start tracing: %v\", err)\n\t}\n\terr = s.metrics.start(s.http.GetMux(), opt.ReportPeriod, opt.PrometheusExport)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to start metrics: %v\", err)\n\t}\n\n\tif err := registerGrpcViews(); err != nil {\n\t\ts.metrics.stop()\n\t\ts.tracing.stop()\n\t\ts.http.Stop()\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Stop stops instrumentation services.\nfunc (s *service) Stop() {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tunregisterGrpcViews()\n\ts.metrics.stop()\n\ts.tracing.stop()\n\ts.http.Stop()\n}\n\n// reconfigure reconfigures instrumentation services.\nfunc (s *service) reconfigure() error {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\terr := s.http.Reconfigure(opt.HTTPEndpoint)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to reconfigure HTTP server: %v\", err)\n\t}\n\terr = s.tracing.reconfigure(opt.JaegerAgent, opt.JaegerCollector, opt.Sampling)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to reconfigure tracing: %v\", err)\n\t}\n\terr = s.metrics.reconfigure(s.http.GetMux(), opt.ReportPeriod, opt.PrometheusExport)\n\tif err != nil {\n\t\treturn instrumentationError(\"failed to reconfigure metrics: %v\", err)\n\t}\n\treturn nil\n}\n\n// Restart restarts instrumentation services.\nfunc (s *service) Restart() error {\n\ts.Stop()\n\treturn s.Start()\n}\n\n// TracingEnabled returns true if the Jaeger tracing sampler is not disabled.\nfunc (s *service) TracingEnabled() bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\n\treturn float64(opt.Sampling) > 0.0\n}\n"
  },
  {
    "path": "pkg/log/default.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n)\n\n// our default logger\nvar deflog = log.get(filepath.Base(filepath.Clean(os.Args[0])))\n\n// Default returns the default Logger.\nfunc Default() Logger {\n\treturn deflog\n}\n\n// Info formats and emits an informational message.\nfunc Info(format string, args ...interface{}) {\n\tdeflog.Info(format, args...)\n}\n\n// Warn formats and emits a warning message.\nfunc Warn(format string, args ...interface{}) {\n\tdeflog.Warn(format, args...)\n}\n\n// Error formats and emits an error message.\nfunc Error(format string, args ...interface{}) {\n\tdeflog.Error(format, args...)\n}\n\n// Fatal formats and emits an error message and os.Exit()'s with status 1.\nfunc Fatal(format string, args ...interface{}) {\n\tdeflog.Fatal(format, args...)\n}\n\n// Panic formats and emits an error messages, and panics with the same.\nfunc Panic(format string, args ...interface{}) {\n\tdeflog.Panic(format, args...)\n}\n\n// Debug formats and emits a debug message.\nfunc Debug(format string, args ...interface{}) {\n\tdeflog.Debug(format, args...)\n}\n\n// InfoBlock formats and emits a multiline information message.\nfunc InfoBlock(prefix string, format string, args ...interface{}) {\n\tdeflog.InfoBlock(prefix, format, args...)\n}\n\n// WarnBlock formats and emits a multiline warning message.\nfunc WarnBlock(prefix string, format string, args ...interface{}) {\n\tdeflog.WarnBlock(prefix, format, args...)\n}\n\n// ErrorBlock formats and emits a multiline error message.\nfunc ErrorBlock(prefix string, format string, args ...interface{}) {\n\tdeflog.ErrorBlock(prefix, format, args...)\n}\n\n// DebugBlock formats and emits a multiline debug message.\nfunc DebugBlock(prefix string, format string, args ...interface{}) {\n\tdeflog.DebugBlock(prefix, format, args...)\n}\n\nfunc init() {\n\tbinary := filepath.Clean(os.Args[0])\n\tsource := filepath.Base(binary)\n\tdeflog = log.get(source)\n}\n"
  },
  {
    "path": "pkg/log/flags.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"strings\"\n\n\tpkgcfg \"github.com/intel/cri-resource-manager/pkg/config\"\n\t\"github.com/intel/cri-resource-manager/pkg/log/klogcontrol\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n)\n\nconst (\n\t// DefaultLevel is the default logging severity level.\n\tDefaultLevel = LevelInfo\n\t// debugEnvVar is the environment variable used to seed debugging flags.\n\tdebugEnvVar = \"LOGGER_DEBUG\"\n\t// configModule is our module name in the runtime configuration.\n\tconfigModule = \"logger\"\n)\n\n// options capture our runtime configuration.\ntype options struct {\n\t// Klog contains klog-specific options.\n\tKlog klogcontrol.Options\n\t// Debug defines which sources produce debug messages.\n\tDebug srcmap\n\t// LogSource determines if messages are prefixed with the logger source\n\tLogSource bool\n}\n\n// srcmap tracks debugging settings for sources.\ntype srcmap map[string]bool\n\nvar (\n\t// Runtime logging configuration.\n\topt *options\n\t// Default debugging configuration.\n\tdefaultDebugFlags srcmap\n\t// Default klog configuration.\n\tdefaultKlogFlags klogcontrol.Options\n\t// klog control\n\tklogctl *klogcontrol.Control\n)\n\n// parse parses the given string and updates the srcmap accordingly.\nfunc (m *srcmap) parse(value string) error {\n\tif *m == nil {\n\t\t*m = make(srcmap)\n\t}\n\tif value = strings.TrimSpace(value); value == \"\" {\n\t\treturn nil\n\t}\n\n\tprev, state, src := \"\", \"\", \"\"\n\tfor _, entry := range strings.Split(value, \",\") {\n\t\tif entry = strings.TrimSpace(entry); entry == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tstatesrc := strings.Split(entry, \":\")\n\t\tswitch len(statesrc) {\n\t\tcase 2:\n\t\t\tstate, src = statesrc[0], strings.TrimSpace(statesrc[1])\n\t\tcase 1:\n\t\t\tstate, src = \"\", strings.TrimSpace(statesrc[0])\n\t\tdefault:\n\t\t\treturn loggerError(\"invalid state spec '%s' in source map\", entry)\n\t\t}\n\t\tif state != \"\" {\n\t\t\tprev = state\n\t\t} else {\n\t\t\tstate = prev\n\t\t\tif state == \"\" {\n\t\t\t\tstate = \"on\"\n\t\t\t}\n\t\t}\n\n\t\tif src == \"all\" {\n\t\t\tsrc = \"*\"\n\t\t}\n\n\t\tenabled, err := utils.ParseEnabled(state)\n\t\tif err != nil {\n\t\t\treturn loggerError(\"invalid state '%s' in source map\", state)\n\t\t}\n\t\t(*m)[src] = enabled\n\t}\n\n\treturn nil\n}\n\n// String returns a string representation of the srcmap.\nfunc (m *srcmap) String() string {\n\toff := \"\"\n\ton := \"\"\n\tfor src, state := range *m {\n\t\tif state {\n\t\t\tif on == \"\" {\n\t\t\t\ton = src\n\t\t\t} else {\n\t\t\t\ton += \",\" + src\n\t\t\t}\n\t\t} else {\n\t\t\tif off == \"\" {\n\t\t\t\toff = src\n\t\t\t} else {\n\t\t\t\toff += \",\" + src\n\t\t\t}\n\t\t}\n\t}\n\n\tswitch {\n\tcase on == \"\" && off == \"\":\n\t\treturn \"\"\n\tcase off == \"\":\n\t\treturn \"on:\" + on\n\tcase on == \"\":\n\t\treturn \"off:\" + off\n\t}\n\treturn \"on:\" + on + \",\" + \"off:\" + off\n}\n\n// MarshalJSON is the JSON marshaller for srcmap.\nfunc (m srcmap) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(m.String())\n}\n\n// UnmarshalJSON is the JSON unmarshaller for srcmap.\nfunc (m *srcmap) UnmarshalJSON(raw []byte) error {\n\tcfgstr := \"\"\n\tif err := json.Unmarshal(raw, &cfgstr); err != nil {\n\t\treturn loggerError(\"failed to unmarshal source map '%s': %v\", string(raw), err)\n\t}\n\tif err := m.parse(cfgstr); err != nil {\n\t\treturn loggerError(\"failed to unmarshal source map '%s': %v\", string(raw), err)\n\t}\n\treturn nil\n}\n\n// cloneFrom state from another srcmap.\nfunc (m *srcmap) cloneFrom(o srcmap) {\n\t*m = make(srcmap)\n\tfor src, state := range o {\n\t\t(*m)[src] = state\n\t}\n}\n\n// clone returns a copy of the srcmap.\nfunc (m srcmap) clone() srcmap {\n\tif m == nil {\n\t\treturn nil\n\t}\n\to := make(srcmap)\n\tfor src, state := range m {\n\t\to[src] = state\n\t}\n\treturn o\n}\n\n// configNotify is the configuration change notification callback for options.\nfunc (o *options) configNotify(event pkgcfg.Event, _ pkgcfg.Source) error {\n\tdeflog.Info(\"logger configuration %v\", event)\n\tdeflog.Info(\" * debugging: %s\", o.Debug.String())\n\tdeflog.Info(\" * log source: %v\", o.LogSource)\n\tdeflog.InfoBlock(\" * klog: \", \"%s\", o.Klog.String())\n\n\t// On the first configuration update event, we record the current values\n\t// of klog flags as the runtime defaults. Effectively this allows one to\n\t// override the built-in defaults using klog command line options (or\n\t// environment variables as interpreted by klogcontrol). The recorded\n\t// defaults will also reflect any potential programmatic changes done by\n\t// (mis-)using flag.Set() but there's not much we can do about that.\n\tif defaultKlogFlags == nil {\n\t\tdefaultKlogFlags = klogctl.CurrentOptions()\n\t}\n\n\tif o.Klog == nil {\n\t\to.Klog = make(klogcontrol.Options)\n\t}\n\n\t// The behavior of the options.Klog map across updates is difficult\n\t// to understand. To make it more user friendly we fill in runtime\n\t// defaults for each unset entry (klog flags) here.\n\tfor flag, value := range defaultKlogFlags {\n\t\tif _, ok := o.Klog[flag]; !ok {\n\t\t\to.Klog[flag] = value\n\t\t}\n\t}\n\n\treturn o.apply()\n}\n\n// apply applies the options to logging.\nfunc (o *options) apply() error {\n\tlog.Lock()\n\tdefer log.Unlock()\n\n\tprefix := o.LogSource\n\tif logToStderr, ok := o.Klog[\"logtostderr\"]; ok && logToStderr.(bool) {\n\t\tif skipHeaders, ok := o.Klog[\"skip_headers\"]; ok && skipHeaders.(bool) {\n\t\t\tprefix = true\n\t\t}\n\t}\n\n\tlog.setDbgMap(o.Debug.clone())\n\tlog.setPrefix(prefix)\n\n\treturn klogctl.Configure(o.Klog)\n}\n\n// defaultOptions returns our current default runtime options.\nfunc defaultOptions() interface{} {\n\to := &options{}\n\n\to.Debug.cloneFrom(defaultDebugFlags)\n\tif defaultKlogFlags != nil {\n\t\to.Klog.CloneFrom(defaultKlogFlags)\n\t} else {\n\t\to.Klog = klogctl.CurrentOptions()\n\t}\n\n\treturn o\n}\n\n// Set up klog control, set pkg/config logger, register us for configuration handling.\nfunc init() {\n\tklogctl = klogcontrol.Get()\n\topt = defaultOptions().(*options)\n\topt.apply()\n\n\tcfglog := log.get(\"config\")\n\tpkgcfg.SetLogger(pkgcfg.Logger{\n\t\tDebugEnabled: cfglog.DebugEnabled,\n\t\tDebug:        cfglog.Debug,\n\t\tInfo:         cfglog.Info,\n\t\tWarning:      cfglog.Warn,\n\t\tError:        cfglog.Error,\n\t\tFatal:        cfglog.Fatal,\n\t\tPanic:        cfglog.Panic,\n\t})\n\n\tdefaultDebugFlags = make(srcmap)\n\tif value, ok := os.LookupEnv(debugEnvVar); ok {\n\t\tif err := defaultDebugFlags.parse(value); err != nil {\n\t\t\tDefault().Error(\"failed to parse %s %q: %v\", debugEnvVar,\n\t\t\t\tvalue, err)\n\t\t} else {\n\t\t\tlog.setDbgMap(defaultDebugFlags)\n\t\t\tDefault().Info(\"seeded debug flags ($%s): %s\", debugEnvVar,\n\t\t\t\tdefaultDebugFlags.String())\n\t\t}\n\t}\n\n\tpkgcfg.Register(configModule, \"logging control\", opt, defaultOptions,\n\t\tpkgcfg.WithNotify(opt.configNotify))\n}\n"
  },
  {
    "path": "pkg/log/grpc-logger.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"fmt\"\n\n\t\"google.golang.org/grpc/grpclog\"\n)\n\n// SetGrpcLogger sets up a logger for (google.golang.org/)grpc.\nfunc SetGrpcLogger(source string, rate *Rate) {\n\tvar l Logger\n\n\tif source == \"\" {\n\t\tl = Default()\n\t} else {\n\t\tl = log.get(source)\n\t}\n\n\tif rate != nil {\n\t\tl = RateLimit(l, *rate)\n\t}\n\n\tgrpclog.SetLoggerV2(&grpclogger{Logger: l})\n}\n\n// grpclogger implements grpclog.LoggerV2 interface for our logger.\ntype grpclogger struct {\n\tLogger\n}\n\nfunc (g grpclogger) Info(args ...interface{}) {\n\tg.Logger.Debug(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Infoln(args ...interface{}) {\n\tg.Logger.Debug(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Infof(format string, args ...interface{}) {\n\tg.Logger.Debug(format, args...)\n}\n\nfunc (g grpclogger) Warning(args ...interface{}) {\n\tg.Logger.Warn(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Warningln(args ...interface{}) {\n\tg.Logger.Warn(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Warningf(format string, args ...interface{}) {\n\tg.Logger.Warn(format, args...)\n}\n\nfunc (g grpclogger) Error(args ...interface{}) {\n\tg.Logger.Error(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Errorln(args ...interface{}) {\n\tg.Logger.Error(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Errorf(format string, args ...interface{}) {\n\tg.Logger.Error(format, args...)\n}\n\nfunc (g grpclogger) Fatal(args ...interface{}) {\n\tg.Logger.Fatal(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Fatalln(args ...interface{}) {\n\tg.Logger.Fatal(\"%s\", fmt.Sprint(args...))\n}\n\nfunc (g grpclogger) Fatalf(format string, args ...interface{}) {\n\tg.Logger.Fatal(format, args...)\n}\n\nfunc (g grpclogger) V(_ int) bool {\n\treturn true\n}\n"
  },
  {
    "path": "pkg/log/klogcontrol/klogcontrol.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage klogcontrol\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\n\t\"k8s.io/klog/v2\"\n)\n\n// Options captures runtime configuration for klog.\ntype Options map[string]interface{}\n\n// Control implements runtime control for klog.\ntype Control struct {\n\tflags *flag.FlagSet\n}\n\n// Our singleton klog Control instance.\nvar ctl *Control\n\n// Get returns our singleton klog Control instance.\nfunc Get() *Control {\n\treturn ctl\n}\n\n// CurrentOptions returns the current klog configuration as Options.\nfunc (c *Control) CurrentOptions() Options {\n\to := make(Options)\n\tc.flags.VisitAll(func(f *flag.Flag) {\n\t\to[f.Name] = flag.Lookup(f.Name).Value.(flag.Getter).Get()\n\t})\n\treturn o\n}\n\n// Configure reconfigures klog with the given Options.\nfunc (c *Control) Configure(options Options) error {\n\tfor name, value := range options {\n\t\tif err := flag.Set(name, fmt.Sprintf(\"%v\", value)); err != nil {\n\t\t\treturn klogError(\"failed to set klog flag %q to %v: %v\", name, value, err)\n\t\t}\n\t}\n\treturn nil\n}\n\n// Set sets the value of the given klog flag.\nfunc (c *Control) Set(name, value string) error {\n\treturn flag.Set(name, value)\n}\n\n// Get returns the current value of the given klog flag.\nfunc (c *Control) Get(name string) (interface{}, error) {\n\tif c.flags.Lookup(name) == nil {\n\t\treturn nil, klogError(\"unknown klog flag %q\", name)\n\t}\n\treturn flag.Lookup(name).Value.(flag.Getter).Get(), nil\n}\n\n// CloneFrom clones src to o.\nfunc (o *Options) CloneFrom(src Options) {\n\t*o = make(Options)\n\tfor name, value := range src {\n\t\t(*o)[name] = value\n\t}\n}\n\n// String returns a string representation of the Options.\nfunc (o *Options) String() string {\n\tif o == nil {\n\t\treturn \"<nil>\"\n\t}\n\tstr := \"\"\n\tsep := \"\"\n\tfor name, value := range *o {\n\t\tstr += sep + name + \"=\" + fmt.Sprintf(\"%v\", value)\n\t\tsep = \"\\n\"\n\t}\n\treturn str\n}\n\n// klogflag wraps a klog flag for configuration.\ntype klogflag struct {\n\tflag *flag.Flag\n}\n\n// Set implements flag.Value.Set() for wrapped klog flags.\nfunc (klogf *klogflag) Set(value string) error {\n\tif klogf.flag.Name == \"stderrthreshold\" { // klog expects thresholds in ALL CAPS\n\t\tvalue = strings.ToUpper(value)\n\t}\n\tif err := klogf.flag.Value.Set(value); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// String implements flag.Value.String() for wrapped klog flags.\nfunc (klogf *klogflag) String() string {\n\tif klogf.flag == nil { // flag.isZeroValue() probing us...\n\t\treturn \"\"\n\t}\n\tvalue := klogf.flag.Value.String()\n\tif klogf.flag.Name == \"log_backtrace_at\" && value == \":0\" {\n\t\tvalue = \"\"\n\t}\n\treturn value\n}\n\n// Get implements flag.Getter.Get() for wrapped klog flags.\nfunc (klogf *klogflag) Get() interface{} {\n\tif getter, ok := klogf.flag.Value.(flag.Getter); ok {\n\t\tif value := getter.Get(); value != nil {\n\t\t\treturn value\n\t\t}\n\t}\n\treturn klogf.String()\n}\n\n// boolFlag is identical to the unexported flag.boolFlag interface.\ntype boolFlag interface {\n\tIsBoolFlag() bool\n}\n\n// IsBoolFlag implements flag.boolFlag.IsBoolFlag() for wrapped klog flags.\nfunc (klogf *klogflag) IsBoolFlag() bool {\n\tif klogf.flag == nil {\n\t\treturn false\n\t}\n\tif boolf, ok := klogf.flag.Value.(boolFlag); ok {\n\t\treturn boolf.IsBoolFlag()\n\t}\n\treturn false\n}\n\n// getEnv returns a default value for the flag from the environment.\nfunc (klogf *klogflag) getEnv() (string, string, bool) {\n\tname := \"LOGGER_\" + strings.ToUpper(strings.ReplaceAll(klogf.flag.Name, \"-\", \"_\"))\n\tif value, ok := os.LookupEnv(name); ok {\n\t\treturn name, value, true\n\t}\n\treturn \"\", \"\", false\n}\n\n// klogError returns a package-specific formatted error.\nfunc klogError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"klogcontrol: \"+format, args...)\n}\n\n// wrapKlogFlag wraps and registers the given klog flag.\nfunc wrapKlogFlag(f *flag.Flag) {\n\tklogf := &klogflag{flag: f}\n\tflag.Var(klogf, f.Name, f.Usage)\n\n\tif name, value, ok := klogf.getEnv(); ok {\n\t\tif err := klogf.Set(value); err != nil {\n\t\t\tklog.Errorf(\"klog flag %q: invalid environment default %s=%q: %v\",\n\t\t\t\tf.Name, name, value, err)\n\t\t}\n\t} else {\n\t\t// Unless explicitly configured in the environment, by default\n\t\t// turn off headers (date, timestamp, etc.) when we're logging\n\t\t// to a journald stream.\n\t\tif f.Name == \"skip_headers\" {\n\t\t\tif value, _ := os.LookupEnv(\"JOURNAL_STREAM\"); value != \"\" {\n\t\t\t\tklog.Infof(\"Logging to journald, forcing headers off...\")\n\t\t\t\tklogf.Set(\"true\")\n\t\t\t}\n\t\t}\n\t}\n}\n\n// init discovers klog flags and sets up dynamic control for them.\nfunc init() {\n\tctl = &Control{flags: flag.NewFlagSet(\"klog flags\", flag.ContinueOnError)}\n\tctl.flags.SetOutput(io.Discard)\n\tklog.InitFlags(ctl.flags)\n\tctl.flags.VisitAll(func(f *flag.Flag) {\n\t\twrapKlogFlag(f)\n\t})\n}\n"
  },
  {
    "path": "pkg/log/log.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"k8s.io/klog/v2\"\n)\n\n// Level describes the severity of a log message.\ntype Level int\n\nconst (\n\t// levelUnset denotes an unset level.\n\tlevelUnset Level = iota\n\t// LevelDebug is the severity for debug messages.\n\tLevelDebug\n\t// LevelInfo is the severity for informational messages.\n\tLevelInfo\n\t// LevelWarn is the severity for warnings.\n\tLevelWarn\n\t// LevelError is the severity for errors.\n\tLevelError\n\t// LevelPanic is the severity for panic messages.\n\tLevelPanic\n\t// LevelFatal is the severity for fatal errors.\n\tLevelFatal\n)\n\n// Per-level prefix tags.\nvar levelTag = map[Level]string{\n\tlevelUnset: \"?: \",\n\tLevelDebug: \"D: \",\n\tLevelInfo:  \"I: \",\n\tLevelWarn:  \"W: \",\n\tLevelError: \"E: \",\n\tLevelFatal: \"F: \",\n\tLevelPanic: \"P: \",\n}\n\n// Logger is the interface for producing log messages for/from a particular source.\ntype Logger interface {\n\t// Standardized Logger interface functions so that this interface can be\n\t// used from goresctrl library.\n\tDebugf(format string, v ...interface{})\n\tInfof(format string, v ...interface{})\n\tWarnf(format string, v ...interface{})\n\tErrorf(format string, v ...interface{})\n\tPanicf(format string, v ...interface{})\n\tFatalf(format string, v ...interface{})\n\n\t// Debug formats and emits a debug message.\n\tDebug(format string, args ...interface{})\n\t// Info formats and emits an informational message.\n\tInfo(format string, args ...interface{})\n\t// Warn formats and emits a warning message.\n\tWarn(format string, args ...interface{})\n\t// Error formats and emits an error message.\n\tError(format string, args ...interface{})\n\t// Panic formats and emits an error message then panics with the same.\n\tPanic(format string, args ...interface{})\n\t// Fatal formats and emits an error message and os.Exit()'s with status 1.\n\tFatal(format string, args ...interface{})\n\n\t// DebugBlock formats and emits a multiline debug message.\n\tDebugBlock(prefix string, format string, args ...interface{})\n\t// InfoBlock formats and emits a multiline information message.\n\tInfoBlock(prefix string, format string, args ...interface{})\n\t// WarnBlock formats and emits a multiline warning message.\n\tWarnBlock(prefix string, format string, args ...interface{})\n\t// ErrorBlock formats and emits a multiline error message.\n\tErrorBlock(prefix string, format string, args ...interface{})\n\n\t// EnableDebug enables debug messages for this Logger.\n\tEnableDebug() bool\n\t// DebugEnabled checks if debug messages are enabled for this Logger.\n\tDebugEnabled() bool\n\n\t// Source returns the source name of this Logger.\n\tSource() string\n}\n\n// logger implements Logger.\ntype logger uint\n\n// logging encapsulates the full runtime state of logging.\ntype logging struct {\n\tsync.RWMutex\n\tlevel   Level               // logging threshold for stderr\n\tdbgmap  srcmap              // debug configuration\n\tloggers map[string]logger   // source to logger mapping\n\tsources map[logger]string   // logger to source mapping\n\tdebug   map[logger]struct{} // loggers with debugging enabled\n\tmaxlen  int                 // max source length.\n\tforced  bool                // forced global debugging\n\tprefix  bool                // prefix messages with logger source\n\taligned map[logger]string   // logger sources aligned to maxlen\n}\n\n// log tracks our runtime state.\nvar log = &logging{\n\tlevel:   DefaultLevel,\n\tloggers: make(map[string]logger),\n\tsources: make(map[logger]string),\n\taligned: make(map[logger]string),\n\tdebug:   make(map[logger]struct{}),\n}\n\n// Get returns the named Logger.\nfunc Get(source string) Logger {\n\tlog.Lock()\n\tdefer log.Unlock()\n\treturn log.get(source)\n}\n\n// NewLogger creates the named logger.\nfunc NewLogger(source string) Logger {\n\treturn Get(source)\n}\n\n// EnableDebug enables debug logging for the source.\nfunc EnableDebug(source string) bool {\n\tlog.Lock()\n\tdefer log.Unlock()\n\treturn log.setDebug(source, true)\n}\n\n// DisableDebug disables debug logging for the source.\nfunc DisableDebug(source string) bool {\n\tlog.Lock()\n\tdefer log.Unlock()\n\treturn log.setDebug(source, false)\n}\n\n// DebugEnabled checks if debug logging is enabled for the source.\nfunc DebugEnabled(source string) bool {\n\tlog.Lock()\n\tdefer log.Unlock()\n\treturn log.getDebug(source)\n}\n\n// SetLevel sets the logging severity level.\nfunc SetLevel(level Level) {\n\tlog.Lock()\n\tdefer log.Unlock()\n\tlog.setLevel(level)\n}\n\n// Flush flushes any pending log messages.\nfunc Flush() {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\tklog.Flush()\n}\n\n//\n// logging\n//\n\nfunc (l Level) String() string {\n\tswitch l {\n\tcase LevelDebug:\n\t\treturn \"debug\"\n\tcase LevelInfo:\n\t\treturn \"info\"\n\tcase LevelWarn:\n\t\treturn \"warning\"\n\tcase LevelError:\n\t\treturn \"error\"\n\tcase LevelPanic:\n\t\treturn \"panic\"\n\tcase LevelFatal:\n\t\treturn \"fatal\"\n\t}\n\treturn \"unknown\"\n}\n\n// setLevel sets the logging severity level.\nfunc (log *logging) setLevel(level Level) error {\n\tlog.level = level\n\tthreshold := \"\"\n\tswitch level {\n\tcase LevelDebug, LevelInfo:\n\t\tthreshold = \"INFO\"\n\tcase LevelWarn:\n\t\tthreshold = \"WARNING\"\n\tcase LevelError, LevelPanic, LevelFatal:\n\t\tthreshold = \"ERROR\"\n\t}\n\tif err := klogctl.Set(\"stderrthreshold\", threshold); err != nil {\n\t\treturn loggerError(\"failed to set log level/threshold to %s: %v\", threshold, err)\n\t}\n\treturn nil\n}\n\n// setDebug sets the debug state for the given source and returns the previous one.\nfunc (log *logging) setDebug(source string, enabled bool) bool {\n\tl := log.get(source)\n\t_, old := log.debug[l]\n\tif enabled {\n\t\tlog.debug[l] = struct{}{}\n\t} else {\n\t\tdelete(log.debug, l)\n\t}\n\treturn old\n}\n\n// getDebug sets the debug state for the given source and returns the previous one.\nfunc (log *logging) getDebug(source string) bool {\n\tif log.forced {\n\t\treturn true\n\t}\n\tl := log.get(source)\n\t_, enabled := log.debug[l]\n\treturn enabled\n}\n\n// setDbgMap updates the debug configuration of logging.\nfunc (log *logging) setDbgMap(dbgmap srcmap) {\n\tlog.dbgmap = dbgmap\n\tlog.debug = make(map[logger]struct{})\n\tfor source := range log.loggers {\n\t\tstate, ok := log.dbgmap[source]\n\t\tif !ok {\n\t\t\tstate = log.dbgmap[\"*\"]\n\t\t}\n\t\tlog.setDebug(source, state)\n\t}\n}\n\n// setPrefix sets the prefix (source) logging preference.\nfunc (log *logging) setPrefix(prefix bool) {\n\tlog.prefix = prefix\n}\n\n// align calculates and stores an aligned prefix for the given logger.\nfunc (log *logging) align(l logger) {\n\tsource := log.sources[l]\n\tsrclen := len(source)\n\n\tif srclen > log.maxlen {\n\t\tlog.realign(srclen)\n\t\treturn\n\t}\n\n\tpad := log.maxlen - srclen\n\tpre := (pad + 1) / 2\n\tsuf := pad - pre\n\tlog.aligned[l] = \"[\" + fmt.Sprintf(\"%*s\", pre, \"\") + source + fmt.Sprintf(\"%*s\", suf, \"\") + \"] \"\n}\n\n// realign recalculates aligned prefixes for all loggers.\nfunc (log *logging) realign(maxlen int) {\n\tif maxlen <= 0 {\n\t\tfor _, source := range log.sources {\n\t\t\tif srclen := len(source); srclen > maxlen {\n\t\t\t\tmaxlen = srclen\n\t\t\t}\n\t\t}\n\t}\n\tlog.maxlen = maxlen\n\tlog.aligned = make(map[logger]string)\n\tfor l := range log.sources {\n\t\tlog.align(l)\n\t}\n}\n\n//\n// Logger\n//\n\n// get returns the logger for source, creating one if necessary.\nfunc (log *logging) get(source string) logger {\n\tif l, ok := log.loggers[source]; ok {\n\t\treturn l\n\t}\n\n\tl := logger(len(log.loggers))\n\tlog.loggers[source] = l\n\tlog.sources[l] = source\n\tlog.align(l)\n\n\tstate, ok := log.dbgmap[source]\n\tif !ok {\n\t\tstate = log.dbgmap[\"*\"]\n\t}\n\tlog.setDebug(source, state)\n\n\treturn l\n}\n\nfunc (l logger) EnableDebug() bool {\n\tlog.Lock()\n\tdefer log.Unlock()\n\tif _, ok := log.sources[l]; !ok {\n\t\treturn false\n\t}\n\t_, old := log.debug[l]\n\tlog.debug[l] = struct{}{}\n\treturn old\n}\n\nfunc (l logger) DebugEnabled() bool {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\t_, enabled := log.debug[l]\n\treturn enabled || log.forced\n}\n\nfunc (l logger) Source() string {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\treturn log.sources[l]\n}\n\nfunc (l logger) Debug(format string, args ...interface{}) {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\n\tif !log.forced {\n\t\tif _, ok := log.debug[l]; !ok {\n\t\t\treturn\n\t\t}\n\t}\n\n\tmsg := fmt.Sprintf(format, args...)\n\n\tif log.prefix {\n\t\tklog.InfoDepth(1, levelTag[LevelDebug], log.aligned[l], msg)\n\t} else {\n\t\tklog.InfoDepth(1, msg)\n\t}\n}\n\nfunc (l logger) Info(format string, args ...interface{}) {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\n\tmsg := fmt.Sprintf(format, args...)\n\n\tif log.prefix {\n\t\tklog.InfoDepth(1, levelTag[LevelInfo], log.aligned[l], msg)\n\t} else {\n\t\tklog.InfoDepth(1, msg)\n\t}\n}\n\nfunc (l logger) Warn(format string, args ...interface{}) {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\n\tmsg := fmt.Sprintf(format, args...)\n\n\tif log.prefix {\n\t\tklog.WarningDepth(1, levelTag[LevelWarn], log.aligned[l], msg)\n\t} else {\n\t\tklog.WarningDepth(1, msg)\n\t}\n}\n\nfunc (l logger) Error(format string, args ...interface{}) {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\n\tmsg := fmt.Sprintf(format, args...)\n\tif log.prefix {\n\t\tklog.ErrorDepth(1, levelTag[LevelError], log.aligned[l], msg)\n\t} else {\n\t\tklog.ErrorDepth(1, msg)\n\t}\n}\n\nfunc (l logger) Fatal(format string, args ...interface{}) {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\n\tmsg := fmt.Sprintf(format, args...)\n\tif log.prefix {\n\t\tklog.ExitDepth(1, levelTag[LevelFatal], log.aligned[l], msg)\n\t} else {\n\t\tklog.ExitDepth(1, msg)\n\t}\n}\n\nfunc (l logger) Panic(format string, args ...interface{}) {\n\tlog.RLock()\n\tdefer log.RUnlock()\n\n\tmsg := fmt.Sprintf(format, args...)\n\tif log.prefix {\n\t\tklog.ErrorDepth(1, levelTag[LevelPanic], log.aligned[l], msg)\n\t} else {\n\t\tklog.ErrorDepth(1, msg)\n\t}\n\tpanic(msg)\n}\n\nfunc (l logger) DebugBlock(prefix string, format string, args ...interface{}) {\n\tif l.DebugEnabled() {\n\t\tl.block(LevelDebug, prefix, format, args...)\n\t}\n}\n\nfunc (l logger) InfoBlock(prefix string, format string, args ...interface{}) {\n\tl.block(LevelInfo, prefix, format, args...)\n}\n\nfunc (l logger) WarnBlock(prefix string, format string, args ...interface{}) {\n\tl.block(LevelWarn, prefix, format, args...)\n}\n\nfunc (l logger) ErrorBlock(prefix string, format string, args ...interface{}) {\n\tl.block(LevelError, prefix, format, args...)\n}\n\nfunc (l logger) block(level Level, prefix, format string, args ...interface{}) {\n\tlog.Lock()\n\tdefer log.Unlock()\n\n\tvar logFn func(int, ...interface{})\n\n\tswitch level {\n\tcase LevelDebug, LevelInfo:\n\t\tlogFn = klog.InfoDepth\n\tcase LevelWarn:\n\t\tlogFn = klog.WarningDepth\n\tcase LevelError:\n\t\tlogFn = klog.ErrorDepth\n\tdefault:\n\t\treturn\n\t}\n\n\tif log.prefix {\n\t\tsrc := log.aligned[l]\n\t\tfor _, msg := range strings.Split(fmt.Sprintf(format, args...), \"\\n\") {\n\t\t\tlogFn(2, levelTag[level], src, prefix, msg)\n\t\t}\n\t} else {\n\t\tfor _, msg := range strings.Split(fmt.Sprintf(format, args...), \"\\n\") {\n\t\t\tlogFn(2, prefix, msg)\n\t\t}\n\t}\n}\n\n// loggerError produces a formatted logger-specific error.\nfunc loggerError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"logger: \"+format, args...)\n}\n\nfunc (l logger) Debugf(format string, args ...interface{}) {\n\tl.Debug(format, args...)\n}\n\nfunc (l logger) Infof(format string, args ...interface{}) {\n\tl.Info(format, args...)\n}\n\nfunc (l logger) Warnf(format string, args ...interface{}) {\n\tl.Warn(format, args...)\n}\n\nfunc (l logger) Errorf(format string, args ...interface{}) {\n\tl.Error(format, args...)\n}\n\nfunc (l logger) Panicf(format string, args ...interface{}) {\n\tl.Panic(format, args...)\n}\n\nfunc (l logger) Fatalf(format string, args ...interface{}) {\n\tl.Fatal(format, args...)\n}\n"
  },
  {
    "path": "pkg/log/ratelimit.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"time\"\n\n\tgoxrate \"golang.org/x/time/rate\"\n)\n\n// Rate specifies maximum per-message logging rate.\ntype Rate struct {\n\t// rate limit\n\tLimit goxrate.Limit\n\t// allowed bursts\n\tBurst int\n\t// optional message window size\n\tWindow int\n}\n\n// ratelimited implements rate-limited logging with a sliding window of unique messages.\ntype ratelimited struct {\n\tLogger\n\tsync.Mutex\n\trate   Rate\n\twindow []string\n\tlimits map[string]*goxrate.Limiter\n}\n\nconst (\n\t// DefaultWindow is the default message window size for rate limiting.\n\tDefaultWindow = 256\n\t// MinimumWindow is the smallest message window size for rate limiting.\n\tMinimumWindow = 32\n)\n\n// Every defines a rate limit for the given interval.\nfunc Every(interval time.Duration) goxrate.Limit {\n\treturn goxrate.Every(interval)\n}\n\n// Interval returns a Rate for the given interval.\nfunc Interval(interval time.Duration) Rate {\n\treturn Rate{Limit: Every(interval), Burst: 1}\n}\n\n// RateLimit returns a ratelimited version of the given logger.\nfunc RateLimit(log Logger, rate Rate) Logger {\n\tswitch {\n\tcase rate.Window == 0:\n\t\trate.Window = DefaultWindow\n\tcase rate.Window < MinimumWindow:\n\t\trate.Window = MinimumWindow\n\t}\n\tif rate.Burst < 1 {\n\t\trate.Burst = 1\n\t}\n\treturn &ratelimited{\n\t\tLogger: log,\n\t\trate:   rate,\n\t\twindow: make([]string, 0, rate.Window),\n\t\tlimits: make(map[string]*goxrate.Limiter),\n\t}\n}\n\nfunc (rl *ratelimited) Debug(format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif limit := rl.getMessageLimit(msg); limit.Allow() {\n\t\trl.Logger.Debug(\"<rate-limited> %s\", msg)\n\t}\n}\n\nfunc (rl *ratelimited) Info(format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif limit := rl.getMessageLimit(msg); limit.Allow() {\n\t\trl.Logger.Info(\"<rate-limited> %s\", msg)\n\t}\n}\n\nfunc (rl *ratelimited) Warn(format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif limit := rl.getMessageLimit(msg); limit.Allow() {\n\t\trl.Logger.Warn(\"<rate-limited> %s\", msg)\n\t}\n}\n\nfunc (rl *ratelimited) Error(format string, args ...interface{}) {\n\tmsg := fmt.Sprintf(format, args...)\n\tif limit := rl.getMessageLimit(msg); limit.Allow() {\n\t\trl.Logger.Error(\"<rate-limited> %s\", msg)\n\t}\n}\n\n// Get existing message limit or create a new one, shifting out the oldest if window is full.\nfunc (rl *ratelimited) getMessageLimit(msg string) *goxrate.Limiter {\n\trl.Lock()\n\tdefer rl.Unlock()\n\n\tlimit, ok := rl.limits[msg]\n\tif ok {\n\t\treturn limit\n\t}\n\n\tlimit = goxrate.NewLimiter(rl.rate.Limit, rl.rate.Burst)\n\tif len(rl.limits) == rl.rate.Window {\n\t\tdelete(rl.limits, rl.window[0])\n\t\trl.window = rl.window[1:]\n\t}\n\trl.window = append(rl.window, msg)\n\trl.limits[msg] = limit\n\n\treturn limit\n}\n"
  },
  {
    "path": "pkg/log/ratelimit_test.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tgoxrate \"golang.org/x/time/rate\"\n)\n\nfunc TestRateLimit(t *testing.T) {\n\tratelimit := RateLimit(Default(), Rate{Window: MinimumWindow, Limit: Every(time.Second)})\n\trl := ratelimit.(*ratelimited)\n\n\tlimiters := make(map[string]*goxrate.Limiter)\n\n\t// fill message window, store limiters for checking\n\tmessages := make([]string, 0, MinimumWindow)\n\tfor idx := 0; idx < cap(messages); idx++ {\n\t\tmsg := fmt.Sprintf(\"message #%d\", idx)\n\t\tmessages = append(messages, msg)\n\t\tlimiters[msg] = rl.getMessageLimit(msg)\n\t}\n\n\t// check looked up vs. stored limters\n\tfor msg, limiter := range limiters {\n\t\tif rl.getMessageLimit(msg) != limiter {\n\t\t\tt.Errorf(\"unexpected new limiter for message %s\", msg)\n\t\t}\n\t}\n\n\t// create more messages, store limiters for checking\n\trecent := make([]string, 0, MinimumWindow/5)\n\tfor i := 0; i < cap(recent); i++ {\n\t\tmsg := fmt.Sprintf(\"message #%d\", len(messages)+i)\n\t\trecent = append(recent, msg)\n\t\tlimiters[msg] = rl.getMessageLimit(msg)\n\t}\n\n\t// check looked up vs. stored limiters\n\tfor _, msg := range recent {\n\t\tif rl.getMessageLimit(msg) != limiters[msg] {\n\t\t\tt.Errorf(\"unexpected new limiter for recent message %s\", msg)\n\t\t}\n\t}\n\n\t// check in-window part of old messages\n\tfor idx := len(recent); idx < len(messages); idx++ {\n\t\tmsg := messages[idx]\n\t\tl := rl.getMessageLimit(msg)\n\t\tif l != limiters[msg] {\n\t\t\tt.Errorf(\"unexpected new limiter for old message %s\", msg)\n\t\t}\n\t}\n\n\t// check shifted out part of old messages\n\tfor idx := 0; idx < len(recent); idx++ {\n\t\tmsg := messages[idx]\n\t\tl := rl.getMessageLimit(msg)\n\t\tif l == limiters[msg] {\n\t\t\tt.Errorf(\"unexpected old limiter for old message %s\", msg)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/log/signal.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"os\"\n\t\"os/signal\"\n)\n\n// signal notification channel\nvar signals chan os.Signal\n\n// SetupDebugToggleSignal sets up a signal handler to toggle full debugging on/off.\nfunc SetupDebugToggleSignal(sig os.Signal) {\n\tlog.Lock()\n\tdefer log.Unlock()\n\n\tclearDebugToggleSignal()\n\n\tsignals = make(chan os.Signal, 1)\n\tsignal.Notify(signals, sig)\n\n\tgo func(sig <-chan os.Signal) {\n\t\tstate := map[bool]string{false: \"off\", true: \"on\"}\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase _, ok := <-sig:\n\t\t\t\tif !ok {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tlog.forced = !log.forced\n\t\t\tdeflog.Warn(\"forced full debugging is now %s...\", state[log.forced])\n\t\t}\n\t}(signals)\n}\n\n// ClearDebugToggleSignal removes any signal handlers for toggling debug on/off.\nfunc ClearDebugToggleSignal() {\n\tlog.Lock()\n\tdefer log.Unlock()\n\tclearDebugToggleSignal()\n}\n\nfunc clearDebugToggleSignal() {\n\tif signals != nil {\n\t\tsignal.Stop(signals)\n\t\tclose(signals)\n\t\tsignals = nil\n\t}\n}\n"
  },
  {
    "path": "pkg/log/stdlog-logger.go",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\tstdlog \"log\"\n)\n\n// stdlogger implements an io.Writer to redirect logging by the stock log package.\ntype stdlogger struct {\n\tl Logger\n}\n\n// SetStdLogger sets up a logger for the standard log package.\nfunc SetStdLogger(source string) {\n\tvar l Logger\n\n\tif source == \"\" {\n\t\tl = Default()\n\t} else {\n\t\tl = log.get(source)\n\t}\n\n\tstdlog.SetPrefix(\"\")\n\tstdlog.SetFlags(0)\n\tstdlog.SetOutput(&stdlogger{l: l})\n}\n\n// Write implements io.Writer for stdlogger.\nfunc (s *stdlogger) Write(p []byte) (int, error) {\n\ts.l.Debug(\"%s\", string(p))\n\treturn len(p), nil\n}\n"
  },
  {
    "path": "pkg/metrics/metrics.go",
    "content": "package metrics\n\nimport (\n\t\"fmt\"\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\nvar (\n\tbuiltInCollectors     = make(map[string]InitCollector)\n\tregisteredCollectors  = []prometheus.Collector{}\n\tinitializedCollectors = make(map[string]struct{})\n\tlog                   = logger.NewLogger(\"collectors\")\n)\n\n// InitCollector is the type for functions that initialize collectors.\ntype InitCollector func() (prometheus.Collector, error)\n\n// RegisterCollector registers the named prometheus.Collector for metrics collection.\nfunc RegisterCollector(name string, init InitCollector) error {\n\tlog.Info(\"registering collector %s...\", name)\n\n\tif _, found := builtInCollectors[name]; found {\n\t\treturn metricsError(\"Collector %s already registered\", name)\n\t}\n\n\tbuiltInCollectors[name] = init\n\n\treturn nil\n}\n\n// NewMetricGatherer creates a new prometheus.Gatherer with all registered collectors.\nfunc NewMetricGatherer() (prometheus.Gatherer, error) {\n\treg := prometheus.NewPedanticRegistry()\n\n\tfor name, cb := range builtInCollectors {\n\t\tif _, ok := initializedCollectors[name]; ok {\n\t\t\tcontinue\n\t\t}\n\n\t\tc, err := cb()\n\t\tif err != nil {\n\t\t\tlog.Error(\"Failed to initialize collector '%s': %v. Skipping it.\", name, err)\n\t\t\tcontinue\n\t\t}\n\t\tregisteredCollectors = append(registeredCollectors, c)\n\t\tinitializedCollectors[name] = struct{}{}\n\t}\n\n\treg.MustRegister(registeredCollectors[:]...)\n\n\treturn reg, nil\n}\n\nfunc metricsError(format string, args ...interface{}) error {\n\treturn fmt.Errorf(\"metrics: \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/metrics/register/register_metrics.go",
    "content": "package register\n\nimport (\n\t// Pull in cgroup-based metric collector.\n\t_ \"github.com/intel/cri-resource-manager/pkg/cgroupstats\"\n)\n"
  },
  {
    "path": "pkg/metrics/register/register_metrics_avx.go",
    "content": "//go:build !noavx\n// +build !noavx\n\npackage register\n\nimport (\n\t// Pull in avx collector.\n\t_ \"github.com/intel/cri-resource-manager/pkg/avx\"\n)\n"
  },
  {
    "path": "pkg/pidfile/pidfile.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pidfile\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/pkg/errors\"\n)\n\nvar (\n\tpidFilePath = defaultPath()\n\tpidFile     *os.File\n)\n\n// GetPath returns the current pidfile path.\nfunc GetPath() string {\n\treturn pidFilePath\n}\n\n// SetPath sets the pidfile path to the given one.\nfunc SetPath(path string) {\n\tclosePIDFile()\n\tpidFilePath = path\n}\n\n// Write opens the PID file and writes os.Getpid() to it. If the PID file already\n// exists Write() fails with an error. On successful completion, Write keeps the\n// PID file open.\nfunc Write() error {\n\tif pidFile != nil {\n\t\treturn nil\n\t}\n\n\terr := os.MkdirAll(filepath.Dir(pidFilePath), 0755)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to create PID file\")\n\t}\n\n\tpidFile, err = os.OpenFile(pidFilePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to create PID file\")\n\t}\n\n\t_, err = pidFile.Write([]byte(fmt.Sprintf(\"%d\\n\", os.Getpid())))\n\tif err != nil {\n\t\tclosePIDFile()\n\t\treturn errors.Wrap(err, \"failed to write PID file\")\n\t}\n\n\treturn nil\n}\n\n// Read reads the content of the PID file. It returns the process ID found\n// in the file. If opening the file or reading an integer process ID fails\n// Read() returns -1 and an error.\nfunc Read() (int, error) {\n\tvar (\n\t\tpid int\n\t\tbuf []byte\n\t\terr error\n\t)\n\n\tif buf, err = os.ReadFile(pidFilePath); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn 0, nil\n\t\t}\n\t\treturn -1, errors.Wrap(err, \"failed to read PID file\")\n\t}\n\n\tif pid, err = strconv.Atoi(strings.TrimRight(string(buf), \"\\n\")); err != nil {\n\t\treturn -1, errors.Wrapf(err, \"invalid PID (%q) in PID file\", string(buf))\n\t}\n\n\treturn pid, nil\n}\n\n// closePIDFile closes the PID file and truncates it to zero length.\nfunc closePIDFile() {\n\tif pidFile != nil {\n\t\tpidFile.Truncate(0)\n\t\tpidFile.Close()\n\t\tpidFile = nil\n\t}\n}\n\n// Remove removes the PID file for the process unconditionally, regardless if\n// the current process had created the PID file or not.\nfunc Remove() error {\n\tclosePIDFile()\n\terr := os.Remove(pidFilePath)\n\tif err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn err\n}\n\n// OwnerPid returns the ID of the process owning the PID file. 0 is returned\n// if it is known that no process owns the file. -1 and an error is returned\n// if the owner or its existence could not be determined.\nfunc OwnerPid() (int, error) {\n\tvar (\n\t\tpid int\n\t\tp   *os.Process\n\t\terr error\n\t)\n\n\tpid, err = Read()\n\tif err != nil {\n\t\treturn -1, err\n\t}\n\tif pid == 0 {\n\t\treturn 0, nil\n\t}\n\n\tp, err = os.FindProcess(pid)\n\tif err != nil {\n\t\treturn -1, errors.Wrapf(err, \"FindProcess() failed for PID %d\", pid)\n\t}\n\n\terr = p.Signal(syscall.Signal(0))\n\tif err == os.ErrProcessDone {\n\t\treturn 0, nil\n\t}\n\tif err == nil {\n\t\treturn pid, nil\n\t}\n\n\treturn -1, errors.Wrapf(err, \"failed to check process %d\", pid)\n}\n\n// defaultPath returns the default pidfile path.\nfunc defaultPath() string {\n\tvar path string\n\n\tif len(os.Args) > 0 {\n\t\tname := filepath.Base(os.Args[0])\n\t\tif euid := os.Geteuid(); euid > 0 {\n\t\t\tpath = filepath.Join(\"/tmp\", name+\".pid\")\n\t\t} else {\n\t\t\tpath = filepath.Join(\"/\", \"var\", \"run\", name+\".pid\")\n\t\t}\n\t}\n\n\treturn path\n}\n"
  },
  {
    "path": "pkg/pidfile/pidfile_test.go",
    "content": "// Copyright 2022 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage pidfile\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nconst (\n\ttestPidFile = \"pidfile-test.pid\"\n)\n\nfunc prepare(t *testing.T) string {\n\tdir, err := mkTestDir(t)\n\tif err != nil {\n\t\tt.Errorf(\"failed to create test directory: %v\", err)\n\t\tos.Exit(1)\n\t}\n\n\tSetPath(filepath.Join(dir, testPidFile))\n\treturn dir\n}\n\nfunc TestDefaults(t *testing.T) {\n\tt.Run(\"TestDefaults\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\terr error\n\t\t)\n\n\t\tRemove()\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\n\t\tclosePIDFile()\n\t\terr = Write()\n\t\trequire.NotNil(t, err)\n\n\t\tRemove()\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\t})\n}\n\nfunc TestGetSetPath(t *testing.T) {\n\tt.Run(\"TestTestGetSetPath\", func(t *testing.T) {\n\t\tvar (\n\t\t\tdir  string\n\t\t\tpath string\n\t\t)\n\n\t\tdir = prepare(t)\n\t\tpath = GetPath()\n\t\trequire.Equal(t, path, filepath.Join(dir, testPidFile))\n\t})\n}\n\nfunc TestReadNonExisting(t *testing.T) {\n\tt.Run(\"TestReadNonExisting\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, 0)\n\t})\n}\n\nfunc TestRemoveNonExisting(t *testing.T) {\n\tt.Run(\"TestRemoveNonExisting\", func(t *testing.T) {\n\t\tprepare(t)\n\t\terr := Remove()\n\t\trequire.Nil(t, err)\n\t})\n}\n\nfunc TestRemoveExisting(t *testing.T) {\n\tt.Run(\"TestRemoveExisting\", func(t *testing.T) {\n\t\tvar (\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\terr = Remove()\n\t\trequire.Nil(t, err)\n\t})\n}\n\nfunc TestWrite(t *testing.T) {\n\tt.Run(\"TestWrite\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\t})\n}\n\nfunc TestReadClosed(t *testing.T) {\n\tt.Run(\"TestReadClosed\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\n\t\tclosePIDFile()\n\t\tpid, err = Read()\n\t\trequire.NotNil(t, err)\n\t\trequire.Equal(t, pid, -1)\n\t})\n}\n\nfunc TestFailToOverwrite(t *testing.T) {\n\tt.Run(\"TestFailToOverwrite\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\n\t\tclosePIDFile()\n\t\terr = Write()\n\t\trequire.NotNil(t, err)\n\t})\n}\n\nfunc TestRemoveToOverwrite(t *testing.T) {\n\tt.Run(\"TestRemoveToOverwrite\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\n\t\terr = Remove()\n\t\trequire.Nil(t, err)\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\t})\n}\n\nfunc TestOwnerPid(t *testing.T) {\n\tt.Run(\"TestOwnerPid\", func(t *testing.T) {\n\t\tvar (\n\t\t\tpid int\n\t\t\tchk int\n\t\t\terr error\n\t\t)\n\n\t\tprepare(t)\n\n\t\terr = Write()\n\t\trequire.Nil(t, err)\n\n\t\tpid, err = Read()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, os.Getpid())\n\n\t\tchk, err = OwnerPid()\n\t\trequire.Nil(t, err)\n\t\trequire.Equal(t, pid, chk)\n\t})\n}\n\nfunc mkTestDir(t *testing.T) (string, error) {\n\ttmp, err := os.MkdirTemp(\"\", \".pidfile-test*\")\n\tif err != nil {\n\t\treturn \"\", errors.Wrapf(err, \"failed to create test directory\")\n\t}\n\n\tt.Cleanup(func() {\n\t\tos.RemoveAll(tmp)\n\t})\n\n\treturn tmp, nil\n}\n"
  },
  {
    "path": "pkg/policycollector/collector.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage policycollector\n\nimport (\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/policy\"\n\t\"github.com/intel/cri-resource-manager/pkg/metrics\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n)\n\ntype PolicyCollector struct {\n\tpolicy policy.Policy\n}\n\nfunc (c *PolicyCollector) SetPolicy(policy policy.Policy) {\n\tc.policy = policy\n}\n\n// HasPolicySpecificMetrics judges whether the policy defines the policy-specific metrics\nfunc (c *PolicyCollector) HasPolicySpecificMetrics() bool {\n\tif c.policy.DescribeMetrics() == nil {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Describe implements prometheus.Collector interface\nfunc (c *PolicyCollector) Describe(ch chan<- *prometheus.Desc) {\n\tfor _, d := range c.policy.DescribeMetrics() {\n\t\tch <- d\n\t}\n}\n\n// Collect implements prometheus.Collector interface\nfunc (c *PolicyCollector) Collect(ch chan<- prometheus.Metric) {\n\tprometheusMetrics, err := c.policy.CollectMetrics(c.policy.PollMetrics())\n\tif err != nil {\n\t\treturn\n\t}\n\tfor _, m := range prometheusMetrics {\n\t\tch <- m\n\t}\n}\n\n// RegisterPolicyMetricsCollector registers policy-specific collector\nfunc (c *PolicyCollector) RegisterPolicyMetricsCollector() error {\n\treturn metrics.RegisterCollector(\"policyMetrics\", func() (prometheus.Collector, error) {\n\t\treturn c, nil\n\t})\n}\n"
  },
  {
    "path": "pkg/procstats/procstats.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage procstats\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/sysfs\"\n)\n\n// CPUTimeStat is used to calculate the CPU usage.\ntype CPUTimeStat struct {\n\tsync.RWMutex\n\tPrevIdleTime       []uint64\n\tPrevTotalTime      []uint64\n\tCurIdleTime        []uint64\n\tCurTotalTime       []uint64\n\tDeltaIdleTime      []uint64\n\tDeltaTotalTime     []uint64\n\tCPUUsage           []float64\n\tIsGetCPUUsageBegin bool\n}\n\nvar (\n\t// procRoot is the mount point for the proc filesystem\n\tprocRoot = \"/proc\"\n\tprocStat = procRoot + \"/stat\"\n)\n\n// GetCPUTimeStat calculates CPU usage by using the CPU time statistics from /proc/stat\nfunc (t *CPUTimeStat) GetCPUTimeStat() error {\n\t// /proc/stat looks like this:\n\t// cpuid: user, nice, system, idle, iowait, irq, softirq\n\t// cpu  130216 19944 162525 1491240 3784 24749 17773 0 0 0\n\t// cpu0 40321 11452 49784 403099 2615 6076 6748 0 0 0\n\t// cpu1 26585 2425 36639 151166 404 2533 3541 0 0 0\n\t// ...\n\tstats, err := os.ReadFile(procStat)\n\tif err != nil {\n\t\treturn err\n\t}\n\tt.Lock()\n\tdefer t.Unlock()\n\tsys, err := sysfs.DiscoverSystem()\n\tif err != nil {\n\t\treturn err\n\t}\n\tcpuCount := len(sys.CPUIDs())\n\tfor index, line := range strings.Split(string(stats), \"\\n\") {\n\t\tif index > cpuCount {\n\t\t\tbreak\n\t\t}\n\t\tsplit := strings.Split(line, \" \")\n\n\t\tif strings.HasPrefix(split[0], \"cpu\") && split[0] != \"cpu\" {\n\t\t\ti, err := strconv.Atoi(split[0][3:])\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(\"Fail to get CPU index.\")\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tt.CurIdleTime[i], err = strconv.ParseUint(split[4], 10, 64)\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(\"Fail to get idle time.\")\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttotalTime := uint64(0)\n\t\t\tfor _, s := range split[1:] {\n\t\t\t\tu, err := strconv.ParseUint(s, 10, 64)\n\t\t\t\tif err == nil {\n\t\t\t\t\ttotalTime += u\n\t\t\t\t}\n\t\t\t}\n\t\t\tt.CurTotalTime[i] = totalTime\n\t\t\tt.CPUUsage[i] = 0.0\n\t\t\tif t.IsGetCPUUsageBegin {\n\t\t\t\tt.DeltaIdleTime[i] = t.CurIdleTime[i] - t.PrevIdleTime[i]\n\t\t\t\tt.DeltaTotalTime[i] = t.CurTotalTime[i] - t.PrevTotalTime[i]\n\t\t\t\tif t.DeltaTotalTime[i] != 0 {\n\t\t\t\t\tt.CPUUsage[i] = (1.0 - float64(t.DeltaIdleTime[i])/float64(t.DeltaTotalTime[i])) * 100.0\n\t\t\t\t}\n\t\t\t}\n\t\t\tt.PrevIdleTime[i] = t.CurIdleTime[i]\n\t\t\tt.PrevTotalTime[i] = t.CurTotalTime[i]\n\t\t}\n\t}\n\tfor _, i := range sys.Offlined().List() {\n\t\tt.DeltaIdleTime[i] = 0.0\n\t\tt.DeltaTotalTime[i] = 0.0\n\t\tt.PrevIdleTime[i] = t.CurIdleTime[i]\n\t\tt.PrevTotalTime[i] = t.CurTotalTime[i]\n\t\tt.CPUUsage[i] = 0.0\n\t}\n\tt.IsGetCPUUsageBegin = true\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/sysfs/error.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sysfs\n\nimport (\n\t\"fmt\"\n)\n\nfunc sysfsError(path, format string, args ...interface{}) error {\n\treturn fmt.Errorf(path+\": \"+format, args...)\n}\n"
  },
  {
    "path": "pkg/sysfs/parsers.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sysfs\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n)\n\n// unit multipliers\nconst (\n\tk = (int64(1) << 10)\n\tM = (int64(1) << 20)\n\tG = (int64(1) << 30)\n\tT = (int64(1) << 40)\n)\n\n// unit name to multiplier mapping\nvar units = map[string]int64{\n\t\"k\": k, \"kB\": k,\n\t\"M\": M, \"MB\": M,\n\t\"G\": G, \"GB\": G,\n\t\"T\": T, \"TB\": T,\n}\n\n// PickEntryFn picks a given input line apart into an entry of key and value.\ntype PickEntryFn func(string) (string, string, error)\n\n// splitNumericAndUnit splits a string into a numeric and a unit part.\nfunc splitNumericAndUnit(path string, value string) (string, int64, error) {\n\tfields := strings.Fields(value)\n\n\tswitch len(fields) {\n\tcase 1:\n\t\treturn fields[0], 1, nil\n\tcase 2:\n\t\tnum := fields[0]\n\t\tunit, ok := units[fields[1]]\n\t\tif !ok {\n\t\t\treturn \"\", -1, sysfsError(path, \"failed to parse '%s', invalid unit '%s'\",\n\t\t\t\tvalue, num, unit)\n\t\t}\n\t\treturn num, unit, nil\n\t}\n\n\treturn \"\", -1, sysfsError(path, \"invalid numeric value %s\", value)\n}\n\n// PparseNumberic parses a numeric string into integer of the right size.\nfunc parseNumeric(path, value string, ptr interface{}) error {\n\tvar numstr string\n\tvar num, unit int64\n\tvar f float64\n\tvar err error\n\n\tif numstr, unit, err = splitNumericAndUnit(path, value); err != nil {\n\t\treturn err\n\t}\n\n\tswitch ptr.(type) {\n\tcase *int:\n\t\tnum, err = strconv.ParseInt(numstr, 0, strconv.IntSize)\n\t\t*ptr.(*int) = int(num * unit)\n\tcase *int8:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 8)\n\t\t*ptr.(*int8) = int8(num * unit)\n\tcase *int16:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 16)\n\t\t*ptr.(*int16) = int16(num * unit)\n\tcase *int32:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 32)\n\t\t*ptr.(*int32) = int32(num * unit)\n\tcase *int64:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 64)\n\t\t*ptr.(*int64) = int64(num * unit)\n\tcase *uint:\n\t\tnum, err = strconv.ParseInt(numstr, 0, strconv.IntSize)\n\t\t*ptr.(*uint) = uint(num * unit)\n\tcase *uint8:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 8)\n\t\t*ptr.(*uint8) = uint8(num * unit)\n\tcase *uint16:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 16)\n\t\t*ptr.(*uint16) = uint16(num * unit)\n\tcase *uint32:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 32)\n\t\t*ptr.(*uint32) = uint32(num * unit)\n\tcase *uint64:\n\t\tnum, err = strconv.ParseInt(numstr, 0, 64)\n\t\t*ptr.(*uint64) = uint64(num * unit)\n\tcase *float32:\n\t\tf, err = strconv.ParseFloat(numstr, 32)\n\t\t*ptr.(*float32) = float32(f) * float32(unit)\n\tcase *float64:\n\t\tf, err = strconv.ParseFloat(numstr, 64)\n\t\t*ptr.(*float64) = f * float64(unit)\n\n\tdefault:\n\t\terr = sysfsError(path, \"can't parse numeric value '%s' into type %T\", value, ptr)\n\t}\n\n\treturn err\n}\n\n// ParseFileEntries parses a sysfs files for the given entries.\nfunc ParseFileEntries(path string, values map[string]interface{}, pickFn PickEntryFn) error {\n\tvar err error\n\n\tdata, err := os.ReadFile(path)\n\tif err != nil {\n\t\tsysfsError(path, \"failed to read file: %v\", err)\n\t}\n\n\tleft := len(values)\n\tfor _, line := range strings.Split(string(data), \"\\n\") {\n\t\tkey, value, err := pickFn(line)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tptr, ok := values[key]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\n\t\tswitch ptr.(type) {\n\t\tcase *int, *int8, *int32, *int16, *int64, *uint, *uint8, *uint16, *uint32, *uint64:\n\t\t\tif err = parseNumeric(path, value, ptr); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase *float32, *float64:\n\t\t\tif err = parseNumeric(path, value, ptr); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase *string:\n\t\t\t*ptr.(*string) = value\n\t\tcase *bool:\n\t\t\t*ptr.(*bool), err = strconv.ParseBool(value)\n\t\t\tif err != nil {\n\t\t\t\treturn sysfsError(path, \"failed to parse line %s, value '%s' for boolean key '%s'\",\n\t\t\t\t\tline, value, key)\n\t\t\t}\n\t\tdefault:\n\t\t\treturn sysfsError(path, \"don't know how to parse key '%s' of type %T\", key, ptr)\n\n\t\t}\n\n\t\tleft--\n\t\tif left == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/sysfs/system.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sysfs\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\t\"github.com/intel/goresctrl/pkg/sst\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\nvar (\n\t// Parent directory under which host sysfs, etc. is mounted (if non-standard location).\n\tsysRoot = \"\"\n)\n\nconst (\n\t// sysfs devices/cpu subdirectory path\n\tsysfsCPUPath = \"devices/system/cpu\"\n\t// sysfs device/node subdirectory path\n\tsysfsNumaNodePath = \"devices/system/node\"\n)\n\n// MemoryType is an enum for the Node memory\ntype MemoryType int\n\nconst (\n\t// MemoryTypeDRAM means that the node has regular DRAM-type memory\n\tMemoryTypeDRAM MemoryType = iota\n\t// MemoryTypePMEM means that the node has persistent memory\n\tMemoryTypePMEM\n\t// MemoryTypeHBM means that the node has high bandwidth memory\n\tMemoryTypeHBM\n)\n\n// System devices\ntype System interface {\n\tDiscover() error\n\tSetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error)\n\tSetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error\n\tPackageIDs() []idset.ID\n\tNodeIDs() []idset.ID\n\tCPUIDs() []idset.ID\n\tPackageCount() int\n\tSocketCount() int\n\tCPUCount() int\n\tNUMANodeCount() int\n\tThreadCount() int\n\tCPUSet() cpuset.CPUSet\n\tPackage(id idset.ID) CPUPackage\n\tNode(id idset.ID) Node\n\tNodeDistance(from, to idset.ID) int\n\tCPU(id idset.ID) CPU\n\tOfflined() cpuset.CPUSet\n\tIsolated() cpuset.CPUSet\n}\n\n// System devices\ntype system struct {\n\tlogger.Logger                          // our logger instance\n\tpath          string                   // sysfs mount point\n\tpackages      map[idset.ID]*cpuPackage // physical packages\n\tnodes         map[idset.ID]*node       // NUMA nodes\n\tcpus          map[idset.ID]*cpu        // CPUs\n\tcache         map[idset.ID]*Cache      // Cache\n\toffline       idset.IDSet              // offlined CPUs\n\tisolated      idset.IDSet              // isolated CPUs\n\tthreads       int                      // hyperthreads per core\n}\n\n// CPUPackage is a physical package (a collection of CPUs).\ntype CPUPackage interface {\n\tID() idset.ID\n\tCPUSet() cpuset.CPUSet\n\tDieIDs() []idset.ID\n\tNodeIDs() []idset.ID\n\tDieNodeIDs(idset.ID) []idset.ID\n\tDieCPUSet(idset.ID) cpuset.CPUSet\n\tSstInfo() *sst.SstPackageInfo\n}\n\ntype cpuPackage struct {\n\tid       idset.ID                 // package id\n\tcpus     idset.IDSet              // CPUs in this package\n\tnodes    idset.IDSet              // nodes in this package\n\tdies     idset.IDSet              // dies in this package\n\tdieCPUs  map[idset.ID]idset.IDSet // CPUs per die\n\tdieNodes map[idset.ID]idset.IDSet // NUMA nodes per die\n\tsstInfo  *sst.SstPackageInfo      // Speed Select Technology info\n}\n\n// Node represents a NUMA node.\ntype Node interface {\n\tID() idset.ID\n\tPackageID() idset.ID\n\tDieID() idset.ID\n\tCPUSet() cpuset.CPUSet\n\tDistance() []int\n\tDistanceFrom(id idset.ID) int\n\tMemoryInfo() (*MemInfo, error)\n\tGetMemoryType() MemoryType\n\tHasNormalMemory() bool\n}\n\ntype node struct {\n\tpath       string      // sysfs path\n\tid         idset.ID    // node id\n\tpkg        idset.ID    // package id\n\tdie        idset.ID    // die id\n\tcpus       idset.IDSet // cpus in this node\n\tmemoryType MemoryType  // node memory type\n\tnormalMem  bool        // node has memory in a normal (kernel space allocatable) zone\n\tdistance   []int       // distance/cost to other NUMA nodes\n}\n\n// CPU is a CPU core.\ntype CPU interface {\n\tID() idset.ID\n\tPackageID() idset.ID\n\tDieID() idset.ID\n\tNodeID() idset.ID\n\tCoreID() idset.ID\n\tThreadCPUSet() cpuset.CPUSet\n\tBaseFrequency() uint64\n\tFrequencyRange() CPUFreq\n\tEPP() EPP\n\tOnline() bool\n\tIsolated() bool\n\tSetFrequencyLimits(min, max uint64) error\n\tSstClos() int\n}\n\ntype cpu struct {\n\tpath     string      // sysfs path\n\tid       idset.ID    // CPU id\n\tpkg      idset.ID    // package id\n\tdie      idset.ID    // die id\n\tnode     idset.ID    // node id\n\tcore     idset.ID    // core id\n\tthreads  idset.IDSet // sibling/hyper-threads\n\tbaseFreq uint64      // CPU base frequency\n\tfreq     CPUFreq     // CPU frequencies\n\tepp      EPP         // Energy Performance Preference from cpufreq governor\n\tonline   bool        // whether this CPU is online\n\tisolated bool        // whether this CPU is isolated\n\tsstClos  int         // SST-CP CLOS the CPU is associated with\n}\n\n// CPUFreq is a CPU frequency scaling range\ntype CPUFreq struct {\n\tmin uint64   // minimum frequency (kHz)\n\tmax uint64   // maximum frequency (kHz)\n\tall []uint64 // discrete set of frequencies if applicable/known\n}\n\n// EPP represents the value of a CPU energy performance profile\ntype EPP int\n\nconst (\n\tEPPPerformance EPP = iota\n\tEPPBalancePerformance\n\tEPPBalancePower\n\tEPPPower\n\tEPPUnknown\n)\n\n// MemInfo contains data read from a NUMA node meminfo file.\ntype MemInfo struct {\n\tMemTotal uint64\n\tMemFree  uint64\n\tMemUsed  uint64\n}\n\n// CPU cache.\n//   Notes: cache-discovery is forced off now (by forcibly clearing the related discovery bit)\n//      Can't seem to make sense of the cache information exposed under sysfs. The cache ids\n//      do not seem to be unique, which IIUC is contrary to the documentation.\n\n// CacheType specifies a cache type.\ntype CacheType string\n\nconst (\n\t// DataCache marks data cache.\n\tDataCache CacheType = \"Data\"\n\t// InstructionCache marks instruction cache.\n\tInstructionCache CacheType = \"Instruction\"\n\t// UnifiedCache marks a unified data/instruction cache.\n\tUnifiedCache CacheType = \"Unified\"\n)\n\n// Cache has details about cache.\ntype Cache struct {\n\tid    idset.ID    // cache id\n\tkind  CacheType   // cache type\n\tsize  uint64      // cache size\n\tlevel uint8       // cache level\n\tcpus  idset.IDSet // CPUs sharing this cache\n}\n\n// SetSysRoot sets the sys root directory.\nfunc SetSysRoot(path string) {\n\tsysRoot = path\n}\n\n// SysRoot returns the sys root directory.\nfunc SysRoot() string {\n\treturn sysRoot\n}\n\n// DiscoverSystem performs discovery of the running systems details.\nfunc DiscoverSystem() (System, error) {\n\treturn DiscoverSystemAt(filepath.Join(\"/\", sysRoot, \"sys\"))\n}\n\n// DiscoverSystemAt performs discovery of the running systems details from sysfs mounted at path.\nfunc DiscoverSystemAt(path string) (System, error) {\n\tsys := &system{\n\t\tLogger:  logger.NewLogger(\"sysfs\"),\n\t\tpath:    path,\n\t\toffline: idset.NewIDSet(),\n\t}\n\n\tif err := sys.Discover(); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn sys, nil\n}\n\n// Discover performs system/hardware discovery.\nfunc (sys *system) Discover() error {\n\tif err := sys.discoverCPUs(); err != nil {\n\t\treturn err\n\t}\n\tif err := sys.discoverNodes(); err != nil {\n\t\treturn err\n\t}\n\tif err := sys.discoverPackages(); err != nil {\n\t\treturn err\n\t}\n\n\tif err := sys.discoverSst(); err != nil {\n\t\t// Just consider SST unsupported if our detection fails for some reason\n\t\tsys.Warn(\"%v\", err)\n\t}\n\n\tif len(sys.nodes) > 0 {\n\t\tfor _, pkg := range sys.packages {\n\t\t\tfor _, nodeID := range pkg.nodes.SortedMembers() {\n\t\t\t\tif node, ok := sys.nodes[nodeID]; ok {\n\t\t\t\t\tnode.pkg = pkg.id\n\t\t\t\t} else {\n\t\t\t\t\treturn sysfsError(\"NUMA nodes\", \"can't find NUMA node for ID %d\", nodeID)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, dieID := range pkg.DieIDs() {\n\t\t\t\tfor _, nodeID := range pkg.DieNodeIDs(dieID) {\n\t\t\t\t\tif node, ok := sys.nodes[nodeID]; ok {\n\t\t\t\t\t\tnode.die = dieID\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn sysfsError(\"NUMA nodes\", \"can't find NUMA node for ID %d\", nodeID)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif sys.DebugEnabled() {\n\t\tfor id, pkg := range sys.packages {\n\t\t\tsys.Info(\"package #%d:\", id)\n\t\t\tsys.Debug(\"   cpus: %s\", pkg.cpus)\n\t\t\tsys.Debug(\"  nodes: %s\", pkg.nodes)\n\t\t\tsys.Debug(\"   dies: %s\", pkg.dies)\n\t\t\tfor die := range pkg.dies {\n\t\t\t\tsys.Debug(\"    die #%v nodes: %v\", die, pkg.DieNodeIDs(die))\n\t\t\t\tsys.Debug(\"    die #%v cpus: %s\", die, pkg.DieCPUSet(die).String())\n\t\t\t}\n\t\t}\n\n\t\tfor id, node := range sys.nodes {\n\t\t\tsys.Debug(\"node #%d:\", id)\n\t\t\tsys.Debug(\"      cpus: %s\", node.cpus)\n\t\t\tsys.Debug(\"  distance: %v\", node.distance)\n\t\t\tsys.Debug(\"   package: #%d\", node.pkg)\n\t\t\tsys.Debug(\"       die: #%d\", node.die)\n\t\t}\n\n\t\tfor id, cpu := range sys.cpus {\n\t\t\tsys.Debug(\"CPU #%d:\", id)\n\t\t\tsys.Debug(\"        pkg: %d\", cpu.pkg)\n\t\t\tsys.Debug(\"        die: %d\", cpu.die)\n\t\t\tsys.Debug(\"       node: %d\", cpu.node)\n\t\t\tsys.Debug(\"       core: %d\", cpu.core)\n\t\t\tsys.Debug(\"    threads: %s\", cpu.threads)\n\t\t\tsys.Debug(\"  base freq: %d\", cpu.baseFreq)\n\t\t\tsys.Debug(\"       freq: %d - %d\", cpu.freq.min, cpu.freq.max)\n\t\t\tsys.Debug(\"        epp: %d\", cpu.epp)\n\t\t}\n\n\t\tsys.Debug(\"offline CPUs: %s\", sys.offline)\n\t\tsys.Debug(\"isolated CPUs: %s\", sys.isolated)\n\n\t\tfor id, cch := range sys.cache {\n\t\t\tsys.Debug(\"cache #%d:\", id)\n\t\t\tsys.Debug(\"   type: %v\", cch.kind)\n\t\t\tsys.Debug(\"   size: %d\", cch.size)\n\t\t\tsys.Debug(\"  level: %d\", cch.level)\n\t\t\tsys.Debug(\"   CPUs: %s\", cch.cpus)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// SetCpusOnline puts a set of CPUs online. Return the toggled set. Nil set implies all CPUs.\nfunc (sys *system) SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error) {\n\tvar entries []string\n\n\tif cpus == nil {\n\t\tentries, _ = filepath.Glob(filepath.Join(sys.path, sysfsCPUPath, \"cpu[0-9]*\"))\n\t} else {\n\t\tentries = make([]string, cpus.Size())\n\t\tfor idx, id := range cpus.Members() {\n\t\t\tentries[idx] = sys.path + \"/\" + sysfsCPUPath + \"/cpu\" + strconv.Itoa(int(id))\n\t\t}\n\t}\n\n\tdesired := map[bool]int{false: 0, true: 1}[online]\n\tchanged := idset.NewIDSet()\n\n\tfor _, entry := range entries {\n\t\tvar current int\n\n\t\tid := getEnumeratedID(entry)\n\t\tif id <= 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tif _, err := writeSysfsEntry(entry, \"online\", desired, &current); err != nil {\n\t\t\treturn nil, sysfsError(entry, \"failed to set online to %d: %v\", desired, err)\n\t\t}\n\n\t\tif desired != current {\n\t\t\tchanged.Add(id)\n\t\t\tif cpu, found := sys.cpus[id]; found {\n\t\t\t\tcpu.online = online\n\n\t\t\t\tif online {\n\t\t\t\t\tsys.offline.Del(id)\n\t\t\t\t} else {\n\t\t\t\t\tsys.offline.Add(id)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn changed, nil\n}\n\n// SetCPUFrequencyLimits sets the CPU frequency scaling limits. Nil set implies all CPUs.\nfunc (sys *system) SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error {\n\tif cpus == nil {\n\t\tcpus = idset.NewIDSet(sys.CPUIDs()...)\n\t}\n\n\tfor _, id := range cpus.Members() {\n\t\tif cpu, ok := sys.cpus[id]; ok {\n\t\t\tif err := cpu.SetFrequencyLimits(min, max); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// PackageIDs gets the ids of all packages present in the system.\nfunc (sys *system) PackageIDs() []idset.ID {\n\tids := make([]idset.ID, len(sys.packages))\n\tidx := 0\n\tfor id := range sys.packages {\n\t\tids[idx] = id\n\t\tidx++\n\t}\n\n\tsort.Slice(ids, func(i, j int) bool {\n\t\treturn int(ids[i]) < int(ids[j])\n\t})\n\n\treturn ids\n}\n\n// NodeIDs gets the ids of all NUMA nodes present in the system.\nfunc (sys *system) NodeIDs() []idset.ID {\n\tids := make([]idset.ID, len(sys.nodes))\n\tidx := 0\n\tfor id := range sys.nodes {\n\t\tids[idx] = id\n\t\tidx++\n\t}\n\n\tsort.Slice(ids, func(i, j int) bool {\n\t\treturn int(ids[i]) < int(ids[j])\n\t})\n\n\treturn ids\n}\n\n// CPUIDs gets the ids of all CPUs present in the system.\nfunc (sys *system) CPUIDs() []idset.ID {\n\tids := make([]idset.ID, len(sys.cpus))\n\tidx := 0\n\tfor id := range sys.cpus {\n\t\tids[idx] = id\n\t\tidx++\n\t}\n\n\tsort.Slice(ids, func(i, j int) bool {\n\t\treturn int(ids[i]) < int(ids[j])\n\t})\n\n\treturn ids\n}\n\n// PackageCount returns the number of discovered CPU packages (sockets).\nfunc (sys *system) PackageCount() int {\n\treturn len(sys.packages)\n}\n\n// SocketCount returns the number of discovered CPU packages (sockets).\nfunc (sys *system) SocketCount() int {\n\treturn len(sys.packages)\n}\n\n// CPUCount resturns the number of discovered CPUs/cores.\nfunc (sys *system) CPUCount() int {\n\treturn len(sys.cpus)\n}\n\n// NUMANodeCount returns the number of discovered NUMA nodes.\nfunc (sys *system) NUMANodeCount() int {\n\tcnt := len(sys.nodes)\n\tif cnt < 1 {\n\t\tcnt = 1\n\t}\n\treturn cnt\n}\n\n// ThreadCount returns the number of threads per core discovered.\nfunc (sys *system) ThreadCount() int {\n\treturn sys.threads\n}\n\n// CPUSet gets the ids of all CPUs present in the system as a CPUSet.\nfunc (sys *system) CPUSet() cpuset.CPUSet {\n\treturn CPUSetFromIDSet(idset.NewIDSet(sys.CPUIDs()...))\n}\n\n// Package gets the package with a given package id.\nfunc (sys *system) Package(id idset.ID) CPUPackage {\n\treturn sys.packages[id]\n}\n\n// Node gets the node with a given node id.\nfunc (sys *system) Node(id idset.ID) Node {\n\treturn sys.nodes[id]\n}\n\n// NodeDistance gets the distance between two NUMA nodes.\nfunc (sys *system) NodeDistance(from, to idset.ID) int {\n\treturn sys.nodes[from].DistanceFrom(to)\n}\n\n// CPU gets the CPU with a given CPU id.\nfunc (sys *system) CPU(id idset.ID) CPU {\n\treturn sys.cpus[id]\n}\n\n// Offlined gets the set of offlined CPUs.\nfunc (sys *system) Offlined() cpuset.CPUSet {\n\treturn CPUSetFromIDSet(sys.offline)\n}\n\n// Isolated gets the set of isolated CPUs.\"\nfunc (sys *system) Isolated() cpuset.CPUSet {\n\treturn CPUSetFromIDSet(sys.isolated)\n}\n\n// Discover Cpus present in the system.\nfunc (sys *system) discoverCPUs() error {\n\tif sys.cpus != nil {\n\t\treturn nil\n\t}\n\n\tsys.cpus = make(map[idset.ID]*cpu)\n\n\t_, err := readSysfsEntry(sys.path, filepath.Join(sysfsCPUPath, \"isolated\"), &sys.isolated, \",\")\n\tif err != nil {\n\t\tsys.Error(\"failed to get set of isolated cpus: %v\", err)\n\t}\n\n\tentries, _ := filepath.Glob(filepath.Join(sys.path, sysfsCPUPath, \"cpu[0-9]*\"))\n\tfor _, entry := range entries {\n\t\tif err := sys.discoverCPU(entry); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to discover cpu for entry %s: %v\", entry, err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Discover details of the given CPU.\nfunc (sys *system) discoverCPU(path string) error {\n\tcpu := &cpu{path: path, id: getEnumeratedID(path), online: true, sstClos: -1}\n\n\tcpu.isolated = sys.isolated.Has(cpu.id)\n\n\tif online, err := readSysfsEntry(path, \"online\", nil); err == nil {\n\t\tcpu.online = (online != \"\" && online[0] != '0')\n\t}\n\n\tif cpu.online {\n\t\tif _, err := readSysfsEntry(path, \"topology/physical_package_id\", &cpu.pkg); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treadSysfsEntry(path, \"topology/die_id\", &cpu.die)\n\t\tif _, err := readSysfsEntry(path, \"topology/core_id\", &cpu.core); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := readSysfsEntry(path, \"topology/thread_siblings_list\", &cpu.threads, \",\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tsys.offline.Add(cpu.id)\n\t}\n\n\tif _, err := readSysfsEntry(path, \"cpufreq/base_frequency\", &cpu.baseFreq); err != nil {\n\t\tcpu.baseFreq = 0\n\t}\n\tif _, err := readSysfsEntry(path, \"cpufreq/cpuinfo_min_freq\", &cpu.freq.min); err != nil {\n\t\tcpu.freq.min = 0\n\t}\n\tif _, err := readSysfsEntry(path, \"cpufreq/cpuinfo_max_freq\", &cpu.freq.max); err != nil {\n\t\tcpu.freq.max = 0\n\t}\n\tif _, err := readSysfsEntry(path, \"cpufreq/energy_performance_preference\", &cpu.epp); err != nil {\n\t\tcpu.epp = EPPUnknown\n\t}\n\tif node, _ := filepath.Glob(filepath.Join(path, \"node[0-9]*\")); len(node) == 1 {\n\t\tcpu.node = getEnumeratedID(node[0])\n\t} else {\n\t\treturn fmt.Errorf(\"exactly one node per cpu allowed\")\n\t}\n\n\tif sys.threads < 1 {\n\t\tsys.threads = 1\n\t}\n\tif cpu.threads.Size() > sys.threads {\n\t\tsys.threads = cpu.threads.Size()\n\t}\n\n\tsys.cpus[cpu.id] = cpu\n\n\treturn nil\n}\n\n// ID returns the id of this CPU.\nfunc (c *cpu) ID() idset.ID {\n\treturn c.id\n}\n\n// PackageID returns package id of this CPU.\nfunc (c *cpu) PackageID() idset.ID {\n\treturn c.pkg\n}\n\n// DieID returns the die id of this CPU.\nfunc (c *cpu) DieID() idset.ID {\n\treturn c.die\n}\n\n// NodeID returns the node id of this CPU.\nfunc (c *cpu) NodeID() idset.ID {\n\treturn c.node\n}\n\n// CoreID returns the core id of this CPU (lowest CPU id of all thread siblings).\nfunc (c *cpu) CoreID() idset.ID {\n\treturn c.core\n}\n\n// ThreadCPUSet returns the CPUSet for all threads in this core.\nfunc (c *cpu) ThreadCPUSet() cpuset.CPUSet {\n\treturn CPUSetFromIDSet(c.threads)\n}\n\n// BaseFrequency returns the base frequency setting for this CPU.\nfunc (c *cpu) BaseFrequency() uint64 {\n\treturn c.baseFreq\n}\n\n// FrequencyRange returns the frequency range for this CPU.\nfunc (c *cpu) FrequencyRange() CPUFreq {\n\treturn c.freq\n}\n\n// EPP returns the energy performance profile of this CPU.\nfunc (c *cpu) EPP() EPP {\n\treturn c.epp\n}\n\n// Online returns if this CPU is online.\nfunc (c *cpu) Online() bool {\n\treturn c.online\n}\n\n// Isolated returns if this CPU is isolated.\nfunc (c *cpu) Isolated() bool {\n\treturn c.isolated\n}\n\n// SstClos returns the Speed Select Core Power CLOS number assigned to the CPU\n// -1 implies that no SST prioritization is in effect\nfunc (c *cpu) SstClos() int {\n\treturn c.sstClos\n}\n\n// SetFrequencyLimits sets the frequency scaling limits for this CPU.\nfunc (c *cpu) SetFrequencyLimits(min, max uint64) error {\n\tif c.freq.min == 0 {\n\t\treturn nil\n\t}\n\n\tmin /= 1000\n\tmax /= 1000\n\tif min < c.freq.min && min != 0 {\n\t\tmin = c.freq.min\n\t}\n\tif min > c.freq.max {\n\t\tmin = c.freq.max\n\t}\n\tif max < c.freq.min && max != 0 {\n\t\tmax = c.freq.min\n\t}\n\tif max > c.freq.max {\n\t\tmax = c.freq.max\n\t}\n\n\tif _, err := writeSysfsEntry(c.path, \"cpufreq/scaling_min_freq\", min, nil); err != nil {\n\t\treturn err\n\t}\n\tif _, err := writeSysfsEntry(c.path, \"cpufreq/scaling_max_freq\", max, nil); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc readCPUsetFile(base, entry string) (cpuset.CPUSet, error) {\n\tpath := filepath.Join(base, entry)\n\n\tblob, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn cpuset.New(), sysfsError(path, \"failed to read sysfs entry: %v\", err)\n\t}\n\n\treturn cpuset.Parse(strings.Trim(string(blob), \"\\n\"))\n}\n\n// Discover NUMA nodes present in the system.\nfunc (sys *system) discoverNodes() error {\n\tif sys.nodes != nil {\n\t\treturn nil\n\t}\n\n\tsysNodesPath := filepath.Join(sys.path, sysfsNumaNodePath)\n\tsys.nodes = make(map[idset.ID]*node)\n\tentries, _ := filepath.Glob(filepath.Join(sysNodesPath, \"node[0-9]*\"))\n\tfor _, entry := range entries {\n\t\tif err := sys.discoverNode(entry); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to discover node for entry %s: %v\", entry, err)\n\t\t}\n\t}\n\n\tnormalMemNodeIDs, err := readSysfsEntry(sysNodesPath, \"has_normal_memory\", nil)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to discover nodes with normal memory: %v\", err)\n\t}\n\tnormalMemNodes, err := cpuset.Parse(normalMemNodeIDs)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to parse nodes with normal memory (%q): %v\",\n\t\t\tnormalMemNodes, err)\n\t}\n\tmemoryNodeIDs, err := readSysfsEntry(sysNodesPath, \"has_memory\", nil)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to discover nodes with memory: %v\", err)\n\t}\n\tmemoryNodes, err := cpuset.Parse(memoryNodeIDs)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to parse nodes with memory (%q): %v\",\n\t\t\tmemoryNodeIDs, err)\n\t}\n\n\tcpuNodesSlice := []int{}\n\tfor id, node := range sys.nodes {\n\t\tif node.cpus.Size() > 0 {\n\t\t\tcpuNodesSlice = append(cpuNodesSlice, int(id))\n\t\t}\n\t\tif normalMemNodes.Contains(int(id)) {\n\t\t\tnode.normalMem = true\n\t\t}\n\t}\n\tcpuNodes := cpuset.New(cpuNodesSlice...)\n\n\tsys.Logger.Info(\"NUMA nodes with CPUs: %s\", cpuNodes.String())\n\tsys.Logger.Info(\"NUMA nodes with (any) memory: %s\", memoryNodes.String())\n\tsys.Logger.Info(\"NUMA nodes with normal memory: %s\", normalMemNodes.String())\n\n\tdramNodes := memoryNodes.Intersection(cpuNodes)\n\tpmemOrHbmNodes := memoryNodes.Difference(dramNodes)\n\n\tdramNodeIds := IDSetFromCPUSet(dramNodes)\n\tpmemOrHbmNodeIds := IDSetFromCPUSet(pmemOrHbmNodes)\n\n\tinfos := make(map[idset.ID]*MemInfo)\n\tdramAvg := uint64(0)\n\tif len(pmemOrHbmNodeIds) > 0 && len(dramNodeIds) > 0 {\n\t\t// There is special memory present in the system.\n\n\t\t// FIXME assumption: if a node only has memory (and no CPUs), it's PMEM or HBM. Otherwise it's DRAM.\n\t\t// Also, we figure out if the memory is HBM or PMEM based on the amount. If the amount of memory is\n\t\t// smaller than the average amount of DRAM per node, it's HBM, otherwise PMEM.\n\t\tdramTotal := uint64(0)\n\t\tfor _, node := range sys.nodes {\n\t\t\tinfo, err := node.MemoryInfo()\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to get memory info for node %v: %s\", node, err)\n\t\t\t}\n\t\t\tinfos[node.id] = info\n\t\t\tif _, ok := dramNodeIds[node.id]; ok {\n\t\t\t\tdramTotal += info.MemTotal\n\t\t\t}\n\t\t}\n\t\tdramAvg = dramTotal / uint64(len(dramNodeIds))\n\t\tif dramAvg == 0 {\n\t\t\t// FIXME: should be no reason to bail out when memory types are properly determined.\n\t\t\treturn fmt.Errorf(\"no dram in the system, cannot determine special memory types\")\n\t\t}\n\t}\n\n\tfor _, node := range sys.nodes {\n\t\tif _, ok := pmemOrHbmNodeIds[node.id]; ok {\n\t\t\tmem, ok := infos[node.id]\n\t\t\tif !ok {\n\t\t\t\treturn fmt.Errorf(\"not able to determine system special memory types\")\n\t\t\t}\n\t\t\tif mem.MemTotal < dramAvg {\n\t\t\t\tsys.Logger.Info(\"node %d has HBM memory\", node.id)\n\t\t\t\tnode.memoryType = MemoryTypeHBM\n\t\t\t} else {\n\t\t\t\tsys.Logger.Info(\"node %d has PMEM memory\", node.id)\n\t\t\t\tnode.memoryType = MemoryTypePMEM\n\t\t\t}\n\t\t} else if _, ok := dramNodeIds[node.id]; ok {\n\t\t\tsys.Logger.Info(\"node %d has DRAM memory\", node.id)\n\t\t\tnode.memoryType = MemoryTypeDRAM\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"Unknown memory type for node %v (pmem nodes: %s, dram nodes: %s)\", node, pmemOrHbmNodes, dramNodes)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Discover details of the given NUMA node.\nfunc (sys *system) discoverNode(path string) error {\n\tnode := &node{path: path, id: getEnumeratedID(path)}\n\n\tif _, err := readSysfsEntry(path, \"cpulist\", &node.cpus, \",\"); err != nil {\n\t\treturn err\n\t}\n\tif _, err := readSysfsEntry(path, \"distance\", &node.distance); err != nil {\n\t\treturn err\n\t}\n\n\tsys.nodes[node.id] = node\n\n\treturn nil\n}\n\n// ID returns id of this node.\nfunc (n *node) ID() idset.ID {\n\treturn n.id\n}\n\n// PackageID returns the package id for this node.\nfunc (n *node) PackageID() idset.ID {\n\treturn n.pkg\n}\n\n// DieID returns the die id for this node.\nfunc (n *node) DieID() idset.ID {\n\treturn n.die\n}\n\n// CPUSet returns the CPUSet for all cores/threads in this node.\nfunc (n *node) CPUSet() cpuset.CPUSet {\n\treturn CPUSetFromIDSet(n.cpus)\n}\n\n// Distance returns the distance vector for this node.\nfunc (n *node) Distance() []int {\n\treturn n.distance\n}\n\n// DistanceFrom returns the distance of this and a given node.\nfunc (n *node) DistanceFrom(id idset.ID) int {\n\tif int(id) < len(n.distance) {\n\t\treturn n.distance[int(id)]\n\t}\n\n\treturn -1\n}\n\n// MemoryInfo memory info for the node (partial content from the meminfo sysfs entry).\nfunc (n *node) MemoryInfo() (*MemInfo, error) {\n\tmeminfo := filepath.Join(n.path, \"meminfo\")\n\tbuf := &MemInfo{}\n\terr := ParseFileEntries(meminfo,\n\t\tmap[string]interface{}{\n\t\t\t\"MemTotal:\": &buf.MemTotal,\n\t\t\t\"MemFree:\":  &buf.MemFree,\n\t\t},\n\t\tfunc(line string) (string, string, error) {\n\t\t\tfields := strings.Fields(strings.TrimSpace(line))\n\t\t\tif len(fields) < 4 {\n\t\t\t\treturn \"\", \"\", sysfsError(meminfo, \"failed to parse entry: '%s'\", line)\n\t\t\t}\n\t\t\tkey := fields[2]\n\t\t\tval := fields[3]\n\t\t\tif len(fields) == 5 {\n\t\t\t\tval += \" \" + fields[4]\n\t\t\t}\n\t\t\treturn key, val, nil\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t//\n\t// On some HW and kernel combinations we've seen more free than total\n\t// memory being reported. This causes exorbitant usage of memory being\n\t// reported which later can cause failures in policies which trust and\n\t// rely on this information.\n\t//\n\t// Give here a clear(er) error about that. This should also prevent us\n\t// immediately from starting up.\n\t//\n\tif buf.MemFree > buf.MemTotal {\n\t\treturn nil, sysfsError(meminfo, \"System reports more free than total memory. \"+\n\t\t\t\"This can be caused by a kernel bug. Please update your kernel.\")\n\t}\n\n\tbuf.MemUsed = buf.MemTotal - buf.MemFree\n\n\treturn buf, nil\n}\n\n// GetMemoryType returns the memory type for this node.\nfunc (n *node) GetMemoryType() MemoryType {\n\treturn n.memoryType\n}\n\n// HasNormalMemory returns true if the node has memory that belongs to a normal zone.\nfunc (n *node) HasNormalMemory() bool {\n\treturn n.normalMem\n}\n\n// Discover physical packages (CPU sockets) present in the system.\nfunc (sys *system) discoverPackages() error {\n\tif sys.packages != nil {\n\t\treturn nil\n\t}\n\n\tsys.packages = make(map[idset.ID]*cpuPackage)\n\n\tfor _, cpu := range sys.cpus {\n\t\tpkg, found := sys.packages[cpu.pkg]\n\t\tif !found {\n\t\t\tpkg = &cpuPackage{\n\t\t\t\tid:       cpu.pkg,\n\t\t\t\tcpus:     idset.NewIDSet(),\n\t\t\t\tnodes:    idset.NewIDSet(),\n\t\t\t\tdies:     idset.NewIDSet(),\n\t\t\t\tdieCPUs:  make(map[idset.ID]idset.IDSet),\n\t\t\t\tdieNodes: make(map[idset.ID]idset.IDSet),\n\t\t\t}\n\t\t\tsys.packages[cpu.pkg] = pkg\n\t\t}\n\t\tpkg.cpus.Add(cpu.id)\n\t\tpkg.nodes.Add(cpu.node)\n\t\tpkg.dies.Add(cpu.die)\n\n\t\tif dieCPUs, ok := pkg.dieCPUs[cpu.die]; !ok {\n\t\t\tpkg.dieCPUs[cpu.die] = idset.NewIDSet(cpu.id)\n\t\t} else {\n\t\t\tdieCPUs.Add(cpu.id)\n\t\t}\n\t\tif dieNodes, ok := pkg.dieNodes[cpu.die]; !ok {\n\t\t\tpkg.dieNodes[cpu.die] = idset.NewIDSet(cpu.node)\n\t\t} else {\n\t\t\tdieNodes.Add(cpu.node)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (sys *system) discoverSst() error {\n\tif !sst.SstSupported() {\n\t\tsys.Info(\"Speed Select Technology (SST) support not detected\")\n\t\treturn nil\n\t}\n\n\tfor _, pkg := range sys.packages {\n\t\tsstInfo, err := sst.GetPackageInfo(pkg.id)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to get SST info for package %d: %v\", pkg.id, err)\n\t\t}\n\t\tsys.DebugBlock(\"\", \"Speed Select Technology info detected for package %d:\\n%s\", pkg.id, utils.DumpJSON(sstInfo))\n\n\t\tif sstInfo[pkg.id].CPEnabled {\n\t\t\tids := pkg.cpus.SortedMembers()\n\n\t\t\tfor _, id := range ids {\n\t\t\t\tclos, err := sst.GetCPUClosID(id)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"failed to get SST-CP clos id for cpu %d: %v\", id, err)\n\t\t\t\t}\n\n\t\t\t\tsys.cpus[id].sstClos = clos\n\t\t\t}\n\t\t}\n\t\tpkg.sstInfo = sstInfo[pkg.id]\n\t}\n\n\treturn nil\n}\n\n// ID returns the id of this package.\nfunc (p *cpuPackage) ID() idset.ID {\n\treturn p.id\n}\n\n// CPUSet returns the CPUSet for all cores/threads in this package.\nfunc (p *cpuPackage) CPUSet() cpuset.CPUSet {\n\treturn CPUSetFromIDSet(p.cpus)\n}\n\n// DieIDs returns the die ids for this package.\nfunc (p *cpuPackage) DieIDs() []idset.ID {\n\treturn p.dies.SortedMembers()\n}\n\n// NodeIDs returns the NUMA node ids for this package.\nfunc (p *cpuPackage) NodeIDs() []idset.ID {\n\treturn p.nodes.SortedMembers()\n}\n\n// DieNodeIDs returns the set of NUMA nodes in the given die of this package.\nfunc (p *cpuPackage) DieNodeIDs(id idset.ID) []idset.ID {\n\tif dieNodes, ok := p.dieNodes[id]; ok {\n\t\treturn dieNodes.SortedMembers()\n\t}\n\treturn []idset.ID{}\n}\n\n// DieCPUSet returns the set of CPUs in the given die of this package.\nfunc (p *cpuPackage) DieCPUSet(id idset.ID) cpuset.CPUSet {\n\tif dieCPUs, ok := p.dieCPUs[id]; ok {\n\t\treturn CPUSetFromIDSet(dieCPUs)\n\t}\n\treturn cpuset.New()\n}\n\nfunc (p *cpuPackage) SstInfo() *sst.SstPackageInfo {\n\treturn p.sstInfo\n}\n\n// eppStrings initialized this way to better catch changes in the enum\nvar eppStrings = func() [EPPUnknown]string {\n\tvar e [EPPUnknown]string\n\te[EPPPerformance] = \"performance\"\n\te[EPPBalancePerformance] = \"balance_performance\"\n\te[EPPBalancePower] = \"balance_power\"\n\te[EPPPower] = \"power\"\n\treturn e\n}()\n\nvar eppValues = func() map[string]EPP {\n\tm := make(map[string]EPP, len(eppStrings))\n\tfor i, v := range eppStrings {\n\t\tm[v] = EPP(i)\n\t}\n\treturn m\n}()\n\n// String returns EPP value as string\nfunc (e EPP) String() string {\n\tif int(e) < len(eppStrings) {\n\t\treturn eppStrings[e]\n\t}\n\treturn \"\"\n}\n\n// EPPFromString converts string to EPP value\nfunc EPPFromString(s string) EPP {\n\tif v, ok := eppValues[s]; ok {\n\t\treturn v\n\t}\n\treturn EPPUnknown\n}\n"
  },
  {
    "path": "pkg/sysfs/utils.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage sysfs\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/utils/cpuset\"\n\tidset \"github.com/intel/goresctrl/pkg/utils\"\n)\n\n// Get the trailing enumeration part of a name.\nfunc getEnumeratedID(name string) idset.ID {\n\tid := 0\n\tbase := 1\n\tfor idx := len(name) - 1; idx > 0; idx-- {\n\t\td := name[idx]\n\n\t\tif '0' <= d && d <= '9' {\n\t\t\tid += base * (int(d) - '0')\n\t\t\tbase *= 10\n\t\t} else {\n\t\t\tif base > 1 {\n\t\t\t\treturn idset.ID(id)\n\t\t\t}\n\n\t\t\treturn idset.ID(-1)\n\t\t}\n\t}\n\n\treturn idset.ID(-1)\n}\n\n// Read content of a sysfs entry and convert it according to the type of a given pointer.\nfunc readSysfsEntry(base, entry string, ptr interface{}, args ...interface{}) (string, error) {\n\tvar buf string\n\n\tpath := filepath.Join(base, entry)\n\n\tblob, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn \"\", sysfsError(path, \"failed to read sysfs entry: %v\", err)\n\t}\n\tbuf = strings.Trim(string(blob), \"\\n\")\n\n\tif ptr == interface{}(nil) {\n\t\treturn buf, nil\n\t}\n\n\tswitch ptr.(type) {\n\tcase *string, *int, *uint, *int8, *uint8, *int16, *uint16, *int32, *uint32, *int64, *uint64:\n\t\terr := parseValue(buf, ptr)\n\t\tif err != nil {\n\t\t\treturn \"\", sysfsError(path, \"%v\", err)\n\t\t}\n\t\treturn buf, nil\n\n\tcase *idset.IDSet, *[]int, *[]uint, *[]int8, *[]uint8, *[]int16, *[]uint16, *[]int32, *[]uint32, *[]int64, *[]uint64:\n\t\tsep, err := getSeparator(\" \", args)\n\t\tif err != nil {\n\t\t\treturn \"\", sysfsError(path, \"%v\", err)\n\t\t}\n\t\terr = parseValueList(buf, sep, ptr)\n\t\tif err != nil {\n\t\t\treturn \"\", sysfsError(path, \"%v\", err)\n\t\t}\n\t\treturn buf, nil\n\tcase *EPP:\n\t\t*ptr.(*EPP) = EPPFromString(buf)\n\t\treturn buf, nil\n\t}\n\n\treturn \"\", sysfsError(path, \"unsupported sysfs entry type %T\", ptr)\n}\n\n// Write a value to a sysfs entry. An optional item separator can be specified for slice values.\nfunc writeSysfsEntry(base, entry string, val, oldp interface{}, args ...interface{}) (string, error) {\n\tvar buf, old string\n\tvar err error\n\n\tif oldp != nil {\n\t\tif old, err = readSysfsEntry(base, entry, oldp, args...); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t}\n\n\tpath := filepath.Join(base, entry)\n\n\tswitch val.(type) {\n\tcase string, int, uint, int8, uint8, int16, uint16, int32, uint32, int64, uint64:\n\t\tbuf, err = formatValue(val)\n\t\tif err != nil {\n\t\t\treturn \"\", sysfsError(path, \"%v\", err)\n\t\t}\n\n\tcase idset.IDSet, []int, []uint, []int8, []uint8, []int16, []uint16, []int32, []uint32, []int64, []uint64:\n\t\tsep, err := getSeparator(\" \", args)\n\t\tif err != nil {\n\t\t\treturn \"\", sysfsError(path, \"%v\", err)\n\t\t}\n\t\tbuf, err = formatValueList(sep, val)\n\t\tif err != nil {\n\t\t\treturn \"\", sysfsError(path, \"%v\", err)\n\t\t}\n\n\tdefault:\n\t\treturn \"\", sysfsError(path, \"unsupported sysfs entry type %T\", val)\n\t}\n\n\tf, err := os.OpenFile(path, os.O_WRONLY, 0)\n\tif err != nil {\n\t\treturn \"\", sysfsError(path, \"cannot open: %v\", err)\n\t}\n\tdefer f.Close()\n\n\tif _, err = f.Write([]byte(buf + \"\\n\")); err != nil {\n\t\treturn \"\", sysfsError(path, \"cannot write: %v\", err)\n\t}\n\n\treturn old, nil\n}\n\n// Determine list separator string, given an optional separator variadic argument.\nfunc getSeparator(defaultVal string, args []interface{}) (string, error) {\n\tswitch len(args) {\n\tcase 0:\n\t\treturn defaultVal, nil\n\tcase 1:\n\t\treturn args[0].(string), nil\n\t}\n\n\treturn \"\", fmt.Errorf(\"invalid separator (%v), 1 expected, %d given\", args, len(args))\n}\n\n// Parse a value from a string.\nfunc parseValue(str string, value interface{}) error {\n\tswitch value.(type) {\n\tcase *string:\n\t\t*value.(*string) = str\n\n\tcase *int, *int8, *int16, *int32, *int64:\n\t\tv, err := strconv.ParseInt(str, 0, 0)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"invalid entry '%s': %v\", str, err)\n\t\t}\n\n\t\tswitch value.(type) {\n\t\tcase *int:\n\t\t\t*value.(*int) = int(v)\n\t\tcase *int8:\n\t\t\t*value.(*int8) = int8(v)\n\t\tcase *int16:\n\t\t\t*value.(*int16) = int16(v)\n\t\tcase *int32:\n\t\t\t*value.(*int32) = int32(v)\n\t\tcase int64:\n\t\t\t*value.(*int64) = v\n\t\t}\n\n\tcase *uint, *uint8, *uint16, *uint32, *uint64:\n\t\tv, err := strconv.ParseUint(str, 0, 0)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"invalid entry: '%s': %v\", str, err)\n\t\t}\n\n\t\tswitch value.(type) {\n\t\tcase *uint:\n\t\t\t*value.(*uint) = uint(v)\n\t\tcase *uint8:\n\t\t\t*value.(*uint8) = uint8(v)\n\t\tcase *uint16:\n\t\t\t*value.(*uint16) = uint16(v)\n\t\tcase *uint32:\n\t\t\t*value.(*uint32) = uint32(v)\n\t\tcase *uint64:\n\t\t\t*value.(*uint64) = v\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Parse a list of values from a string into a slice.\nfunc parseValueList(str, sep string, valuep interface{}) error {\n\tvar value interface{}\n\n\tswitch valuep.(type) {\n\tcase *idset.IDSet:\n\t\tvalue = idset.NewIDSet()\n\tcase *[]int:\n\t\tvalue = []int{}\n\tcase *[]uint:\n\t\tvalue = []uint{}\n\tcase *[]int8:\n\t\tvalue = []int8{}\n\tcase *[]uint8:\n\t\tvalue = []uint8{}\n\tcase *[]int16:\n\t\tvalue = []int16{}\n\tcase *[]uint16:\n\t\tvalue = []uint16{}\n\tcase *[]int32:\n\t\tvalue = []int32{}\n\tcase *[]uint32:\n\t\tvalue = []uint32{}\n\tcase *[]int64:\n\t\tvalue = []int64{}\n\tcase *[]uint64:\n\t\tvalue = []uint64{}\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid slice value type: %T\", valuep)\n\t}\n\n\tfor _, s := range strings.Split(str, sep) {\n\t\tif s == \"\" {\n\t\t\tbreak\n\t\t}\n\t\tswitch value.(type) {\n\t\tcase idset.IDSet:\n\t\t\tif rng := strings.Split(s, \"-\"); len(rng) == 1 {\n\t\t\t\tid, err := strconv.Atoi(s)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"invalid entry '%s': %v\", s, err)\n\t\t\t\t}\n\t\t\t\tvalue.(idset.IDSet).Add(idset.ID(id))\n\t\t\t} else {\n\t\t\t\tbeg, err := strconv.Atoi(rng[0])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"invalid entry '%s': %v\", s, err)\n\t\t\t\t}\n\t\t\t\tend, err := strconv.Atoi(rng[1])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"invalid entry '%s': %v\", s, err)\n\t\t\t\t}\n\t\t\t\tfor id := beg; id <= end; id++ {\n\t\t\t\t\tvalue.(idset.IDSet).Add(idset.ID(id))\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase []int, []int8, []int16, []int32, []int64:\n\t\t\tv, err := strconv.ParseInt(s, 0, 0)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"invalid entry '%s': %v\", s, err)\n\t\t\t}\n\t\t\tswitch value.(type) {\n\t\t\tcase []int:\n\t\t\t\tvalue = append(value.([]int), int(v))\n\t\t\tcase []int8:\n\t\t\t\tvalue = append(value.([]int8), int8(v))\n\t\t\tcase []int16:\n\t\t\t\tvalue = append(value.([]int16), int16(v))\n\t\t\tcase []int32:\n\t\t\t\tvalue = append(value.([]int32), int32(v))\n\t\t\tcase []int64:\n\t\t\t\tvalue = append(value.([]int64), v)\n\t\t\t}\n\n\t\tcase []uint, []uint8, []uint16, []uint32, []uint64:\n\t\t\tv, err := strconv.ParseUint(s, 0, 0)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"invalid entry '%s': %v\", s, err)\n\t\t\t}\n\t\t\tswitch value.(type) {\n\t\t\tcase []uint:\n\t\t\t\tvalue = append(value.([]uint), uint(v))\n\t\t\tcase []uint8:\n\t\t\t\tvalue = append(value.([]uint8), uint8(v))\n\t\t\tcase []uint16:\n\t\t\t\tvalue = append(value.([]uint16), uint16(v))\n\t\t\tcase []uint32:\n\t\t\t\tvalue = append(value.([]uint32), uint32(v))\n\t\t\tcase []uint64:\n\t\t\t\tvalue = append(value.([]uint64), v)\n\t\t\t}\n\t\t}\n\t}\n\n\tswitch valuep.(type) {\n\tcase *idset.IDSet:\n\t\t*valuep.(*idset.IDSet) = value.(idset.IDSet)\n\tcase *[]int:\n\t\t*valuep.(*[]int) = value.([]int)\n\tcase *[]uint:\n\t\t*valuep.(*[]uint) = value.([]uint)\n\tcase *[]int8:\n\t\t*valuep.(*[]int8) = value.([]int8)\n\tcase *[]uint8:\n\t\t*valuep.(*[]uint8) = value.([]uint8)\n\tcase *[]int16:\n\t\t*valuep.(*[]int16) = value.([]int16)\n\tcase *[]uint16:\n\t\t*valuep.(*[]uint16) = value.([]uint16)\n\tcase *[]int32:\n\t\t*valuep.(*[]int32) = value.([]int32)\n\tcase *[]uint32:\n\t\t*valuep.(*[]uint32) = value.([]uint32)\n\tcase *[]int64:\n\t\t*valuep.(*[]int64) = value.([]int64)\n\tcase *[]uint64:\n\t\t*valuep.(*[]uint64) = value.([]uint64)\n\t}\n\n\treturn nil\n}\n\n// Format a value into a string.\nfunc formatValue(value interface{}) (string, error) {\n\tswitch value.(type) {\n\tcase string:\n\t\treturn value.(string), nil\n\tcase int, uint, int8, uint8, int16, uint16, int32, uint32, int64, uint64:\n\t\treturn fmt.Sprintf(\"%d\", value), nil\n\tdefault:\n\t\treturn \"\", fmt.Errorf(\"invalid value type %T\", value)\n\t}\n}\n\n// Format a list of values from a slice into a string.\nfunc formatValueList(sep string, value interface{}) (string, error) {\n\tvar v []interface{}\n\n\tswitch value.(type) {\n\tcase idset.IDSet:\n\t\treturn value.(idset.IDSet).StringWithSeparator(sep), nil\n\tcase []int, []uint, []int8, []uint8, []int16, []uint16, []int32, []uint32, []int64, []uint64:\n\t\tv = value.([]interface{})\n\tdefault:\n\t\treturn \"\", fmt.Errorf(\"invalid value type %T\", value)\n\t}\n\n\tstr := \"\"\n\tt := \"\"\n\tfor idx := range v {\n\t\tstr = str + t + fmt.Sprintf(\"%d\", v[idx])\n\t\tt = sep\n\t}\n\n\treturn \"\", nil\n}\n\n// IDSetFromCPUSet returns an id set corresponding to a cpuset.CPUSet.\nfunc IDSetFromCPUSet(cset cpuset.CPUSet) idset.IDSet {\n\treturn idset.NewIDSetFromIntSlice(cset.List()...)\n}\n\n// CPUSetFromIDSet returns a cpuset.CPUSet corresponding to an id set.\nfunc CPUSetFromIDSet(s idset.IDSet) cpuset.CPUSet {\n\tcpus := []int{}\n\tfor id := range s {\n\t\tcpus = append(cpus, int(id))\n\t}\n\treturn cpuset.New(cpus...)\n}\n"
  },
  {
    "path": "pkg/testutils/verify.go",
    "content": "package testutils\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n)\n\n// VerifyDeepEqual checks that two values (including structures) are equal, or else it fails the test.\nfunc VerifyDeepEqual(t *testing.T, valueName string, expectedValue interface{}, seenValue interface{}) bool {\n\tif reflect.DeepEqual(expectedValue, seenValue) {\n\t\treturn true\n\t}\n\tt.Errorf(\"expected %s value %+v, got %+v\", valueName, expectedValue, seenValue)\n\treturn false\n}\n\n// VerifyError checks a (multi)error has expected properties, or else it fails the test.\nfunc VerifyError(t *testing.T, err error, expectedCount int, expectedSubstrings []string) bool {\n\tif expectedCount > 0 {\n\t\tif err == nil {\n\t\t\tt.Errorf(\"error expected, got nil\")\n\t\t\treturn false\n\t\t}\n\t\tif merr, ok := err.(interface{ Unwrap() []error }); !ok {\n\t\t\tt.Errorf(\"expected %d errors, but got %#v instead of multierror\", expectedCount, err)\n\t\t\treturn false\n\t\t} else if errs := merr.Unwrap(); len(errs) != expectedCount {\n\t\t\tt.Errorf(\"expected %d errors, but got %d: %v\", expectedCount, len(errs), merr)\n\t\t\treturn false\n\t\t}\n\t} else if expectedCount == 0 {\n\t\tif err != nil {\n\t\t\tt.Errorf(\"expected 0 errors, but got %v\", err)\n\t\t\treturn false\n\t\t}\n\t}\n\tfor _, substring := range expectedSubstrings {\n\t\tif !strings.Contains(err.Error(), substring) {\n\t\t\tt.Errorf(\"expected error with substring %#v, got \\\"%v\\\"\", substring, err)\n\t\t}\n\t}\n\treturn true\n}\n"
  },
  {
    "path": "pkg/topology/go.mod",
    "content": "module github.com/intel/cri-resource-manager/pkg/topology\n\ngo 1.22.0\n\nrequire (\n\tgithub.com/pkg/errors v0.9.1\n\tgolang.org/x/sys v0.18.0\n)\n"
  },
  {
    "path": "pkg/topology/test-cleanup.sh",
    "content": "rm -fr testdata\n"
  },
  {
    "path": "pkg/topology/test-setup.sh",
    "content": "tar -xvzf test-data.tar.gz\n"
  },
  {
    "path": "pkg/topology/topology.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topology\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/sys/unix\"\n)\n\n// to mock in tests\nvar (\n\tsysRoot = \"\"\n)\n\nconst (\n\t// ProviderKubelet is a constant to distinguish that topology hint comes\n\t// from parameters passed to CRI create/update requests from Kubelet\n\tProviderKubelet = \"kubelet\"\n)\n\n// Hint represents various hints that can be detected from sysfs for the device\ntype Hint struct {\n\tProvider string\n\tCPUs     string\n\tNUMAs    string\n\tSockets  string\n}\n\n// Hints represents set of hints collected from multiple providers\ntype Hints map[string]Hint\n\n// SetSysRoot sets the sysfs root directory to use.\nfunc SetSysRoot(root string) {\n\tsysRoot = root\n}\n\nfunc getDevicesFromVirtual(realDevPath string) (devs []string, err error) {\n\tif !filepath.HasPrefix(realDevPath, \"/sys/devices/virtual\") {\n\t\treturn nil, fmt.Errorf(\"%s is not a virtual device\", realDevPath)\n\t}\n\n\trelPath, _ := filepath.Rel(\"/sys/devices/virtual\", realDevPath)\n\n\tdir, file := filepath.Split(relPath)\n\tswitch dir {\n\tcase \"vfio/\":\n\t\tiommuGroup := filepath.Join(sysRoot, \"/sys/kernel/iommu_groups\", file, \"devices\")\n\t\tfiles, err := os.ReadDir(iommuGroup)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to read IOMMU group %s\", iommuGroup)\n\t\t}\n\t\tfor _, file := range files {\n\t\t\trealDev, err := filepath.EvalSymlinks(filepath.Join(iommuGroup, file.Name()))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrapf(err, \"failed to get real path for %s\", file.Name())\n\t\t\t}\n\t\t\tdevs = append(devs, realDev)\n\t\t}\n\t\treturn devs, nil\n\tdefault:\n\t\treturn nil, nil\n\t}\n}\n\nfunc getTopologyHint(sysFSPath string) (*Hint, error) {\n\thint := Hint{Provider: sysFSPath}\n\tfileMap := map[string]*string{\n\t\t\"local_cpulist\": &hint.CPUs,\n\t\t\"numa_node\":     &hint.NUMAs,\n\t}\n\tif err := readFilesInDirectory(fileMap, sysFSPath); err != nil {\n\t\treturn nil, err\n\t}\n\t// Workarounds for broken information provided by kernel\n\tif hint.NUMAs == \"-1\" {\n\t\t// non-NUMA aware device or system, ignore it\n\t\thint.NUMAs = \"\"\n\t}\n\tif hint.NUMAs != \"\" && hint.CPUs == \"\" {\n\t\t// broken topology hint. BIOS reports socket id as NUMA node\n\t\t// First, try to get hints from parent device or bus.\n\t\tparentHints, er := NewTopologyHints(filepath.Dir(sysFSPath))\n\t\tif er == nil {\n\t\t\tcpulist := map[string]bool{}\n\t\t\tnumalist := map[string]bool{}\n\t\t\tfor _, h := range parentHints {\n\t\t\t\tif h.CPUs != \"\" {\n\t\t\t\t\tcpulist[h.CPUs] = true\n\t\t\t\t}\n\t\t\t\tif h.NUMAs != \"\" {\n\t\t\t\t\tnumalist[h.NUMAs] = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif cpus := strings.Join(mapKeys(cpulist), \",\"); cpus != \"\" {\n\t\t\t\thint.CPUs = cpus\n\t\t\t}\n\t\t\tif numas := strings.Join(mapKeys(numalist), \",\"); numas != \"\" {\n\t\t\t\thint.NUMAs = numas\n\t\t\t}\n\t\t}\n\t\t// if after parent hints we still don't have CPUs hints, use numa hint as sockets.\n\t\tif hint.CPUs == \"\" && hint.NUMAs != \"\" {\n\t\t\thint.Sockets = hint.NUMAs\n\t\t\thint.NUMAs = \"\"\n\t\t}\n\t}\n\treturn &hint, nil\n}\n\n// NewTopologyHints return array of hints for the device and its slaves (e.g. RAID).\nfunc NewTopologyHints(devPath string) (hints Hints, err error) {\n\thints = make(Hints)\n\trealDevPath, err := filepath.EvalSymlinks(devPath)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"failed get realpath for %s\", devPath)\n\t}\n\tfor p := realDevPath; strings.HasPrefix(p, sysRoot+\"/sys/devices/\"); p = filepath.Dir(p) {\n\t\thint, err := getTopologyHint(p)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif hint.CPUs != \"\" || hint.NUMAs != \"\" || hint.Sockets != \"\" {\n\t\t\thints[hint.Provider] = *hint\n\t\t\tbreak\n\t\t}\n\t}\n\tfromVirtual, _ := getDevicesFromVirtual(realDevPath)\n\tslaves, _ := filepath.Glob(filepath.Join(realDevPath, \"slaves/*\"))\n\tfor _, device := range append(slaves, fromVirtual...) {\n\t\tdeviceHints, er := NewTopologyHints(device)\n\t\tif er != nil {\n\t\t\treturn nil, er\n\t\t}\n\t\thints = MergeTopologyHints(hints, deviceHints)\n\t}\n\treturn\n}\n\n// MergeTopologyHints combines org and hints.\nfunc MergeTopologyHints(org, hints Hints) (res Hints) {\n\tif org != nil {\n\t\tres = org\n\t} else {\n\t\tres = make(Hints)\n\t}\n\tfor k, v := range hints {\n\t\tif _, ok := res[k]; ok {\n\t\t\tcontinue\n\t\t}\n\t\tres[k] = v\n\t}\n\treturn\n}\n\n// String returns the hints as a string.\nfunc (h *Hint) String() string {\n\tcpus, nodes, sockets, sep := \"\", \"\", \"\", \"\"\n\n\tif h.CPUs != \"\" {\n\t\tcpus = \"CPUs:\" + h.CPUs\n\t\tsep = \", \"\n\t}\n\tif h.NUMAs != \"\" {\n\t\tnodes = sep + \"NUMAs:\" + h.NUMAs\n\t\tsep = \", \"\n\t}\n\tif h.Sockets != \"\" {\n\t\tsockets = sep + \"sockets:\" + h.Sockets\n\t}\n\n\treturn \"<hints \" + cpus + nodes + sockets + \" (from \" + h.Provider + \")>\"\n}\n\n// FindSysFsDevice for given argument returns physical device where it is linked to.\n// For device nodes it will return path for device itself. For regular files or directories\n// this function returns physical device where this inode resides (storage device).\n// If result device is a virtual one (e.g. tmpfs), error will be returned.\n// For non-existing path, no error returned and path is empty.\nfunc FindSysFsDevice(dev string) (string, error) {\n\tfi, err := os.Stat(dev)\n\tif err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn \"\", nil\n\t\t}\n\t\treturn \"\", errors.Wrapf(err, \"unable to get stat for %s\", dev)\n\t}\n\n\tdevType := \"block\"\n\trdev := fi.Sys().(*syscall.Stat_t).Dev\n\tif mode := fi.Mode(); mode&os.ModeDevice != 0 {\n\t\trdev = fi.Sys().(*syscall.Stat_t).Rdev\n\t\tif mode&os.ModeCharDevice != 0 {\n\t\t\tdevType = \"char\"\n\t\t}\n\t}\n\n\tmajor := unix.Major(rdev)\n\tminor := unix.Minor(rdev)\n\tif major == 0 {\n\t\treturn \"\", errors.Errorf(\"%s is a virtual device node\", dev)\n\t}\n\tdevPath := fmt.Sprintf(\"/sys/dev/%s/%d:%d\", devType, major, minor)\n\trealDevPath, err := filepath.EvalSymlinks(devPath)\n\tif err != nil {\n\t\treturn \"\", errors.Wrapf(err, \"failed get realpath for %s\", devPath)\n\t}\n\treturn realDevPath, nil\n}\n\n// readFilesInDirectory small helper to fill struct with content from sysfs entry\nfunc readFilesInDirectory(fileMap map[string]*string, dir string) error {\n\tfor k, v := range fileMap {\n\t\tb, err := os.ReadFile(filepath.Join(dir, k))\n\t\tif err != nil {\n\t\t\tif os.IsNotExist(err) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn errors.Wrapf(err, \"%s: unable to read file %q\", dir, k)\n\t\t}\n\t\t*v = strings.TrimSpace(string(b))\n\t}\n\treturn nil\n}\n\n// mapKeys is a small helper that returns slice of keys for a given map\nfunc mapKeys(m map[string]bool) []string {\n\tret := make([]string, len(m))\n\ti := 0\n\tfor k := range m {\n\t\tret[i] = k\n\t\ti++\n\t}\n\treturn ret\n}\n"
  },
  {
    "path": "pkg/topology/topology_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage topology\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testing\"\n)\n\nfunc setupTestEnv(t *testing.T) func() {\n\tpwd, err := os.Getwd()\n\tif err != nil {\n\t\tt.Fatal(\"unable to get current directory\")\n\t}\n\tif path, err := filepath.EvalSymlinks(pwd); err == nil {\n\t\tpwd = path\n\t}\n\tSetSysRoot(pwd + \"/testdata\")\n\tteardown := func() {\n\t\tSetSysRoot(\"\")\n\t}\n\treturn teardown\n}\n\nfunc TestMapKeys(t *testing.T) {\n\tcases := []struct {\n\t\tname   string\n\t\tinput  map[string]bool\n\t\toutput []string\n\t}{\n\t\t{\n\t\t\tname:   \"empty\",\n\t\t\tinput:  map[string]bool{},\n\t\t\toutput: []string{},\n\t\t},\n\t\t{\n\t\t\tname:   \"one\",\n\t\t\tinput:  map[string]bool{\"a\": false},\n\t\t\toutput: []string{\"a\"},\n\t\t},\n\t\t{\n\t\t\tname:   \"multiple\",\n\t\t\tinput:  map[string]bool{\"a\": false, \"b\": true, \"c\": false},\n\t\t\toutput: []string{\"a\", \"b\", \"c\"},\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\ttest := tc\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\toutput := mapKeys(test.input)\n\t\t\tsort.Strings(output)\n\t\t\tif !reflect.DeepEqual(output, test.output) {\n\t\t\t\tt.Fatalf(\"expected output: %+v got: %+v\", test.output, output)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestFindSysFsDevice(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skip(\"skipping test in short mode.\")\n\t}\n\tteardown := setupTestEnv(t)\n\tdefer teardown()\n\tcases := []struct {\n\t\tname        string\n\t\tinput       string\n\t\toutput      string\n\t\texpectedErr bool\n\t}{\n\t\t{\n\t\t\tname:        \"empty\",\n\t\t\tinput:       \"\",\n\t\t\toutput:      \"\",\n\t\t\texpectedErr: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"null\",\n\t\t\tinput:       \"/dev/null\",\n\t\t\toutput:      \"/sys/devices/virtual/mem/null\",\n\t\t\texpectedErr: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"proc\",\n\t\t\tinput:       \"/proc/self\",\n\t\t\toutput:      \"\",\n\t\t\texpectedErr: true,\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\ttest := tc\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\toutput, err := FindSysFsDevice(test.input)\n\t\t\tswitch {\n\t\t\tcase err != nil && !test.expectedErr:\n\t\t\t\tt.Fatalf(\"unexpected error returned: %+v\", err)\n\t\t\tcase err == nil && test.expectedErr:\n\t\t\t\tt.Fatalf(\"unexpected success: %+v\", output)\n\t\t\tcase output != test.output:\n\t\t\t\tt.Fatalf(\"expected: %q got: %q\", test.output, output)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReadFilesInDirectory(t *testing.T) {\n\tvar file, empty string\n\tfname := \"test-a\"\n\tcontent := []byte(\" something\\n\")\n\texpectedContent := \"something\"\n\n\tfileMap := map[string]*string{\n\t\tfname:          &file,\n\t\t\"non_existing\": &empty,\n\t}\n\n\tdir, err := os.MkdirTemp(\"\", \"readFilesInDirectory\")\n\tif err != nil {\n\t\tt.Fatalf(\"unable to create test directory: %+v\", err)\n\t}\n\tdefer os.RemoveAll(dir)\n\tos.WriteFile(filepath.Join(dir, fname), content, 0644)\n\n\tif err = readFilesInDirectory(fileMap, dir); err != nil {\n\t\tt.Fatalf(\"unexpected failure: %v\", err)\n\t}\n\tif empty != \"\" {\n\t\tt.Fatalf(\"unexpected content: %q\", empty)\n\t}\n\tif file != expectedContent {\n\t\tt.Fatalf(\"unexpected content: %q expected: %q\", file, expectedContent)\n\t}\n}\n\nfunc TestGetDevicesFromVirtual(t *testing.T) {\n\tteardown := setupTestEnv(t)\n\tdefer teardown()\n\n\tcases := []struct {\n\t\tname        string\n\t\tinput       string\n\t\toutput      []string\n\t\texpectedErr bool\n\t}{\n\t\t{\n\t\t\tname:        \"vfio\",\n\t\t\tinput:       \"/sys/devices/virtual/vfio/42\",\n\t\t\toutput:      []string{sysRoot + \"/sys/devices/pci0000:00/0000:00:02.0\"},\n\t\t\texpectedErr: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"misc\",\n\t\t\tinput:       \"/sys/devices/virtual/misc/vfio\",\n\t\t\toutput:      nil,\n\t\t\texpectedErr: false,\n\t\t},\n\t\t{\n\t\t\tname:        \"missing-iommu-group\",\n\t\t\tinput:       \"/sys/devices/virtual/vfio/84\",\n\t\t\toutput:      nil,\n\t\t\texpectedErr: true,\n\t\t},\n\t\t{\n\t\t\tname:        \"non-virtual\",\n\t\t\tinput:       \"/sys/devices/pci0000:00/0000:00:02.0\",\n\t\t\toutput:      nil,\n\t\t\texpectedErr: true,\n\t\t},\n\t}\n\n\tfor _, tc := range cases {\n\t\ttest := tc\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\toutput, err := getDevicesFromVirtual(test.input)\n\t\t\tswitch {\n\t\t\tcase err != nil && !test.expectedErr:\n\t\t\t\tt.Fatalf(\"unexpected error returned: %+v\", err)\n\t\t\tcase err == nil && test.expectedErr:\n\t\t\t\tt.Fatalf(\"unexpected success: %+v\", output)\n\t\t\tcase len(output) != len(test.output):\n\t\t\t\tt.Fatalf(\"expected: %q got: %q\", len(test.output), len(output))\n\t\t\t}\n\t\t\tfor i, p := range test.output {\n\t\t\t\tif test.output[i] != p {\n\t\t\t\t\tt.Fatalf(\"expected: %q got: %q\", test.output[i], p)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMergeTopologyHints(t *testing.T) {\n\tcases := []struct {\n\t\tname           string\n\t\tinputA         Hints\n\t\tinputB         Hints\n\t\texpectedOutput Hints\n\t\texpectedErr    bool\n\t}{\n\t\t{\n\t\t\tname:           \"empty\",\n\t\t\tinputA:         nil,\n\t\t\tinputB:         nil,\n\t\t\texpectedOutput: Hints{},\n\t\t},\n\t\t{\n\t\t\tname:           \"one,nil\",\n\t\t\tinputA:         Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t\tinputB:         nil,\n\t\t\texpectedOutput: Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t},\n\t\t{\n\t\t\tname:           \"nil, one\",\n\t\t\tinputA:         nil,\n\t\t\tinputB:         Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t\texpectedOutput: Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t},\n\t\t{\n\t\t\tname:           \"duplicate\",\n\t\t\tinputA:         Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t\tinputB:         Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t\texpectedOutput: Hints{\"test\": Hint{Provider: \"test\", CPUs: \"0\"}},\n\t\t},\n\t\t{\n\t\t\tname:   \"two\",\n\t\t\tinputA: Hints{\"test1\": Hint{Provider: \"test1\", CPUs: \"0\"}},\n\t\t\tinputB: Hints{\"test2\": Hint{Provider: \"test2\", CPUs: \"1\"}},\n\t\t\texpectedOutput: Hints{\n\t\t\t\t\"test1\": Hint{Provider: \"test1\", CPUs: \"0\"},\n\t\t\t\t\"test2\": Hint{Provider: \"test2\", CPUs: \"1\"},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tc := range cases {\n\t\ttest := tc\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\toutput := MergeTopologyHints(test.inputA, test.inputB)\n\t\t\tif !reflect.DeepEqual(output, test.expectedOutput) {\n\t\t\t\tt.Fatalf(\"expected output: %+v got: %+v\", test.expectedOutput, output)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewTopologyHints(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skip(\"skipping test in short mode.\")\n\t}\n\tteardown := setupTestEnv(t)\n\tdefer teardown()\n\tcases := []struct {\n\t\tname        string\n\t\tinput       string\n\t\toutput      Hints\n\t\texpectedErr bool\n\t}{\n\t\t{\n\t\t\tname:        \"empty\",\n\t\t\tinput:       \"non-existing\",\n\t\t\toutput:      nil,\n\t\t\texpectedErr: true,\n\t\t},\n\t\t{\n\t\t\tname:  \"pci card1\",\n\t\t\tinput: sysRoot + \"/sys/devices/pci0000:00/0000:00:02.0/drm/card1\",\n\t\t\toutput: Hints{\n\t\t\t\tsysRoot + \"/sys/devices/pci0000:00/0000:00:02.0\": Hint{\n\t\t\t\t\tProvider: sysRoot + \"/sys/devices/pci0000:00/0000:00:02.0\",\n\t\t\t\t\tCPUs:     \"0-7\",\n\t\t\t\t\tNUMAs:    \"\",\n\t\t\t\t\tSockets:  \"\"},\n\t\t\t},\n\t\t\texpectedErr: false,\n\t\t},\n\t}\n\tfor _, test := range cases {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\toutput, err := NewTopologyHints(test.input)\n\t\t\tswitch {\n\t\t\tcase err != nil && !test.expectedErr:\n\t\t\t\tt.Fatalf(\"unexpected error returned: %+v\", err)\n\t\t\tcase err == nil && test.expectedErr:\n\t\t\t\tt.Fatalf(\"unexpected success: %+v\", output)\n\t\t\tcase !reflect.DeepEqual(output, test.output):\n\t\t\t\tt.Fatalf(\"expected: %q got: %q\", test.output, output)\n\t\t\t}\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/utils/cpuset/cpuset.go",
    "content": "// Copyright The NRI Plugins Authors. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpuset\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"k8s.io/utils/cpuset\"\n)\n\n// CPUSet is an alias for k8s.io/utils/cpuset.CPUSet.\ntype CPUSet = cpuset.CPUSet\n\nvar (\n\t// New is an alias for cpuset.New.\n\tNew = cpuset.New\n\t// Parse is an alias for cpuset.Parse.\n\tParse = cpuset.Parse\n)\n\n// MustParse panics if parsing the given cpuset string fails.\nfunc MustParse(s string) cpuset.CPUSet {\n\tcset, err := cpuset.Parse(s)\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"failed to parse CPUSet %s: %w\", s, err))\n\t}\n\treturn cset\n}\n\n// ShortCPUSet prints the cpuset as a string, trying to further shorten compared to .String().\nfunc ShortCPUSet(cset cpuset.CPUSet) string {\n\tstr, sep := \"\", \"\"\n\n\tbeg, end, step := -1, -1, -1\n\tfor _, cpu := range strings.Split(cset.String(), \",\") {\n\t\tif strings.Contains(cpu, \"-\") {\n\t\t\tstr += sep + cpu\n\t\t\tsep = \",\"\n\t\t\tcontinue\n\t\t}\n\t\ti, err := strconv.ParseInt(cpu, 10, 0)\n\t\tif err != nil {\n\t\t\treturn cset.String()\n\t\t}\n\t\tid := int(i)\n\t\tif beg < 0 {\n\t\t\tbeg, end = id, id\n\t\t\tcontinue\n\t\t}\n\t\tif step < 0 {\n\t\t\tend = id\n\t\t\tstep = end - beg\n\t\t\tcontinue\n\t\t}\n\t\tif id-end == step {\n\t\t\tend = id\n\t\t\tcontinue\n\t\t}\n\t\tstr += sep + mkRange(beg, end, step)\n\t\tsep = \",\"\n\t\tbeg, end = id, id\n\t\tstep = -1\n\t}\n\n\tif beg >= 0 {\n\t\tstr += sep + mkRange(beg, end, step)\n\t}\n\n\treturn str\n}\n\nfunc mkRange(beg, end, step int) string {\n\tif beg < 0 {\n\t\treturn \"\"\n\t}\n\tif beg == end {\n\t\treturn strconv.FormatInt(int64(beg), 10)\n\t}\n\n\tb, e := strconv.FormatInt(int64(beg), 10), strconv.FormatInt(int64(end), 10)\n\tif step == 1 {\n\t\treturn b + \"-\" + e\n\t}\n\tif beg+step == end {\n\t\treturn b + \",\" + e\n\t}\n\n\ts := strconv.FormatInt(int64(step), 10)\n\treturn b + \"-\" + e + \":\" + s\n}\n"
  },
  {
    "path": "pkg/utils/cpuset/cpuset_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage cpuset\n\nimport (\n\t\"testing\"\n)\n\nfunc TestShortCPUSet(t *testing.T) {\n\ttcases := []struct {\n\t\tsource string\n\t\tnative string\n\t\tshort  string\n\t}{\n\t\t{source: \"\", native: \"\", short: \"\"},\n\t\t{source: \"1\", native: \"1\", short: \"1\"},\n\t\t{source: \"1,2\", native: \"1-2\", short: \"1,2\"},\n\t\t{source: \"1,2,3,4,5,6,7\", native: \"1-7\", short: \"1-7\"},\n\t\t{source: \"1,3,5,7,9,11\", native: \"1,3,5,7,9,11\", short: \"1-11:2\"},\n\t\t{source: \"1,3,5,7,8,10,12,14,16\", native: \"1,3,5,7-8,10,12,14,16\", short: \"1-7:2,10-16:2\"},\n\t\t{\n\t\t\tsource: \"0,2,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110\",\n\t\t\tnative: \"0,2,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110\",\n\t\t\tshort:  \"0-110:2\",\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tcset := MustParse(tc.source)\n\t\tnative := cset.String()\n\t\tif native != tc.native {\n\t\t\tt.Errorf(\"incorrect native CPUSet for %q, expected %q, got %q\",\n\t\t\t\ttc.source, tc.native, native)\n\t\t}\n\t\tshort := ShortCPUSet(cset)\n\t\tif native != tc.native {\n\t\t\tt.Errorf(\"incorrect shortened CPUSet for %q, expected %q, got %q\",\n\t\t\t\ttc.source, tc.short, short)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/utils/json.go",
    "content": "/*\nCopyright 2019 Intel Corporation\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage utils\n\nimport (\n\t\"fmt\"\n\n\t\"sigs.k8s.io/yaml\"\n)\n\n// DumpJSON dumps a json-compatible struct in human-readable form\nfunc DumpJSON(r interface{}) string {\n\tout, err := yaml.Marshal(r)\n\tif err != nil {\n\t\treturn fmt.Sprintf(\"!!!!!\\nUnable to stringify %T: %v\\n!!!!!\", r, err)\n\t}\n\treturn string(out)\n}\n"
  },
  {
    "path": "pkg/utils/net.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage utils\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"google.golang.org/grpc\"\n)\n\n// WaitForServer waits for a gRPC server to start accepting connections on a socket.\nfunc WaitForServer(socket string, timeout time.Duration, opts ...interface{}) error {\n\tvar errChecker []func(error) bool\n\tvar dialOpts []grpc.DialOption\n\tvar connp **grpc.ClientConn\n\n\tfor _, o := range opts {\n\t\tswitch o.(type) {\n\t\tcase func(error) bool:\n\t\t\terrChecker = append(errChecker, o.(func(error) bool))\n\t\tcase grpc.DialOption:\n\t\t\tdialOpts = append(dialOpts, o.(grpc.DialOption))\n\t\tcase []grpc.DialOption:\n\t\t\tdialOpts = append(dialOpts, o.([]grpc.DialOption)...)\n\t\tcase **grpc.ClientConn:\n\t\t\tif connp != nil {\n\t\t\t\treturn fmt.Errorf(\"WaitForServer: multiple net.Conn pointer options given\")\n\t\t\t}\n\t\t\tconnp = o.(**grpc.ClientConn)\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"WaitForServer: invalid option of type %T\", o)\n\t\t}\n\t}\n\n\tif len(errChecker) < 1 {\n\t\terrChecker = []func(error) bool{isFatalDialError}\n\t}\n\n\tif len(dialOpts) == 0 {\n\t\tdialOpts = []grpc.DialOption{\n\t\t\tgrpc.WithInsecure(),\n\t\t\tgrpc.WithBlock(),\n\t\t\tgrpc.FailOnNonTempDialError(true),\n\t\t\tgrpc.WithTimeout(timeout),\n\t\t\tgrpc.WithDialer(func(socket string, timeout time.Duration) (net.Conn, error) {\n\t\t\t\tconn, err := net.Dial(\"unix\", socket)\n\t\t\t\treturn conn, err\n\t\t\t}),\n\t\t}\n\t}\n\n\tstart := time.Now()\n\tfor {\n\t\tconn, err := grpc.Dial(socket, dialOpts...)\n\t\tif err == nil {\n\t\t\tif connp != nil {\n\t\t\t\t*connp = conn\n\t\t\t} else {\n\t\t\t\tconn.Close()\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\n\t\tfor _, f := range errChecker {\n\t\t\tif f(err) {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tswitch {\n\t\tcase timeout >= 0 && start.Add(timeout).Before(time.Now()):\n\t\t\treturn err\n\t\tcase timeout < 0 || timeout > time.Second:\n\t\t\ttime.Sleep(time.Second)\n\t\tdefault:\n\t\t\ttime.Sleep(timeout / 2)\n\t\t}\n\t}\n}\n\n// IsListeningSocket returns true if connections are accepted on the socket.\nfunc IsListeningSocket(socket string) (bool, error) {\n\tconn, err := net.Dial(\"unix\", socket)\n\tif err == nil {\n\t\tconn.Close()\n\t\treturn true, nil\n\t}\n\n\tif errors.Is(err, syscall.ECONNREFUSED) || os.IsNotExist(err) {\n\t\treturn false, nil\n\t}\n\n\treturn false, err\n}\n\n// Check if a socket connection error looks fatal.\n//\n// Notes:\n//   Hmm... I wonder if it is really so difficult or I am just doing\n//   it wrong ? We would like to find out if a connection attempt to\n//   a unix-domain socket fails with a fatal error, in which case we\n//   don't want to stick around retrying it later.\n//\n//   We treat errors which the originating layer considers a timeout\n//   or a temporary error as non-fatal one. Otherwise, we single out\n//   a few special errors:\n//     - EPERM: fatal error\n//     - EACCES: fatal error\n//     - ENOENT: non-fatal, server might still come around\n//     - ECONNREFUSED: non-fatal, maybe a lingering socket\n//\n\ntype temporary interface {\n\tTemporary() bool\n}\n\ntype timeout interface {\n\tTimeout() bool\n}\n\ntype origin interface {\n\tOrigin() error\n}\n\nfunc isFatalDialError(err error) bool {\n\tfor {\n\t\tif e, ok := err.(temporary); ok {\n\t\t\tif e.Temporary() {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\tif e, ok := err.(timeout); ok {\n\t\t\tif e.Timeout() {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\n\t\tswitch err.(type) {\n\t\tcase *net.OpError:\n\t\t\terr = err.(*net.OpError).Err\n\t\t\tcontinue\n\t\tcase *os.SyscallError:\n\t\t\tne := err.(*os.SyscallError)\n\t\t\tswitch {\n\t\t\tcase os.IsPermission(ne):\n\t\t\t\treturn true\n\t\t\tcase os.IsNotExist(ne):\n\t\t\t\treturn false\n\t\t\tcase ne.Err == syscall.ECONNREFUSED:\n\t\t\t\treturn true\n\t\t\tdefault:\n\t\t\t\terr = ne\n\t\t\t\tcontinue\n\t\t\t}\n\t\tdefault:\n\t\t\tif oe, ok := err.(origin); ok {\n\t\t\t\terr = oe.Origin()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\treturn true\n\t}\n}\n"
  },
  {
    "path": "pkg/utils/parse.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\n// ParseEnabled returns whether the given string represents an 'enabled' state.\nfunc ParseEnabled(value string) (bool, error) {\n\tswitch strings.ToLower(value) {\n\tcase \"true\", \"on\", \"enable\", \"enabled\", \"1\":\n\t\treturn true, nil\n\tcase \"false\", \"off\", \"disable\", \"disabled\", \"0\":\n\t\treturn false, nil\n\tdefault:\n\t\treturn false, fmt.Errorf(\"ParseEnabled: invalid string %q\", value)\n\t}\n}\n"
  },
  {
    "path": "pkg/utils/sort.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage utils\n\nimport (\n\t\"sort\"\n)\n\n// SortUint64s sorts a slice of uint64 in increasing order.\nfunc SortUint64s(a []uint64) {\n\tsort.Sort(Uint64Slice(a))\n}\n\n// Uint64Slice implmenents sort.Interface for a slice of uint64.\ntype Uint64Slice []uint64\n\n// Len returns the length of an UintSlice\nfunc (s Uint64Slice) Len() int { return len(s) }\n\n// Less returns true if element at 'i' is less than the element at 'j'\nfunc (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }\n\n// Swap swaps the values of two elements\nfunc (s Uint64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }\n"
  },
  {
    "path": "pkg/utils/tar.go",
    "content": "// Copyright 2019-2021 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage utils\n\nimport (\n\t\"archive/tar\"\n\t\"compress/bzip2\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n)\n\nfunc UncompressTbz2(archive string, dir string) error {\n\tfile, err := os.Open(archive)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\n\tdata := bzip2.NewReader(file)\n\ttr := tar.NewReader(data)\n\tfor {\n\t\theader, err := tr.Next()\n\t\tif err != nil {\n\t\t\tif err == io.EOF {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif header.Typeflag == tar.TypeDir {\n\t\t\t// Create a directory.\n\t\t\terr = os.MkdirAll(path.Join(dir, header.Name), 0755)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else if header.Typeflag == tar.TypeReg {\n\t\t\t// Create a regular file.\n\t\t\ttargetFile, err := os.Create(path.Join(dir, header.Name))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\t_, err = io.Copy(targetFile, tr)\n\t\t\ttargetFile.Close()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else if header.Typeflag == tar.TypeSymlink {\n\t\t\t// Create a symlink and all the directories it needs.\n\t\t\terr = os.MkdirAll(path.Dir(path.Join(dir, header.Name)), 0755)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr := os.Symlink(header.Linkname, path.Join(dir, header.Name))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/version/version.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//\n// This module lets one tag built binaries with version metadata.\n//\n// Currently two pieces of metadata tracked/provided:\n//   - Version: version number, by convention one provided by 'git describe'\n//   - Build:   build id, by convention the git SHA1 the binary has been built from.\n//\n// To enable automatic versioning metadata for your binary, you need to\n//\n//   1) import this package\n//   2) add the linker flags to override the dummy package variables, for instance:\n//        LDFLAGS=-ldflags \\\n//          \"-X=github.com/intel/cri-resource-manager/pkg/version.Version=<version> \\\n//           -X=github.com/intel/cri-resource-manager/pkg/version.Build=<build-id>\"\n//\n// Note that further metadata can be trivially added in a similar fashion:\n//\n//   1) add the corresponding variables to this modules\n//   2) arrange the default values to be correctly overridden during linking\n//   3) add printing of the new metadata to PrintVersionInfo()\n//\n\npackage version\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n)\n\n// Default values of variables we'll override with the linker.\nvar (\n\t// Version is our version as given by 'git describe'.\n\tVersion = \"<If you see this, you ain't doin' it right, Jimbo...>\"\n\t// Build is the SHA1 of the repository we've been built from.\n\tBuild = \"<If you see this, you ain't doin' it right, Jimbo...>\"\n)\n\n// PrintVersionInfo prints version information about this binary.\nfunc PrintVersionInfo() {\n\tfmt.Printf(\"%s version information:\\n\", filepath.Base(os.Args[0]))\n\tfmt.Printf(\"  - version: %s\\n\", Version)\n\tfmt.Printf(\"  - build:   %s\\n\", Build)\n}\n\n// Dummy struct used to hook into flag.Value.Set of -version during commandline parsing.\ntype version struct{}\n\n// IsBoolFlag tell flag that we only have optional arguments.\nfunc (version) IsBoolFlag() bool {\n\treturn true\n}\n\n// Set is our dummy flag.Value setter.\nfunc (version) Set(value string) error {\n\tprintVersion, err := strconv.ParseBool(value)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif printVersion {\n\t\tPrintVersionInfo()\n\t\tos.Exit(0)\n\t}\n\n\treturn nil\n}\n\n// String is our dummy flag.Value stringification function.\nfunc (*version) String() string {\n\treturn \"false\"\n}\n\n// Put in place a '--version' command line option for us.\nfunc init() {\n\tflag.Var(&version{}, \"version\", \"Print version information about \"+filepath.Base(os.Args[0]))\n}\n"
  },
  {
    "path": "runtime-deps.csv",
    "content": "Go,https://github.com/golang/go,9051\nfsnotify,https://github.com/fsnotify/fsnotify,8402\nyaml,https://github.com/ghodss/yaml,9746\ngrpc-go,https://github.com/grpc/grpc-go,7283\nkubernetes,https://github.com/kubernetes/kubernetes,9641\n"
  },
  {
    "path": "sample-configs/balloons-policy.cfg",
    "content": "policy:\n  Active: balloons\n  # Use only 15 CPUs in total, leave cpu0 for other than Kubernetes\n  # processes.\n  AvailableResources:\n    CPU: cpuset:1-15\n  # Reserve one of our CPUs (cpu15) for kube-system tasks.\n  ReservedResources:\n    CPU: cpuset:15\n  balloons:\n    # PinCPU: allow containers to use only the CPUs in their balloons.\n    PinCPU: true\n    # PinMemory: allow containers to use only the closest memory to\n    # the CPUs in their balloons.\n    PinMemory: true\n    # IdleCPUClass: how to configure CPUs that are not included in any\n    # of the balloons.\n    IdleCPUClass: idle\n    BalloonTypes:\n      - Name: \"full-core-turbo\"\n        # MinCPUs: minimum number of logical cores in every balloon\n        # instance of this type.\n        # The default is 0.\n        MinCPUs: 2\n        # MaxCPUs: maximum number of logical cores in every balloon\n        # instance of this type.\n        # The default is 0 (unlimited).\n        MaxCPUs: 2\n        # CPUClass: how to configure CPUs of these balloons.\n        # The default is \"\".\n        CPUClass: \"turbo\"\n        # Namespaces: assign pods in listed namespaces to these\n        # balloons, even if there is no explicit annotation:\n        # balloon.balloons.cri-resource-manager.intel.com: full-core-turbo\n        # The default is to assign only annotated pods.\n        Namespaces:\n          - \"highperf\"\n        # AllocatorPriotity: CPU allocator priority (0: High, 1:\n        # Normal, 2: Low, 3: None). Affects the performance/type of\n        # CPUs that are selected into the balloon. CPUs for static\n        # balloon instances (MinBalloons > 0) with highest\n        # AllocatorPriority are reserved first.\n        # The default is 0.\n        AllocatorPriority: 2\n        # MinBalloons: how many balloon instances of this type are always\n        # kept in the system, even if there would not be workloads to them.\n        # The default is 0.\n        MinBalloons: 2\n        # PreferNewBalloons: prefer creating a new balloon for\n        # separate pods, even if their CPU requirements would allow\n        # putting them in the same balloon.\n        # The default is: false.\n        PreferNewBalloons: true\n        # PreferPerNamespaceBalloon: if true, containers in the same\n        # namespace are preferrably placed in the same balloon, and\n        # containers in different namespaces to different\n        # balloons. The default is false: namespaces have no effect on\n        # placement.\n        PreferPerNamespaceBalloon: false\n        # PreferSpreadingPods: if true, containers of single pod can\n        # be assigned in different balloons, based on which balloons\n        # have most free CPU resources.\n        # The default is: false: prefer running containers of a same\n        # pod in the same balloon(s).\n        PreferSpreadingPods: false\n\n      - Name: \"socket-size\"\n        MaxCPUs: 8\n        AllocatorPriority: 2\n        Namespaces:\n          - \"default\"\n        CPUClass: \"normal\"\n# CPU controller configuration specifies CPU class properties. CPUs of\n# each balloon are configured based on its CPUClass. If a balloon has\n# no CPUClass, the properties of the default class are applied.\ncpu:\n  classes:\n    default:\n      minFreq: 800\n      maxFreq: 1600\n    turbo:\n      minFreq: 3300\n      maxFreq: 3600\n    normal:\n      minFreq: 800\n      maxFreq: 2400\ninstrumentation:\n  # The balloons policy exports containers running in each balloon,\n  # and cpusets of balloons. Accessible in command line:\n  # curl --silent http://localhost:8891/metrics\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n"
  },
  {
    "path": "sample-configs/blockio.cfg",
    "content": "# This configuration demonstrates how to configure cgroups block io\n# controller for pods.\n#\n# The configuration defines block device parameters for three blockio\n# classes (LowPrioThrottled, HighPrioFullSpeed and Default, feel free\n# to choose any names here). Finally resource-manager.blockio maps QOS\n# classes BestEffort, Burstable (via wildcard), and Guaranteed to\n# these classes.\n#\n# Try with: cri-resmgr -force-config blockio.cfg\n\npolicy:\n  Active: none\n\nlogger:\n  Debug: blockio,cgroupblkio\n\nblockio:\n  Classes:\n    # LowPrioThrottled and HighPrioFullSpeed are user-defined blockio classes\n    # in this example. Pods and containers can be assigned to these classes using Pod\n    # metadata annotations. For example in Pod yaml:\n    # ...\n    # metadata:\n    #   annotations:\n    #     # Default blockio class for containers in the pod:\n    #     blockioclass.cri-resource-manager.intel.com/pod: LowPrioThrottled\n    #     # Special blockio class for a container in the pod:\n    #     blockioclass.cri-resource-manager.intel.com/container.mycontainer: HighPrioFullSpeed\n    LowPrioThrottled:\n      # Default io-scheduler weight for all devices that are not\n      # explicitly mentioned in following items.\n      - Weight: 80 # will be written to cgroups(.bfq).weight\n\n      # Configuration for all virtio and scsi block devices.\n      - Devices:\n          - /dev/vd*\n          - /dev/sd*\n        ThrottleReadBps: 50M   # max read bytes per second\n        ThrottleWriteBps: 10M  # max write bytes per second\n        ThrottleReadIOPS: 10k  # max read io operations per second\n        ThrottleWriteIOPS: 5k  # max write io operations per second\n        Weight: 50             # io-scheduler (cfq/bfq) weight for these devices,\n                               # will be written to cgroups(.bfq).weight_device\n\n      # Configuration for SSD devices.\n      # This overrides above configuration for those /dev/sd* devices\n      # whose disk id contains \"SSD\"\n      - Devices:\n          - /dev/disk/by-id/*SSD*\n        ThrottleReadBps: 100M\n        ThrottleWriteBps: 40M\n        # Not mentioning Throttle*IOPS means no io operations throttling for matching devices.\n        Weight: 50\n\n    HighPrioFullSpeed:\n      - Weight: 400\n\n    # When Pod annotations do not define blockio class, QoS class\n    # names (BestEffort, Burstable, Guaranteed) are used as blockio\n    # class names for the pod. By default no blockio configuration\n    # takes place for them, but here we define I/O scheduler weight\n    # difference:\n    BestEffort:\n      - Weight: 90\n    Guaranteed:\n      - Weight: 200\n"
  },
  {
    "path": "sample-configs/cri-full-message-dump.cfg",
    "content": "# run with no-op policy\npolicy:\n  Active: none\n# enable full dumps of all messages\ndump:\n  Config: full:.*\n"
  },
  {
    "path": "sample-configs/cri-resmgr-configmap.example.yaml",
    "content": "#\n# This example creates 3 ConfigMaps:\n#  - cri-resmgr-config.default: the default configuration\n#  - cri-resmgr-config.group.foo: the configuration for nodes in group foo\n#  - cri-resmgr-config.node.cl0-slave1: the configuration for node cl0-slave1\n#\n# You can assign nodes to group foo using the command\n#   kubectl label --overwrite node $NODE_NAME cri-resource-manager.intel.com/group=foo\n#\n# You can remove nodes from group foo using the command\n#   kubectl label node $NODE_NAME cri-resource-manager.intel.com/group-\n#\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.default\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: topology-aware\n    AvailableResources:\n      cpu: cpuset:0-63\n    ReservedResources:\n      cpu: cpuset:0-1\n    topology-aware:\n      PinCPU: true\n      PinMemory: true\n      PreferIsolatedCPUs: true\n      PreferSharedCPUs: false\n    static:\n      RelaxedIsolation: true\n    static-pools:\n      # Filesystem path to legacy configuration directory structure\n      ConfDirPath: \"/etc/cmk\"\n      # Filesystem path to legacy configuration file\n      ConfFilePath: \"\"\n      # Whether to create CMK node label\n      LabelNode: false\n      # Whether to create CMK node taint\n      TaintNode: false\n      # Pool configuration.\n      # The imaginary example system below consists of 4 sockets, 4 cores, 2\n      # threads each.\n      pools:\n        exclusive:\n          # 6 exclusive cores, 3 on sockets 1, 2 and 3 each\n          cpuLists:\n          - Cpuset: 8,9\n            Socket: 1\n          - Cpuset: 10,11\n            Socket: 1\n          - Cpuset: 16,17\n            Socket: 2\n          - Cpuset: 18,19\n            Socket: 2\n          - Cpuset: 24,25\n            Socket: 3\n          - Cpuset: 26,27\n            Socket: 3\n          exclusive: true\n        shared:\n          # 2 cores in shared pool, all on socket 1\n          cpuLists:\n          - Cpuset: 12,13,14,15\n            Socket: 1\n          exclusive: false\n        infra:\n          # Rest of cores designated to infra pool\n          cpuLists:\n          - Cpuset: 0,1,2,3,4,5,6,7\n            Socket: 0\n          - Cpuset: 20,21,22,23\n            Socket: 2\n          - Cpuset: 28,29,30,31\n            Socket: 3\n          exclusive: false\n  rdt: |+\n    # Common options\n    options:\n      # One of Full, Discovery or Disabled\n      mode: Full\n      # Set to true to disable creation of monitoring groups\n      monitoringDisabled: false\n      l3:\n        # Make this false if L3 CAT must be available\n        optional: true\n      mb:\n        # Make this false if MBA must be available\n        optional: true\n\n    # Configuration of classes\n    partitions:\n      exclusive:\n        # Allocate 60% of all L3 cache to the \"exclusive\" partition\n        l3Allocation: \"60%\"\n        mbAllocation: [\"100%\"]\n        classes:\n          Guaranteed:\n            # Allocate all of the partitions cache lines to \"Guaranteed\"\n            l3Allocation: \"100%\"\n      shared:\n        # Allocate 40% L3 cache IDs to the \"shared\" partition\n        # These will NOT overlap with the cache lines allocated for \"exclusive\" partition\n        l3Allocation: \"40%\"\n        mbAllocation: [\"50%\"]\n        classes:\n          Burstable:\n            # Allow \"Burstable\" to use all cache lines of the \"shared\" partition\n            l3Allocation: \"100%\"\n          BestEffort:\n            # Allow \"Besteffort\" to use only half of the L3 cache # lines of the \"shared\" partition.\n            # These will overlap with those used by \"Burstable\"\n            l3Allocation: \"50%\"\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.group.foo\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: topology-aware\n    AvailableResources:\n      cpu: cpuset:0-63\n    ReservedResources:\n      cpu: cpuset:0-1\n    topology-aware:\n      PinCPU: true\n      PinMemory: false\n      PreferIsolatedCPUs: false\n      PreferSharedCPUs: false\n    static:\n      RelaxedIsolation: true\n    static-pools:\n      # This is an example configuration for static-pools policy.\n      # The imaginary example system here consists of 4 sockets, 4 cores, 2 threads each.\n      pools:\n        exclusive:\n          # 6 exclusive cores, 3 on sockets 1, 2 and 3 each\n          cpuLists:\n          - Cpuset: 8,9\n            Socket: 1\n          - Cpuset: 10,11\n            Socket: 1\n          - Cpuset: 16,17\n            Socket: 2\n          - Cpuset: 18,19\n            Socket: 2\n          - Cpuset: 24,25\n            Socket: 3\n          - Cpuset: 26,27\n            Socket: 3\n          exclusive: true\n        shared:\n          # 2 cores in shared pool, all on socket 1\n          cpuLists:\n          - Cpuset: 12,13,14,15\n            Socket: 1\n          exclusive: false\n        infra:\n          # Rest of cores designated to infra pool\n          cpuLists:\n          - Cpuset: 0,1,2,3,4,5,6,7\n            Socket: 0\n          - Cpuset: 20,21,22,23\n            Socket: 2\n          - Cpuset: 28,29,30,31\n            Socket: 3\n          exclusive: false\n  rdt: |+\n    # Common options\n    options:\n      # One of Full, Discovery or Disabled\n      mode: Full\n      # Set to true to disable creation of monitoring groups\n      monitoringDisabled: false\n      l3:\n        # Make this false if L3 CAT must be available\n        optional: true\n      mb:\n        # Make this false if MBA must be available\n        optional: true\n\n    # Configuration of classes\n    partitions:\n      exclusive:\n        # Allocate 60% of all L3 cache to the \"exclusive\" partition\n        l3Allocation: \"60%\"\n        mbAllocation: [\"100%\"]\n        classes:\n          Guaranteed:\n            # Allocate all of the partitions cache lines to \"Guaranteed\"\n            l3Allocation: \"100%\"\n      shared:\n        # Allocate 40% L3 cache IDs to the \"shared\" partition\n        # These will NOT overlap with the cache lines allocated for \"exclusive\" partition\n        l3Allocation: \"40%\"\n        mbAllocation: [\"50%\"]\n        classes:\n          Burstable:\n            # Allow \"Burstable\" to use all cache lines of the \"shared\" partition\n            l3Allocation: \"100%\"\n          BestEffort:\n            # Allow \"Besteffort\" to use only half of the L3 cache # lines of the \"shared\" partition.\n            # These will overlap with those used by \"Burstable\"\n            l3Allocation: \"50%\"\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.node.cl0-slave1\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: topology-aware\n    AvailableResources:\n      cpu: cpuset:0-63\n    ReservedResources:\n      cpu: cpuset:0-1\n    topology-aware:\n      PinCPU: false\n      PinMemory: true\n      PreferIsolatedCPUs: false\n      PreferSharedCPUs: false\n    static:\n      RelaxedIsolation: true\n    static-pools:\n      # This is an example configuration for static-pools policy.\n      # The imaginary example system here consists of 4 sockets, 4 cores, 2 threads each.\n      pools:\n        exclusive:\n          # 6 exclusive cores, 3 on sockets 1, 2 and 3 each\n          cpuLists:\n          - Cpuset: 8,9\n            Socket: 1\n          - Cpuset: 10,11\n            Socket: 1\n          - Cpuset: 16,17\n            Socket: 2\n          - Cpuset: 18,19\n            Socket: 2\n          - Cpuset: 24,25\n            Socket: 3\n          - Cpuset: 26,27\n            Socket: 3\n          exclusive: true\n        shared:\n          # 2 cores in shared pool, all on socket 1\n          cpuLists:\n          - Cpuset: 12,13,14,15\n            Socket: 1\n          exclusive: false\n        infra:\n          # Rest of cores designated to infra pool\n          cpuLists:\n          - Cpuset: 0,1,2,3,4,5,6,7\n            Socket: 0\n          - Cpuset: 20,21,22,23\n            Socket: 2\n          - Cpuset: 28,29,30,31\n            Socket: 3\n          exclusive: false\n  rdt: |+\n    # Common options\n    options:\n      # One of Full, Discovery or Disabled\n      mode: Full\n      # Set to true to disable creation of monitoring groups\n      monitoringDisabled: false\n      l3:\n        # Make this false if L3 CAT must be available\n        optional: true\n      mb:\n        # Make this false if MBA must be available\n        optional: true\n\n    # Configuration of classes\n    partitions:\n      exclusive:\n        # Allocate 60% of all L3 cache to the \"exclusive\" partition\n        l3Allocation: \"60%\"\n        mbAllocation: [\"100%\"]\n        classes:\n          Guaranteed:\n            # Allocate all of the partitions cache lines to \"Guaranteed\"\n            l3Allocation: \"100%\"\n      shared:\n        # Allocate 40% L3 cache IDs to the \"shared\" partition\n        # These will NOT overlap with the cache lines allocated for \"exclusive\" partition\n        l3Allocation: \"40%\"\n        mbAllocation: [\"50%\"]\n        classes:\n          Burstable:\n            # Allow \"Burstable\" to use all cache lines of the \"shared\" partition\n            l3Allocation: \"100%\"\n          BestEffort:\n            # Allow \"Besteffort\" to use only half of the L3 cache # lines of the \"shared\" partition.\n            # These will overlap with those used by \"Burstable\"\n            l3Allocation: \"50%\"\n  dump: |+\n    Config: full:.*,short:.*Stop.*,off:.*List.*\n    File: /tmp/cri-selective-debug.dump\n  logger: |+\n    Debug: resource-manager,cache\n"
  },
  {
    "path": "sample-configs/external-adjustment.yaml",
    "content": "apiVersion: criresmgr.intel.com/v1alpha1\nkind: Adjustment\nmetadata:\n  name: external-adjustment\n  namespace: kube-system\nspec:\n  scope:\n    - nodes: [ node-1 ]\n      containers:\n        - key: \":,:pod/name,name\"\n          operator: Matches\n          values: [ \"*:container\" ]\n    - nodes: [ node-2 ]\n      containers:\n        - key: \":,:pod/name,name\"\n          operator: Matches\n          values: [ \"pod:*\" ]\n    - nodes: [ node-3, node-4 ]\n      containers:\n        - key: \":,:pod/name,name\"\n          operator: Equals\n          values: [ \"anotherpod:container\" ]\n  resources:\n    requests:\n      cpu: 750m\n      memory: 500Mi\n    limits:\n      cpu: 1500m\n      memory: 750Mi\n  toptierLimit: 500Mi\n  classes:\n    rdt: rdt-class-1\n    blockio: blockio-class-1\n"
  },
  {
    "path": "sample-configs/podpools-policy.cfg",
    "content": "# This example demonstrates pod-based CPU and memory pinning.\n# All containers of a pod run in the same CPU/memory pool.\n# The capacity of a pool is defined as a number of pods it can\n# contain.\n#\n# The two steps for running a pod in a pod pool are:\n#\n# 1. Annotate the pod:\n#\n#    metadata:\n#      annotations:\n#        pool.podpools.cri-resource-manager.intel.com: POOLNAME\n#\n# 2. Make sure that total CPU resources required by the containers\n#    in the pod match the CPUs per pod in the pod pool.\n\npolicy:\n  # pod-based CPU and memory pinning is implemented in the podpools policy.\n  Active: podpools\n\n  # AvailableResources specifies CPUs that active policy is allowed to\n  # use: containers will not run outside AvailableResources\n  # CPUs. Other CPUs are considered reserved for system. Corresponding\n  # kubelet parameter: --system-reserved. By default\n  # AvailableResources contains all CPUs.\n  AvailableResources:\n    # \"CPU\" can be the number of CPUs or explicitly defined set of\n    # CPUs. In this example we use 14 CPUs, excluding CPUs #0 and #1\n    # (hyperthreads of core 0).\n    CPU: cpuset:2-15\n\n  # ReservedResources specifies CPU(s) that active policy dedicates\n  # for running kube-system pods. Corresponding kubelet parameter:\n  # --kube-reserved.\n  ReservedResources:\n    # Here we dedicate CPU #15 for these pods.\n    # This leaves 13 out of 14 available CPUs unallocated.\n    CPU: cpuset:15\n\n  # podpools-specific configuration specifies the following.\n  # 1. Pod pool definitions (\"Pools\").\n  #    The policy creates one or more pool instances from a definition.\n  # 2. Resources (CPUs) needed by each pod pool definition in total.\n  #    This can be given as one of the following:\n  #    1. a number of pool instances:      \"Instances: <NUM>\"\n  #    2. a number of CPUs:                \"Instances: <NUM> CPUs\"\n  #    3. percentage of non-reserved CPUs: \"Instances: <NUM> %\"\n  #    In case 1, CPUs needed by the definition is <NUM> * CPUs per pool.\n  # 3. How many CPUs each pool instance gets from the CPUs allocated\n  #    to its definition in total.\n  # 4. Capacity of each pool instance.\n  #    This is the maximum number of pods in a single pool instance.\n  podpools:\n    # By default podpools pins both CPU and memory of all containers.\n    # Pinning either of them can be disabled with:\n    # pinCPU: false\n    # pinMemory: false\n    Pools:\n      # Define the \"singlecpu\" pod pool type:\n      - Name: singlecpu\n        # Take 3 out of 13 AvailableResources CPUs to be used by\n        # all \"singlecpu\" pod pool instances in total.\n        # This leaves 10 CPUs unallocated for other pools.\n        Instances: 3 CPUs\n        # Every \"singlecpu\" pod pool instance has 1 CPU to run all\n        # pods assigned to the instance.\n        # As the definition can use 3 CPUs in total, there will be 3\n        # \"singlecpu\" pool instances.\n        CPU: 1\n        # Every \"singlecpu\" pod pool instance holds at most 2 pods.\n        MaxPods: 2\n\n        # Note that every pod that is annotated to run on a singlecpu\n        # pool is assumed to consume CPU/MaxPods = 500m CPU. Therefore\n        # the sum of request.cpu's of all containers in this kind of\n        # pod should be 500m. Otherwise kube-scheduler may overload or\n        # underload the node.\n\n      # Define the \"dualcpu\" pod pool type:\n      - Name: dualcpu\n        # FillOrder specifies the order in which the capacity of pod\n        # pool instances of this pool type is filled with pods. The\n        # default is Balanced: new pod is assigned to a pool instance\n        # with most free capacity. The opposite is Packed: new pod is\n        # assigned to a pool instance with least free capacity.\n        FillOrder: Packed\n        # Take at most 50 % of non-reserved CPUs (50 % * 13 = 6.5)\n        # to be used by all \"dualcpu\" pool instances in total.\n        Instances: 50 %\n        # Every \"dualcpu\" pool instance has 2 CPUs.\n        # That is, floor(6.5 / 2) = 3 pool instances of this type will\n        # be created, and therefore 6 CPUs actually consumed to this\n        # pool type.\n        # This leaves 4 CPUs unallocated.\n        CPU: 2\n        # Every \"dualcpu\" pool instance holds at most 3 pods.\n        MaxPods: 3\n\n      # In addition to user-defined pools, there are two built-in\n      # pools:\n      #\n      # - \"reserved\" contains the ReservedResources CPUs and runs all\n      #   kube-system pods.\n      #\n      # - \"default\" contains CPUs that are neither reserved nor\n      #   allocated to any user-defined pools. It runs all pods that\n      #   are not kube-system and are not assigned to any user-defined\n      #   pool. The number of CPUs in the default pool can be\n      #   overridden by defining \"default\" pool like other pools. If\n      #   CPUs were not left over for the default pool, it will use\n      #   the same CPUs as the reserved pool.\nlogger:\n  Debug: policy\n"
  },
  {
    "path": "sample-configs/static-policy.cfg",
    "content": "policy:\n  Active: static\n  ReservedResources:\n    CPU: 1000m\nlogger:\n  Debug: policy,static\ndump:\n  Config: off:.*,full:((Create)|(Remove)|(Run)|(Update)|(Start)|(Stop)).*\n"
  },
  {
    "path": "sample-configs/static-pools-policy.conf.example",
    "content": "# This is an example configuration file for the builtin cmk policy\n# The imaginary example system here consists of 4 sockets, 4 cores (8\n# multithreaded CPUs)\n#\n# NOTE: only pools configuration may be specified in this file. Other\n# configuration options must be set through the dynamic configration system\npools:\n  exclusive:\n    # 6 exclusive cores, 3 on sockets 1, 2 and 3 each\n    cpuLists:\n    - Cpuset: 8,9\n      Socket: 1\n    - Cpuset: 10,11\n      Socket: 1\n    - Cpuset: 16,17\n      Socket: 2\n    - Cpuset: 18,19\n      Socket: 2\n    - Cpuset: 24,25\n      Socket: 3\n    - Cpuset: 26,27\n      Socket: 3\n    exclusive: true\n  shared:\n    # 2 cores in shared pool, all on socket 1\n    cpuLists:\n    - Cpuset: 12,13,14,15\n      Socket: 1\n    exclusive: false\n  infra:\n    # Rest of cores designated to infra pool\n    cpuLists:\n    - Cpuset: 0,1,2,3,4,5,6,7\n      Socket: 0\n    - Cpuset: 20,21,22,23\n      Socket: 2\n    - Cpuset: 28,29,30,31\n      Socket: 3\n    exclusive: false\n\n"
  },
  {
    "path": "sample-configs/topology-aware-policy.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*\n"
  },
  {
    "path": "scripts/build/docker-build-image",
    "content": "#!/bin/bash\n\nIMAGE=$1\nDOCKERFILE=dockerfiles/cross-build/Dockerfile.${IMAGE%-build}\nshift 1\n\necho \"* Building docker images with\"\necho \"  - Dockerfile: $DOCKERFILE\"\necho \"  - image name: $IMAGE\"\necho \"  - options   : $@\"\n\ndocker build . \\\n       -f \"$DOCKERFILE\" -t \"$IMAGE\" \\\n       --build-arg \"CREATE_USER=$USER\" \\\n       --build-arg USER_UID=\"$(id -u)\" \\\n       \"$@\" || exit 1\n"
  },
  {
    "path": "scripts/build/get-buildid",
    "content": "#!/bin/bash\n\n#\n# Script to determine a version string, a buildid as well as related RPM\n# and debian package versions. These are determined using the following\n# sources in decreasing order of preference:\n#\n#  1. git metadata:\n#    - version: git describe --tags --long --dirty\n#    - buildid: git rev-parse --short HEAD\n#  2. stored git metadata:\n#    - version: git-version\n#    - buildid: git-buildid\n#  3. directory name:\n#    - version: cri-resource-manager-(.*):\n#    - buildid: unknown\n#  4. date:\n#    - version: 0.0.0-$(date +%Y%m%d%H%M)\n#    - buildid: unknown\n#\n\nPARENT_DIRNAME=cri-resource-manager\nVERSION_FILE=version\nBUILDID_FILE=buildid\nVERSION=\"\"\nBUILDID=\"\"\nRPM=\"\"\nDEB=\"\"\n\nfail() {\n    echo \"$*\" 2>&1\n    exit 1\n}\n\nlog() {\n    echo \"$*\" 1>&2\n}\n\nprint_usage() {\n    local _status=0\n    if [ -n \"$*\" ]; then\n        echo \"$*\"\n        _status=1\n    fi\n    echo \"usage $0 [--store[=<dir>]] [--version] [--buildid] [--rpm] [--deb] [--tar] [--all]\"\n    exit $_status\n}\n\ndotgit_hasrepo() {\n    git status >& /dev/null\n}\n\ndotgit_version() {\n    local _v _id _dirty _count\n\n    if  [ -z \"$TEST_DESCRIBE\" ]; then\n        if ! dotgit_hasrepo; then\n            return 1\n        fi\n        _id=$(git rev-parse --short HEAD)\n        _dirty=$(git diff --quiet -- ':!go.mod' ':!go.sum' || echo '-dirty')\n        _v=$(git describe --tags --long --dirty 2>/dev/null)\n    else\n        _v=\"$TEST_DESCRIBE\"\n        _id=\"$TEST_REV\"\n        _dirty=\"\"\n    fi\n\n    case \"$_v\" in\n        v*) _v=\"${_v#v}\"\n            ;;\n        *)\n            _count=$(git rev-list --count HEAD)\n            _v=\"0.0.0-$_count-g$_id$_dirty\"\n            ;;\n    esac\n\n    VERSION=\"$_v\"\n    BUILDID=\"$_id$_dirty\"\n}\n\nstored_hasdata() {\n    if [ ! -f \"$OUTDIR/$VERSION_FILE\" ] || [ ! -f \"$OUTDIR/$BUILDID_FILE\" ]; then\n        return 1\n    fi\n    STORED_VERSION=$(cat \"$OUTDIR/$VERSION_FILE\") && \\\n        STORED_BUILDID=$(cat \"$OUTDIR/$BUILDID_FILE\")\n}\n\nstored_version() {\n    if ! stored_hasdata; then\n        return 1\n    fi\n    VERSION=\"$STORED_VERSION\"\n    BUILDID=\"$STORED_BUILDID\"\n}\n\nstored_update() {\n    if stored_hasdata; then\n        if [ \"$STORED_VERSION\" = \"$VERSION\" ] && [ \"$STORED_BUILDID\" = \"$BUILDID\" ]; then\n            return 0\n        fi\n    fi\n    mkdir -p \"$OUTDIR\" || fail \"failed to create $OUTDIR\"\n    echo \"$VERSION\" > \"$OUTDIR/$VERSION_FILE\"\n    echo \"$BUILDID\" > \"$OUTDIR/$BUILDID_FILE\"\n}\n\nparent_version() {\n    local _dir\n\n    _dir=$(basename \"$(realpath .)\")\n    case \"$_dir\" in\n        \"${PARENT_DIRNAME}\"-*)\n            VERSION=\"${_dir##${PARENT_DIRNAME}-}\"\n            BUILDID=unknown\n            return 0\n            ;;\n    esac\n    return 1\n}\n\nunknown_version() {\n    VERSION=\"0.0.0-$(date +%Y%m%d%H%M)\"\n    BUILDID=unknown\n}\n\npackage_versions() {\n    case \"$VERSION\" in\n        [0-9.]**-g[0-9a-f]*)\n            local _full=\"$VERSION\"\n            local _numeric=${_full%%-*}\n            local _cntsha1=${_full#*-}\n            local _clean=${_cntsha1%-dirty}\n            local _dirty=${_cntsha1#$_clean}; _cntsha1=\"$_clean\"\n            local _sha1=${_cntsha1##*-g}\n            local _cnt=${_cntsha1%-g*}\n            VERSION=$_numeric\n            if [ -n \"$_cnt\" ] && [ \"$_cnt\" != \"0\" ]; then\n                VERSION=\"$VERSION-$_cnt-g$_sha1\"\n            fi\n            VERSION=$VERSION$_dirty\n            RPM=$(echo \"$VERSION\" | tr '+-' '_')\n            DEB=$VERSION\n            ;;\n        [0-9.]*)\n            RPM=$VERSION\n            DEB=$VERSION\n            ;;\n        *)\n            fail \"can't parse version $VERSION\"\n            ;;\n    esac\n}\n\nprint_variables() {\n    local _what _var _val\n\n    for _what in $PRINT; do\n        case $_what in\n            version)\n                [ -n \"$SHVAR\" ] && _var='gitversion='\n                _val=\"$VERSION\"\n                ;;\n            buildid)\n                [ -n \"$SHVAR\" ] && _var='gitbuildid='\n                _val=\"$BUILDID\"\n                ;;\n            rpm)\n                [ -n \"$SHVAR\" ] && _var='rpmversion='\n                _val=\"$RPM\"\n                ;;\n            deb)\n                [ -n \"$SHVAR\" ] && _var='debversion='\n                _val=\"$DEB\"\n                ;;\n            tar)\n                [ -n \"$SHVAR\" ] && _var='tarversion='\n                _val=\"$VERSION\"\n                ;;\n            *)\n                print_usage \"unknown version/buildid-related tag \\\"$_what\\\"\"\n                ;;\n        esac\n        echo \"$_var$_val\"\n    done\n}\n\n#########################\n# main script\n#\n\nOUTDIR=\".\"\nSTORE=\"\"\nPRINT=\"\"\nSHVAR=y\nTEST_DESCRIBE=\"\"\nTEST_REV=\"\"\n\nwhile [ \"$#\" != \"0\" ]; do\n    case $1 in\n        --help|-h)\n            print_usage\n            ;;\n        --debug)\n            set -x\n            ;;\n        --store=*|-s*)\n            STORE=y\n            out=\"${1##*=}\"\n            if [ \"$out\" != \"$1\" ]; then\n                OUTDIR=\"$out\"\n            fi\n            ;;\n        --version|-v)\n            PRINT=\"$PRINT version\"\n            ;;\n        --buildid|-b)\n            PRINT=\"$PRINT buildid\"\n            ;;\n        --rpm)\n            PRINT=\"$PRINT rpm\"\n            ;;\n        --deb)\n            PRINT=\"$PRINT deb\"\n            ;;\n        --tar)\n            PRINT=\"$PRINT tar\"\n            ;;\n        --all)\n            PRINT=\"version buildid rpm deb tar\"\n            ;;\n        --shell*|--sh-syntax*)\n            val=\"${1##*=}\"\n            if [ \"$val\" != \"$1\" ]; then\n                case $val in\n                    y*|t*) SHVAR=y;;\n                    n*|f*) SHVAR=\"\";;\n                esac\n            else\n                SHVAR=y\n            fi\n            ;;\n        --no-shell|--no-sh-syntax)\n            SHVAR=\"\"\n            ;;\n        --test)\n            TEST_DESCRIBE=\"$2\"\n            TEST_REV=\"$3\"\n            shift 2\n            ;;\n        *)\n            print_usage \"unknown option \\\"$1\\\"\"\n            ;;\n    esac\n    shift\ndone\n\nif ! dotgit_version; then\n    if ! stored_version; then\n        if ! parent_version; then\n            unknown_version\n        fi\n    fi\nfi\n\nif [ -z \"$STORE\" ] && [ -z \"$PRINT\" ]; then\n    PRINT=\"version buildid\"\nfi\n\npackage_versions\nprint_variables\n\nif [ -n \"$STORE\" ]; then\n    stored_update\nfi\n"
  },
  {
    "path": "scripts/build/update-gh-pages.sh",
    "content": "#!/bin/bash -e\nset -o pipefail\n\nscript=`basename $0`\n\nusage () {\ncat << EOF\nUsage: $script [-h] [-a] [BUILD_SUBDIR]\n\nOptions:\n  -h         show this help and exit\n  -a         amend (with --reset-author) instead of creating a new commit\nEOF\n}\n\n# Helper function for detecting available versions from the current directory\ncreate_versions_js() {\n    _baseurl=\"/cri-resource-manager\"\n\n    echo -e \"function getVersionsMenuItems() {\\n  return [\"\n    # 'stable' is a symlink pointing to the latest version\n    [ -f stable ] && echo \"    { name: 'stable', url: '$_baseurl/stable' },\"\n    for f in `ls -d */  | tr -d / | sed s'/releases//'`; do\n        echo \"    { name: '$f', url: '$_baseurl/$f' },\"\n    done\n    echo -e \"  ];\\n}\"\n}\n\n# Helper function for detecting archived releases from the current directory\ncreate_releases_js() {\n    echo -e \"function getReleaseListItems() {\\n  return [\"\n    for f in `ls -d v*/  | tr -d /`; do\n        echo \"    { name: '$f', url: '$f' },\"\n    done\n    echo -e \"  ];\\n}\"\n}\n\n#\n# Argument parsing\n#\nwhile [ \"${1#-}\" != \"$1\" -a -n \"$1\" ]; do\n    case \"$1\" in\n        -a|--amend)\n            amend=\"--amend --reset-author\"\n            ;;\n        -h|--help)\n            usage\n            exit 0\n            ;;\n        *)\n            usage\n            exit 1\n            ;;\n    esac\n    shift\ndone\n\nbuild_subdir=\"$1\"\n\n# Check that no extra args were provided\nif [ $# -gt 1 ]; then\n    echo \"ERROR: unknown arguments: $@\"\n    usage\n    exit 1\nfi\n\n#\n# Build the documentation\n#\nbuild_dir=\"_build\"\necho \"Creating new Git worktree at $build_dir\"\ngit worktree add \"$build_dir\" gh-pages\n\n# Drop worktree on exit\ntrap \"echo 'Removing Git worktree $build_dir'; git worktree remove --force '$build_dir'\" EXIT\n\n# Parse subdir name from GITHUB_REF\nrelease_tag=\nif [ -z \"$build_subdir\" ]; then\n    case \"$GITHUB_REF\" in\n        refs/tags/*)\n            _base_ref=${GITHUB_REF#refs/tags/}\n            release_tag=$_base_ref\n            ;;\n        refs/heads/*)\n            _base_ref=${GITHUB_REF#refs/heads/}\n            ;;\n        *) _base_ref=\n    esac\n    echo \"Parsed baseref: '$_base_ref'\"\n\n    case \"$GITHUB_REF\" in\n        refs/tags/v*)\n            _version=${GITHUB_REF#refs/tags/v}\n            ;;\n        refs/heads/release-*)\n            _version=${GITHUB_REF#refs/heads/release-}\n            ;;\n        *) _version=\n    esac\n    echo \"Detected version: '$_version'\"\n\n    _version=`echo -n $_version | sed -nE s'!^([0-9]+\\.[0-9]+).*$!\\1!p'`\n\n    # Use version as the subdir\n    build_subdir=${_version:+v$_version}\n    # Fallback to base-ref i.e. name of the branch or tag\n    if [ -z \"$build_subdir\" ]; then\n        # For master branch we use the name 'devel'\n        [ \"$_base_ref\" = \"master\" ] && build_subdir=devel || build_subdir=$_base_ref\n    fi\nfi\n\n# Default to 'devel' if no subdir was given and we couldn't parse\n# it\nbuild_subdir=${build_subdir:-devel}\necho \"Updating site version subdir: '$build_subdir'\"\nexport SITE_BUILDDIR=\"$build_dir/$build_subdir\"\nexport VERSIONS_MENU=1\nexport VERSIONS_MENU_THIS_VERSION=$build_subdir\n\nmake html\n\n# Update releases/ subdir\nif [ \"$release_tag\" ]; then\n    echo \"Building archived docs for release $release_tag\"\n\n    export SITE_BUILDDIR=\"$build_dir/releases/$release_tag\"\n    make html\n\nfi\n\n# Only update the releases \"site\" from master\nif [ \"$GITHUB_REF\" = \"refs/heads/master\" ]; then\n    echo \"Building releases/\"\n    sphinx-build docs/releases \"$build_dir\"/releases\nfi\n\n#\n# Update gh-pages branch\n#\ncommit_hash=`git describe --dirty --always`\n\n# Switch to work in the gh-pages worktree\npushd \"$build_dir\"\n\n# Add \"const\" files we need in root dir\ntouch .nojekyll\n\n_stable=`(ls -d1 v*/ || :) | sort -n | tail -n1`\nif [ -n \"$_stable\" ]; then\n    ln -sfT \"$_stable\" stable\n    redirect_to=\"stable\"\nelse\n    redirect_to=$build_subdir\nfi\n\n# Detect existing versions from the gh-pages branch\ncreate_versions_js > versions.js\n\n# Update releases directory\nmkdir -p releases\ncp versions.js releases/\npushd releases\ncreate_releases_js > releases.js\npopd\n\ncat > index.html << EOF\n<meta http-equiv=\"refresh\" content=\"0; URL='$redirect_to'\" />\nEOF\n\nif [ -z \"`git status --short`\" ]; then\n    echo \"No new content, gh-pages branch already up-to-date\"\n    exit 0\nfi\n\n# Create a new commit\ncommit_msg=`echo -e \"Update documentation for $build_subdir\\n\\nAuto-generated from $commit_hash by '$script'\"`\n\necho \"Committing changes...\"\n# Exclude doctrees dir\ngit add -- \":!$build_subdir/.doctrees\"\ngit commit $amend -m \"$commit_msg\"\n\npopd\n\necho \"gh-pages branch successfully updated\"\n"
  },
  {
    "path": "scripts/code-generator/boilerplate.go.txt",
    "content": "// Copyright 2019-2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n"
  },
  {
    "path": "scripts/code-generator/generate-groups.sh",
    "content": "#!/bin/bash\n\n# This is a helper for running the identically named code-generator script from\n# https://github.com/kubernetes/code-generator.\n\nREPO=https://github.com/kubernetes/code-generator\nSCRIPT=\"$(realpath \"$0\")\"\nHEADER=\"${SCRIPT%/*}\"/boilerplate.go.txt\nTOPDIR=${SCRIPT%/scripts/*}\n\nMODDIR=$TOPDIR\nMODURL=$(grep ^module \"$TOPDIR\"/go.mod | sed 's/^module *//g')\nMODULES=${MODULES:-pkg/topology}\n\nfail() {\n    echo \"error: $*\"\n    exit 1\n}\n\n# Parse $* for --output-base, set $gendir and $repo accordingly.\npick-gen-dir() {\n    local _save=\"\" _a\n\n    gendir=$TOPDIR/generate\n    for _a in \"$@\"; do\n        case $_a in\n            --output-base)\n                _save=y;;\n            *)\n                if [ -n \"$_save\" ]; then\n                    gendir=$_a\n                    _save=\"\"\n                fi\n                ;;\n        esac\n    done\n    repo=$gendir/${REPO##*/}\n}\n\n# Set $tag to correspond to $KUBERNETE_VERSION.\npick-git-tag() {\n    if [ -z \"$KUBERNETES_VERSION\" ]; then\n        fail \"KUBERNETES_VERSION not set, please set it to the desired version to match/use.\"\n    fi\n    case $KUBERNETES_VERSION in\n        v1.[0-9.]*) tag=${KUBERNETES_VERSION/#v1/v0};;\n        *)\n            fail \"Don't know how to convert KUBERNETES_VERSION $KUBERNETES_VERSION to tag.\"\n            ;;\n    esac\n}\n\n# Clone $REPO as $repo.\ngit-clone() {\n    if [ ! -d \"$repo\"/.git ]; then\n        mkdir -p \"$gendir\" || fail \"failed to clone git repo\"\n        (cd \"$gendir\" && git clone $REPO) || fail \"failed to clone git repo $REPO\"\n    else\n        (cd \"$repo\" && git fetch -q origin) || fail \"failed to update/fetch git repo $REPO\"\n    fi\n}\n\n# Check out the $tag corresponding to $KUBERNETES_VERSION.\ngit-switch() {\n    (set -e\n     cd \"$repo\"\n         git reset -q --hard HEAD 2> /dev/null\n         git checkout -q \"$tag\"\n    ) || fail \"failed to checkout git tag $tag\"\n}\n\n# Patch $repo/go.mod with replacement rules from $TOPDIR and add replacement rules for $TOPDIR.\ngo-mod-patch() {\n    (set -e\n     cd \"$repo\"\n         grep -A 640 '^replace ' \"$TOPDIR\"/go.mod | grep -v pkg/topology >> go.mod\n         go mod edit -replace=\"$MODURL=$MODDIR\"\n         for mod in $MODULES; do\n             go mod edit -replace=\"$MODURL/$mod=$MODDIR/$mod\"\n         done\n    ) || fail \"failed to patch go.mod\"\n}\n\n# Check any previously generated files for $MODURL, bail out if they exist.\ncheck-existing() {\n    local _pkg=${3%:*} _ver=${3#*:} _dir\n    for _dir in \"$gendir/$1\" \"$gendir/$2/$_pkg/$_ver\"; do\n        if [ -d \"$_dir\" ]; then\n            fail \"$_dir already exists, refusing to overwrite it\"\n        fi\n    done\n}\n\n# Run generate\nrun-generator() {\n    (set -e\n     cd \"$repo\"\n         ./generate-groups.sh \"$@\" --go-header-file \"$HEADER\"\n    ) || fail \"code generation failed\"\n}\n\npick-gen-dir \"$@\"\npick-git-tag\ngit-clone\ngit-switch\ngo-mod-patch\ncheck-existing \"$2\" \"$3\" \"$4\"\nrun-generator \"$@\"\n"
  },
  {
    "path": "scripts/hack/create-webhook-secrets.sh",
    "content": "#!/bin/sh -e\n\nthis=$(realpath \"$0\")\nthis_dir=$(dirname \"$this\")\ntemplate_dir=$(realpath \"$this_dir/../../cmd/cri-resmgr-webhook/\")\noutdir=\"deploy/cri-resmgr-webhook\"\noutdir_abs=\"$(pwd)/$outdir\"\n\ncat << EOF\n***                                 ***\n*** WARNING: NOT FOR PRODUCTION USE ***\n***                                 ***\n\nEOF\n\ninfo () {\n    echo \"[INFO] $1\"\n}\n\ninfo \"Generating x509 keys...\"\n\nmkdir -p \"$outdir\"\n\n# Create temp workdir and remove it on exit\ntmpdir=$(mktemp -d --suffix=.cri-resmgr)\ntrap 'rm -rf $tmpdir' EXIT\n\ncd \"$tmpdir\"\n\n# Create a self-signed CA certificate\nopenssl req -batch -new -newkey rsa:2048 -x509 -sha256 -nodes -days=30 -out ca.crt -keyout ca.key\n\nexport cn=cri-resmgr-webhook.cri-resmgr.svc\nopenssl req -batch -newkey rsa:2048 -nodes -keyout svc.key -out $cn.csr -subj \"/CN=cri-resmgr-webhook.cri-resmgr.svc\"\nopenssl x509 -req -in $cn.csr -CA ca.crt  -CAkey ca.key -CAcreateserial -sha256 -out svc.crt -days 3650\n\n# Copy artifacts to outdir\ncp ca.crt svc.crt svc.key \"$outdir_abs\"\n\ninfo \"Done\"\ninfo \"Sample cert and key files successfully generated under '$outdir'\"\n\ninfo \"Creating MutatingWebhookConfiguration template\"\nsed \"s/CA_BUNDLE_PLACEHOLDER/$(base64 -w0 < ca.crt)/\" \"$template_dir/mutating-webhook-config.yaml\" > \"$outdir_abs/mutating-webhook-config.yaml\"\n\n# Print instructions\ncat << EOF\n\nInstructions for example deployment\n===================================\n0. Create cri-resmgr namespace, if it does not exist:\n   kubectl create ns cri-resmgr\n\n1. Create Kubernetes secrets with:\n   kubectl -n cri-resmgr create secret generic cri-resmgr-webhook-secret \\\\\n    --from-file=$outdir/svc.crt --from-file=$outdir/svc.key\n\n2. Build and publish webhook container:\n   make image-webhook IMAGE_REPO=my-image-repo IMAGE_TAG=my-version\n\n   And deploy it:\n   sed s'!IMAGE_PLACEHOLDER!my-image-repo/cri-resmgr-webhook:my-version!' cmd/webhook/webhook-deployment.yaml | kubectl apply -f -\n\n3. Create MutatingWebhookConfiguration with:\n   kubectl apply -f $outdir/mutating-webhook-config.yaml\n\nEOF\n"
  },
  {
    "path": "scripts/hack/go-mod-replace-helper.sh",
    "content": "#!/bin/bash -e\nset -o pipefail\n\nthis=`basename $0`\n\nusage () {\ncat << EOF\nUSAGE: $this REPO_CACHE_DIR VERSION MODULE...\n\nOPTIONS\n  -h         show this help and exit\n\nEXAMPLES\n  Print replace directives for all k8s.io/* updated to v0.19.4:\n\n    $ sed -n '/replace/,$p' go.mod  | grep k8s.io | awk '{print $1}' | \\\\\n      xargs ./scripts/hack/go-mod-replace-helper.sh ../k8s-cache/ v0.19.4\n\nEOF\n}\n\nupdate_cache() {\n    local module_base=`basename \"$1\"`\n    local module_cache_dir=\"$cache_dir/$module_base\"\n\n\n    if [ ! -e \"$module_cache_dir\" ]; then\n        module_repo=\"https://github.com/kubernetes/$module_base\"\n        echo \"Cloning $module_repo to $module_cache_dir\"\n        git clone -q --depth=1 \"$module_repo\" \"$module_cache_dir\"\n    fi\n\n    echo \"Updating $1 at $module_cache_dir\"\n    cd \"$module_cache_dir\"\n    git fetch -q --tags --depth=1\n    cd ->/dev/null\n}\n\ngomodrev() {\n    local module_base=`basename \"$1\"`\n    local module_cache_dir=\"$cache_dir/$module_base\"\n    cd \"$module_cache_dir\"\n\n    # Resolve to a commit\n    sha=`git rev-parse \"$2\"~0`\n\n    short_sha=`git rev-parse --short=12 $sha`\n\n    unix_ts=`git show $sha --format=%ct --date=unix | head -n1`\n\n    gomod_ts=`date -u --date=@$unix_ts +'%Y%m%d%H%M%S'`\n\n    echo \"v0.0.0-$gomod_ts-$short_sha\"\n\n    cd - >/dev/null\n}\n\nwhile [ \"${1#-}\" != \"$1\" -a -n \"$1\" ]; do\n    case \"$1\" in\n        -h|--help)\n            usage\n            exit 0\n            ;;\n        *)\n            usage\n            exit 1\n            ;;\n    esac\n    shift\ndone\n\nif [ $# -lt 3 ]; then\n    usage\n    exit 1\nfi\n\ncache_dir=\"$1\"\nshift\nmodule_version=\"$1\"\nshift\nmodule_names=\"$@\"\n\ncat << EOF\n\nUPDATING CACHE\n==============\nEOF\nfor m in $@; do\n    update_cache $m\ndone\n\ncat << EOF\n\nGO.MOD REPLACE\n==============\nEOF\n\nfor m in $@; do\n    r=`gomodrev $m $module_version`\n    echo -e \"\\t$m v0.0.0 => $m $r\"\ndone\n"
  },
  {
    "path": "scripts/hack/go-mod-tree",
    "content": "#!/usr/bin/env python3\n\n\"\"\"go-mod-tree - inspect go module import hierarchy\n\nUsage: go mod graph | go-mod-tree [options]\n\nOptions:\n  -h, --help                    print help.\n\n  Input:\n  -g, --graph FILE              read graph from FILE instead of stdin.\n\n  Dependency tree selection: (MODULEs are regular expressions)\n  -r, --reverse                 print reverse tree: from importees to importers.\n  -f, --from MODULE             print tree starting from matching MODULEs.\n  -t, --to MODULE               print tree with only branches that end to\n                                matching MODULEs.\n  -x, --exclude MODULE          exclude matching MODULEs from the graph.\n  -s, --shortest-path MODULE    print only a shortest path to matching MODULEs.\n  -d, --depth DEPTH             limit printed tree to DEPTH.\n\n  Output format:\n  -H <L|D|I|R>                  hide from line format:\n                                  L: line number\n                                  D: depth\n                                  I: indentation\n                                  R: reference to already printed line\n  -I STRING                     indentation by repeating STRING\n\nExamples:\n  - Print full import graph as a tree:\n    go mod graph | go-mod-tree\n\n  - Print which of the direct dependencies lead to importing x/net:\n    go mod graph | go-mod-tree --to golang.org/x/net --depth 1\n\n  - Print modules directly imported by different versions of x/net:\n    go mod graph | go-mod-tree --from golang.org/x/net --depth 1\n\n  - Print modules that directly depend on any version of x/net:\n    go mod graph | go-mod-tree --reverse --from golang.org/x/net --depth 1\n\n  - Print shortest import paths to 2010-2019 versions of x/net:\n    go mod graph | go-mod-tree --shortest-path .*/x/net@.*201[0-9].*\n\n  - Print full reverse import tree from a specific x/net version:\n    go mod graph | go-mod-tree --reverse --from .*20190311183353-d8887717615a\n\"\"\"\n\nimport getopt\nimport re\nimport sys\n\ng_command = \"go-mod-tree\"\n\nopt_fmt = \"%(prefix)s%(indent)s%(node)s %(ref)s\\n\"\nopt_indent = \":   \"\nopt_reverse = False\nopt_graph = \"-\"\nopt_shortest_path = None\nopt_from = None\nopt_to = None\nopt_exclude = None\nopt_depth = float(\"inf\")\nopt_hide = \"\"\n\ndef error(msg, exit_status=1):\n    \"\"\"print error message and exit\"\"\"\n    if msg:\n        sys.stderr.write(\"%s: %s\\n\" % (g_command, msg))\n    if exit_status != None:\n        sys.exit(1)\n\ndef output(msg):\n    try:\n        sys.stdout.write(msg)\n    except:\n        error(\"broken pipe\")\n\ndef read_graph(s):\n    \"\"\"read go mod graph output from a string\"\"\"\n    deps = {} # {importer: set(importee, ...)}\n    for line in s.splitlines():\n        if not line:\n            continue\n        if not \" \" in line:\n            continue\n        importer, importee = line.split(\" \", 1)\n        if not importer in deps:\n            deps[importer] = set()\n        deps[importer].add(importee)\n    return deps\n\ng_lineno = 0\ndef dump_tree(graph, module, depth=0, already_seen={}, max_depth=opt_depth):\n    def dump_line(depth, node):\n        global g_lineno\n        g_lineno += 1\n        if \"D\" not in opt_hide:\n            pp_depth = \"D%d\" % (depth,)\n        else:\n            pp_depth = \"\"\n        if \"L\" not in opt_hide:\n            pp_lineno = \"L%d\" % (g_lineno,)\n        else:\n            pp_lineno = \"\"\n        if \"D\" in opt_hide and \"L\" in opt_hide:\n            pp_lineprefix = \"\"\n        else:\n            pp_lineprefix = \"%-8s\" % ((pp_lineno + pp_depth),)\n        if \"I\" in opt_hide:\n            pp_indent = \"\"\n        else:\n            pp_indent = opt_indent * depth\n        pp_ref = \"\"\n        if node in already_seen and \"R\" not in opt_hide:\n            pp_ref = \" (see L%(line)sD%(depth)s...)\" % already_seen[node]\n        output((opt_fmt % {\n            'prefix': pp_lineprefix,\n            'indent': pp_indent,\n            'node': node,\n            'ref': pp_ref}))\n    if depth > max_depth:\n        return\n    dump_line(depth, module)\n    if module in already_seen:\n        return\n    already_seen[module] = {\"line\": g_lineno, \"depth\": depth}\n    for child in sorted(graph.get(module, set())):\n        dump_tree(graph, child, depth+1, already_seen, max_depth=max_depth)\n\ndef graph_clear(graph):\n    \"\"\"return graph without node keys that have no outgoing edges\"\"\"\n    new_graph = {}\n    for node in graph:\n        if graph[node]:\n            new_graph[node] = set(graph[node])\n    return new_graph\n\ndef graph_exclude(graph, exclude_nodes):\n    \"\"\"return graph without nodes in the exclude_nodes set\"\"\"\n    new_graph = {}\n    for node in graph:\n        if node not in exclude_nodes:\n            new_graph[node] = graph[node] - exclude_nodes\n    return graph_clear(new_graph)\n\ndef graph_reverse(graph):\n    \"\"\"return reversed graph\"\"\"\n    new_graph = {}\n    for from_node, to_nodes in graph.items():\n        for to_node in to_nodes:\n            if not to_node in new_graph:\n                new_graph[to_node] = set()\n            new_graph[to_node].add(from_node)\n    return new_graph\n\ndef graph_reachable_part(graph, from_nodes):\n    \"\"\"return the part of the graph that is reachable from a set of nodes\"\"\"\n    new_graph = {}\n    stack = list(set(graph.keys()).intersection(from_nodes))\n    while stack:\n        node = stack.pop()\n        if node in new_graph:\n            continue\n        new_graph[node] = set()\n        for child in graph.get(node, set()):\n            new_graph[node].add(child)\n            stack.append(child)\n    return graph_clear(new_graph)\n\ndef graph_from_to(graph, from_nodes, to_nodes):\n    \"\"\"return graph between from_nodes and to_nodes\"\"\"\n    new_graph = graph\n    new_graph = graph_reverse(new_graph)\n    new_graph = graph_reachable_part(new_graph, to_nodes)\n    new_graph = graph_reverse(new_graph)\n    new_graph = graph_reachable_part(new_graph, from_nodes)\n    return new_graph\n\ndef shortest_path(graph, from_node, to_node):\n    \"\"\"return new graph that contains only a shorest path between nodes\"\"\"\n    shortest_path = None\n    bfs_queue = [(child, [from_node]) for child in sorted(graph.get(from_node, set()))]\n    seen = set(from_node)\n    while bfs_queue:\n        node, history = bfs_queue.pop(0)\n        seen.add(node)\n        if node == to_node:\n            shortest_path = history + [node]\n            break\n        for child in sorted(graph.get(node, set())):\n            if child in seen:\n                continue\n            bfs_queue.append((child, history + [node]))\n    return shortest_path\n\ndef graph_add_path(graph, path):\n    \"\"\"add a path to current graph\"\"\"\n    for n, node in enumerate(path):\n        if not node in graph:\n            graph[node] = set()\n        if n > 0:\n            graph[path[n-1]].add(node)\n    return graph\n\ndef matching_nodes(graph, node_regexp):\n    matching = set()\n    nodes = set.union(set(graph.keys()), set.union(*graph.values()))\n    for node in nodes:\n        if re.match(node_regexp, node):\n            matching.add(node)\n    return sorted(matching)\n\ndef root_nodes(graph):\n    dest_nodes = set.union(*graph.values())\n    src_nodes = set(graph.keys())\n    roots = src_nodes - dest_nodes\n    return sorted(roots)\n\nif __name__ == \"__main__\":\n    try:\n        opts, remainder = getopt.gnu_getopt(\n            sys.argv[1:],\n            'd:f:g:hrs:t:x:H:I:',\n            ['depth=', 'exclude=', 'from=', 'graph=', 'help', 'reverse',\n             'shortest-path=', 'to='])\n    except getopt.GetoptError as e:\n        error(str(e))\n    for opt, arg in opts:\n        if opt in [\"-h\", \"--help\"]:\n            print(__doc__)\n            error(None, exit_status=0)\n        elif opt in [\"-d\", \"--depth\"]:\n            try:\n                opt_depth = int(arg)\n                if opt_depth <= 0:\n                    raise Exception(\"depth <= 0\")\n            except:\n                error('invalid --depth=%r, positive integer expected', (arg,))\n        elif opt in [\"-f\", \"--from\"]:\n            opt_from = arg\n        elif opt in [\"-g\", \"--graph\"]:\n            opt_graph = arg\n        elif opt in [\"-r\", \"--reverse\"]:\n            opt_reverse = True\n        elif opt in [\"-s\", \"--shortest-path\"]:\n            opt_shortest_path = arg\n        elif opt in [\"-t\", \"--to\"]:\n            opt_to = arg\n        elif opt in [\"-H\"]:\n            opt_hide = arg\n        elif opt in [\"-I\"]:\n            opt_indent = arg\n        elif opt in [\"-x\", \"--exclude\"]:\n            opt_exclude = arg\n        else:\n            error('internal error: option \"%s\" not handled' % (opt,))\n    if len(remainder) > 0:\n        error('too many parameters')\n\n    if opt_graph == \"-\":\n        graph_string = sys.stdin.read()\n    else:\n        try:\n            graph_string = open(opt_graph).read()\n        except Exception as err:\n            error('failed to read graph from file \"%s\": %s' % (opt_graph, err))\n    graph = read_graph(graph_string)\n\n    if opt_exclude:\n        exclude_modules = matching_nodes(graph, opt_exclude)\n        if not exclude_modules:\n            error('no modules matching regular expression --exclude %r' % (opt_exclude,))\n        graph = graph_exclude(graph, set(exclude_modules))\n\n    if opt_reverse:\n        graph = graph_reverse(graph)\n\n    if opt_from:\n        from_modules = matching_nodes(graph, opt_from)\n        if not from_modules:\n            error('no modules matching regular expression --from %r' % (opt_from,))\n    else:\n        from_modules = root_nodes(graph)\n\n    if opt_to:\n        to_modules = matching_nodes(graph, opt_to)\n        if not to_modules:\n            error('no modules matching regular expression --to %r' % (opt_to,))\n        graph = graph_from_to(graph, set(from_modules), set(to_modules))\n        from_modules = set(from_modules).intersection(\n            set.union(set(graph.keys()), set.union(*graph.values())))\n\n    if opt_shortest_path:\n        new_graph = {}\n        to_modules = matching_nodes(graph, opt_shortest_path)\n        if not to_modules:\n            error('no modules matching regular expression --shortest-path %r' % (opt_shortest_path,))\n        for from_node in from_modules:\n            for to_node in to_modules:\n                path = shortest_path(graph, from_node, to_node)\n                if path:\n                    graph_add_path(new_graph, path)\n        graph = new_graph\n\n    for from_node in from_modules:\n        dump_tree(graph, from_node, max_depth=opt_depth)\n"
  },
  {
    "path": "scripts/hack/install-protobuf",
    "content": "#!/usr/bin/env bash\n\n#   Copyright The containerd Authors.\n\n#   Licensed under the Apache License, Version 2.0 (the \"License\");\n#   you may not use this file except in compliance with the License.\n#   You may obtain a copy of the License at\n\n#       http://www.apache.org/licenses/LICENSE-2.0\n\n#   Unless required by applicable law or agreed to in writing, software\n#   distributed under the License is distributed on an \"AS IS\" BASIS,\n#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#   See the License for the specific language governing permissions and\n#   limitations under the License.\n\n\n#\n# Downloads and installs protobuf\n#\nset -eu -o pipefail\n\nPROTOBUF_VERSION=3.20.1\nGOARCH=$(go env GOARCH)\nGOOS=$(go env GOOS)\nPROTOBUF_DIR=$(mktemp -d)\n\ncase $GOARCH in\n\narm64)\n\twget -O \"$PROTOBUF_DIR/protobuf\" \"https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-aarch64.zip\"\n\tunzip \"$PROTOBUF_DIR/protobuf\" -d /usr/local\n\t;;\n\namd64|386)\n\tif [ \"$GOOS\" = windows ]; then\n\t\twget -O \"$PROTOBUF_DIR/protobuf\" \"https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-win32.zip\"\n\telif [ \"$GOOS\" = linux ]; then\n\t\twget -O \"$PROTOBUF_DIR/protobuf\" \"https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-x86_64.zip\"\n\tfi\n\tunzip \"$PROTOBUF_DIR/protobuf\" -d /usr/local\n\t;;\n\nppc64le)\n\twget -O \"$PROTOBUF_DIR/protobuf\" \"https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-ppcle_64.zip\"\n\tunzip \"$PROTOBUF_DIR/protobuf\" -d /usr/local\n\t;;\n*)\n\twget -O \"$PROTOBUF_DIR/protobuf\" \"https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.zip\"\n\tunzip \"$PROTOBUF_DIR/protobuf\" -d /usr/src/protobuf\n\tcd \"/usr/src/protobuf/protobuf-$PROTOBUF_VERSION\"\n\t./autogen.sh\n\t./configure --disable-shared\n\tmake\n\tmake check\n\tmake install\n\tldconfig\n\t;;\nesac\nrm -rf \"$PROTOBUF_DIR\"\n\n# Download status.proto. grpc repos' one seems copied from\n# https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto,\n# but we use grpc's since the repos has tags/releases.\nmkdir -p /usr/local/include/google/rpc\ncurl \\\n\t-L https://raw.githubusercontent.com/grpc/grpc/v1.45.2/src/proto/grpc/status/status.proto \\\n\t-o /usr/local/include/google/rpc/status.proto\n"
  },
  {
    "path": "scripts/testing/crictl",
    "content": "#!/bin/sh\n\n\nRELAY_SOCKET=unix:///var/run/cri-relay.sock\nif [ -z \"$CRICTL\" ]; then\n    CRICTL=crictl\nfi\n\nsudo $CRICTL -i $RELAY_SOCKET -r $RELAY_SOCKET \"$@\"\n"
  },
  {
    "path": "scripts/testing/jaeger",
    "content": "#!/bin/sh\n\nENVVARS=\"-e COLLECTOR_ZIPKIN_HTTP_PORT=9411\"\nPORTS=\"-p 5775:5775/udp \\\n  -p 6831:6831/udp \\\n  -p 6832:6832/udp \\\n  -p 5778:5778 \\\n  -p 16686:16686 \\\n  -p 14268:14268 \\\n  -p 9411:9411\"\n\nif [ \"$1\" = \"--permanent\" ]; then\n    storage=/tmp/jaeger-trace\n    data=$storage/data\n    key=$storage/key\n    echo \"Using $data and $key to store (badger) traces...\"\n    mkdir -p $storage\n    STORAGE=\"-e SPAN_STORAGE_TYPE=badger \\\n      -e BADGER_EPHEMERAL=false \\\n      -e BADGER_DIRECTORY_VALUE=$data \\\n      -e BADGER_DIRECTORY_KEY=$key \\\n      -v $storage:$storage\"\nfi\n\ncmd=\"docker run $ENVVARS $PORTS $STORAGE jaegertracing/all-in-one:latest\"\necho \"Running command $cmd...\"\n$cmd\n"
  },
  {
    "path": "scripts/testing/kube-cgroups",
    "content": "#!/bin/bash\n\nusage() {\n    cat <<EOF\nUsage: kube-cgroups [options]\n\nOptions:\n  -g CGDIR            print cgroup data under CGDIR.\n                      The default is /sys/fs/cgroup.\n  -E                  print also empty files.\n                      The default is to print non-empty files only.\n  -F                  print full cgroup filename.\n                      The default is basename only.\n  Filtering options:\n  -n NS_REGEXP        print only pods in namespaces matching NS_REGEXP\n  -p POD_REGEXP       print only pods matching POD_REGEXP\n  -c CNTR_REGEXP      print only containers matching CNTR_REGEXP\n  -f CGFILE_REGEXP    print only cgroup files matching CGFILE_REGEXP\n\nExamples:\n\n  # print cgroup information of pods in any namespace\n  kube-cgroups -n .\n\n  # print read bps and iops throttling of containers in mypod\n  kube-cgroups -g /sys/fs/cgroup/blkio -p mypod -f read\nEOF\n}\n\nerror() {\n    echo \"kube-cgroups: $*\" >&2\n    exit 1\n}\n\nfull_filename=0\nempty_files=0\nns_regexp=\"default\" # regexp matching\npod_regexp=\".\" # regexp matching any pod name\ncntr_regexp=\".\" # regexp matching any container line\ncgfile_regexp=\"cpuset.cpus|cpuset.mems|blkio.throttle.*_device\" # regexp matching any cgroup file\n\ncg_controller_dir=/sys/fs/cgroup\n\nwhile getopts \"hg:EFn:p:c:f:\" OPTION; do\n    case $OPTION in\n        h)\n            usage\n            exit 0\n            ;;\n        g)\n            cg_controller_dir=\"$OPTARG\"\n            ;;\n        E)\n            empty_files=1\n            ;;\n        F)\n            full_filename=1\n            ;;\n        n)\n            ns_regexp=\"$OPTARG\"\n            ;;\n        p)\n            pod_regexp=\"$OPTARG\"\n            ;;\n        c)\n            cntr_regexp=\"$OPTARG\"\n            ;;\n        f)\n            cgfile_regexp=\"$OPTARG\"\n            ;;\n        *)\n            error \"invalid option $OPTION\"\n            ;;\n    esac\ndone\n\nif [ ! -d \"$cg_controller_dir\" ]; then\n    error \"cgroup directory '$cg_controller_dir' does not exist\"\nfi\n\nkubectl get pods -A | grep -E \"$pod_regexp\" | while read -r namespace podname rest; do\n\n    [ \"$namespace\" == \"NAMESPACE\" ] && continue\n\n    grep -q -E \"$ns_regexp\" <<< \"$namespace\" || continue\n\n    kubectl describe pod -n \"$namespace\" \"$podname\" | grep -B1 'Container ID:' | while read -r container _ containerid; do\n\n        if [[ \"$container\" != \"Container\" ]] && [[ \"$container\" != \"--\" ]]; then\n            containername=\"${container%%:*}\"\n            continue\n        fi\n\n        containerID=${containerid#*://}\n\n        if [[ -z \"$containerID\" ]]; then\n            continue\n        fi\n\n        grep -q -E \"$cntr_regexp\" <<< \"$containername\" || continue\n\n        while read -r cgroupdir; do\n            if [[ \"$cgroupdir\" == *crio-conmon* ]]; then\n                continue\n            fi\n            for filename in \"$cgroupdir\"/*; do\n                if [[ ! -f \"$filename\" ]]; then\n                    continue\n                fi\n                filename_nodir=\"${filename##*/}\"\n                grep -q -E \"$cgfile_regexp\" <<< \"$filename_nodir\" || continue\n                if [[ -n \"$podname\" ]]; then\n                    echo \"$namespace/$podname:\"\n                    unset podname\n                fi\n                [[ -n \"$containername\" ]] && {\n                    echo \"  $containername:\"\n                    unset containername\n                }\n                linecount=\"$(wc -l < \"$filename\")\"\n                if [[ \"$linecount\" == \"0\" ]] && [[ \"$empty_files\" == \"0\" ]]; then\n                    continue\n                fi\n                if [[ \"$full_filename\" == \"1\" ]]; then\n                    print_filename=\"$filename\"\n                else\n                    print_filename=\"$filename_nodir\"\n                fi\n                if (( \"$linecount\" <= 1 )); then\n                    # print contents of a single-line file after filename\n                    echo \"    $print_filename: $(< \"$filename\")\"\n                else\n                    # print contents of a multiline file indented\n                    echo \"    $print_filename:\"\n                    sed \"s/^/      /g\" < \"$filename\"\n                fi\n            done\n        done <<< \"$(find \"$cg_controller_dir\" -name \"*${containerID}*\")\"\n    done\ndone\n"
  },
  {
    "path": "scripts/testing/pairwise",
    "content": "#!/usr/bin/env python3\n\n\"\"\"pairwise - print var-value combinations that cover all value pairs\n\nUsage: pairwise VAR=VALUE [VAR=VALUE...]\n\nExample:\n$ pairwise \\\\\n    distro={debian-sid,opensuse,fedora} \\\\\n    k8scni={cilium,weavenet,flannel} \\\\\n    k8scri={crio,containerd} \\\\\n    k8s={1.22.0,1.23.0}\n\"\"\"\n\nimport sys\n\ndef error(msg, exit_status=1):\n    sys.stderr.write('pairwise: %s\\n' % (msg,))\n    if exit_status is not None:\n        sys.exit(exit_status)\n\ndef output(msg):\n    sys.stdout.write(msg)\n\n# This program prints an optimized set of value combinations\n# that covers all value pairs.\n\ndef all_combinations(var_values):\n    combinations = [{}]\n    for var in var_values:\n        new_combinations = []\n        for d in combinations:\n            for value in var_values[var]:\n                new_comb = dict(d)\n                new_comb[var] = value\n                new_combinations.append(new_comb)\n        combinations = new_combinations\n    return combinations\n\ndef combination_to_triplets(d):\n    triplets = set()\n    keys = sorted(d.keys())\n    for key1_index, key1 in enumerate(keys):\n        val1 = d[key1]\n        for key2_index, key2 in enumerate(keys[key1_index+1:]):\n            val2 = d[key2]\n            for key3 in keys[key1_index + key2_index + 2:]:\n                val3 = d[key3]\n                triplets.add(frozenset(((key1, val1), (key2, val2), (key3, val3))))\n    return triplets\n\ndef combination_to_pairs(d):\n    pairs = set()\n    keys = sorted(d.keys())\n    for key1_index, key1 in enumerate(keys):\n        val1 = d[key1]\n        for key2 in keys[key1_index+1:]:\n            val2 = d[key2]\n            pairs.add(frozenset(((key1, val1), (key2, val2))))\n    return pairs\n\ndef combination_to_singles(d):\n    singles = set()\n    for key1 in d.keys():\n        val1 = d[key1]\n        singles.add(frozenset((key1, val1)))\n    return singles\n\ndef cover_pairwise(var_values):\n    chosen_combinations = []\n    covered_pairs = set()\n    combination_pairs = {}\n    all_triplets = set()\n    all_pairs = set()\n    all_singles = set()\n    combinations = all_combinations(var_values)\n    for c in combinations:\n        all_triplets = all_triplets.union(combination_to_triplets(c))\n        all_pairs = all_pairs.union(combination_to_pairs(c))\n        all_singles = all_singles.union(combination_to_singles(c))\n    uncovered_triplets = set(all_triplets)\n    number_of_triplets = len(uncovered_triplets)\n    uncovered_pairs = set(all_pairs)\n    uncovered_singles = set(all_singles)\n    while uncovered_pairs:\n        combination_score = []\n        for c in combinations:\n            covers_triplets = combination_to_triplets(c)\n            covers_pairs = combination_to_pairs(c)\n            covers_singles = combination_to_singles(c)\n            combination_score.append(\n                (len(uncovered_pairs.intersection(covers_pairs)) +\n                 len(uncovered_singles.intersection(covers_singles)) +\n                 len(uncovered_triplets.intersection(covers_triplets)) / number_of_triplets,\n                 c, covers_pairs, covers_singles, covers_triplets))\n        best_score, best_comb, best_pairs, best_singles, best_triplets = sorted(combination_score, key=lambda comb_score: comb_score[0])[-1]\n        chosen_combinations.append(best_comb)\n        uncovered_triplets = uncovered_triplets - best_triplets\n        uncovered_pairs = uncovered_pairs - best_pairs\n        uncovered_singles = uncovered_singles - best_singles\n    return chosen_combinations\n\nif __name__ == \"__main__\":\n    if len(sys.argv) < 2 or \"-h\" in sys.argv or \"--help\" in sys.argv:\n        output(__doc__)\n        error('missing VAR=VALUE...', exit_status=0)\n    # construct var_values from command line arguments\n    var_values = {} # {var: list-of-values}\n    for var_value in sys.argv[1:]:\n        try:\n            var, value = var_value.split(\"=\", 1)\n        except:\n            error('bad argument %r, VAR=VALUE expected', var_value)\n        if var not in var_values:\n            var_values[var] = []\n        var_values[var].append(value)\n\n    for comb in cover_pairwise(var_values):\n        var_value_row = []\n        for var in sorted(comb.keys()):\n            var_value_row.append('%s=\"%s\"' % (var, comb[var]))\n        output(\" \".join(var_value_row) + \"\\n\")\n"
  },
  {
    "path": "scripts/testing/prometheus",
    "content": "#!/bin/sh\n\ndir=$(dirname \"$0\")\ncfg=$dir/prometheus.yaml\n\ncmd=\"docker run -p 9090:9090 \\\n    -v $cfg:/etc/prometheus/prometheus.yml \\\n    prom/prometheus --config.file=/etc/prometheus/prometheus.yml $*\"\n\necho \"Running command $cmd...\"\n$cmd\n"
  },
  {
    "path": "scripts/testing/prometheus.yaml",
    "content": "global:\n  scrape_interval: 10s\n  external_labels:\n    monitor: 'CRI-RM'\n\nscrape_configs:\n  - job_name: 'CRI-RM'\n    scrape_interval: 10s\n    static_configs:\n      - targets: ['10.0.0.2:8888']\n"
  },
  {
    "path": "scripts/testing/set-path",
    "content": "#!/bin/sh\n\n# set -x\n\ndirpart=packages/src/github.com/intel/cri-interceptor\n\ncase $(pwd) in\n    */$dirpart*)\n        ;;\n    *)\n        echo \"Don't know how: I don't see $dirpart in $(pwd)...\"\n        return 1\n        ;;\nesac\n\ndir=$(pwd)\nkubedir=${dir%%/github.com*}/k8s.io/kubernetes\nkubebin=$kubedir/_output/local/bin/linux/amd64\n\nif [ ! -d \"$kubebin\" ]; then\n    echo \"*** You don't seem to have a $kubebin directory.\"\n    return 1\nfi\n\nif [ ! -x \"$kubebin\"/kubelet ]; then\n    echo \"*** You don't seem to have kubelet in $kubebin (done a make WHAT=cmd/kubelet ?)\"\n    ls -ls \"$kubebin\"\n    return 1\nfi\n\nexport PATH=\"$kubebin:$PATH\"\n"
  },
  {
    "path": "test/critest/run.sh",
    "content": "#!/bin/bash\n\nTEST_TITLE=\"CRI validation tests with critest\"\n\nPV='pv -qL'\n\nSCRIPT_DIR=\"$(dirname \"${BASH_SOURCE[0]}\")\"\nDEMO_LIB_DIR=$(realpath \"$SCRIPT_DIR/../../demo/lib\")\nBIN_DIR=$(realpath \"$SCRIPT_DIR/../../bin\")\nOUTPUT_DIR=${outdir-$SCRIPT_DIR/output}\nCOMMAND_OUTPUT_DIR=$OUTPUT_DIR/commands\n\n# shellcheck disable=SC1091\n# shellcheck source=../../demo/lib/command.bash\nsource \"$DEMO_LIB_DIR\"/command.bash\n# shellcheck disable=SC1091\n# shellcheck source=../../demo/lib/host.bash\nsource \"$DEMO_LIB_DIR\"/host.bash\n# shellcheck disable=SC1091\n# shellcheck source=../../demo/lib/vm.bash\nsource \"$DEMO_LIB_DIR\"/vm.bash\n\nusage() {\n    echo \"$TEST_TITLE\"\n    echo \"Usage: [VAR=VALUE] ./run.sh MODE\"\n    echo \"  MODE:     \\\"play\\\" plays the test as a demo.\"\n    echo \"            \\\"test\\\" runs fast, reports pass or fail.\"\n    echo \"  VARs:\"\n    echo \"    tests:   space-separated list of cri-resmgr configurations.\"\n    echo \"             The default is all *.cfg files in $SCRIPT_DIR.\"\n    echo \"    vm:      govm virtual machine name.\"\n    echo \"             The default is \\\"crirm-test-critest\\\".\"\n    echo \"    speed:   Demo play speed.\"\n    echo \"             The default is 10 (keypresses per second).\"\n    echo \"    cleanup: Level of cleanup after a test run:\"\n    echo \"             0: leave VM running (the default)\"\n    echo \"             1: delete VM\"\n    echo \"             2: stop VM, but do not delete it.\"\n    echo \"    outdir:  Save output under given directory.\"\n    echo \"             The default is \\\"${SCRIPT_DIR}/output\\\".\"\n}\n\nerror() {\n    (echo \"\"; echo \"error: $1\" ) >&2\n    exit 1\n}\n\nout() {\n    if [ -n \"$PV\" ]; then\n        speed=${speed-10}\n        echo \"$1\" | $PV \"$speed\"\n    else\n        echo \"$1\"\n    fi\n    echo \"\"\n}\n\nscreen-create-vm() {\n    speed=60 out \"### Running the test in VM \\\"$vm\\\".\"\n    host-create-vm \"$vm\" \"$topology\"\n    if [ -z \"$VM_IP\" ]; then\n        error \"creating VM failed\"\n    fi\n    vm-networking\n}\n\nscreen-install-containerd() {\n    speed=60 out \"### Installing Containerd to the VM.\"\n    vm-install-cri\n    vm-install-containernetworking\n}\n\nscreen-copy-cri-resmgr() {\n    prefix=/usr/local\n    host-command \"scp \\\"$BIN_DIR/cri-resmgr\\\" \\\"$SCRIPT_DIR/tsl\\\" $VM_SSH_USER@$VM_IP:\" || {\n        command-error \"copying cri-resmgr failed\"\n    }\n    vm-command \"mv cri-resmgr tsl $prefix/bin/\" || {\n        command-error \"installing cri-resmgr to $prefix/bin failed\"\n    }\n    PV=\"\" vm-command \"command -v cri-resmgr\" >/dev/null\n    ( echo \"$COMMAND_OUTPUT\" | grep -q $prefix/bin/cri-resmgr ) || {\n        command-error \"\\\"cri-resmgr\\\" does not execute $prefix/bin/cri-resmgr on VM\"\n    }\n\n}\n\nscreen-install-critest() {\n    speed=60 out \"### Installing critest to VM.\"\n    vm-command \"apt update && apt install -y golang make socat\"\n    vm-command \"go get -d github.com/kubernetes-sigs/cri-tools\"\n    CRI_TOOLS_SOURCE_DIR=$(awk '/package.*cri-tools/{print $NF}' <<< \"$COMMAND_OUTPUT\")\n    [ -n \"$CRI_TOOLS_SOURCE_DIR\" ] || {\n        command-error \"downloading cri-tools failed\"\n    }\n    vm-command \"pushd \\\"$CRI_TOOLS_SOURCE_DIR\\\" && make && make install && popd\" || {\n        command-error \"building and installing cri-tools failed\"\n    }\n}\n\nscreen-critest-crirm-config() {\n    config_file=$1\n    cri_endpoint=/var/run/containerd/containerd.sock\n    cri_resmgr_endpoint=/var/run/cri-resmgr/cri-resmgr.sock\n    host-command \"scp $config_file $VM_SSH_USER@$VM_IP:\"\n    vm-command \"rm -rf *.tsl; killall cri-resmgr; systemctl stop containerd; sleep 1; systemctl start containerd; sleep 1; rm -rf /var/lib/cri-resmgr\"\n    vm-command \"cri-resmgr -force-config $config_file -runtime-socket $cri_endpoint -relay-socket $cri_resmgr_endpoint 2>&1 | tsl -uU -F \\\"%(ts) s cri-resmgr: %(line)s\\\" -o cri-resmgr.output.tsl\" bg\n    sleep 5\n    vm-command \"critest -runtime-endpoint unix://$cri_resmgr_endpoint 2>&1 | tsl -uU -F \\\"%(ts) s critest: %(line)s\\\" -o critest.output.tsl\"\n    vm-command \"killall cri-resmgr\"\n    vm-command-q \"cat *.tsl | sort -n | awk '{if (t_start==0) t_start=\\$1; \\$1=sprintf(\\\"%.6fs\\\", \\$1-t_start); print;}'\" > \"$OUTPUT_DIR/test-$config_file.log\"\n}\n\nscreen-critest-containerd() {\n    cri_endpoint=/var/run/containerd/containerd.sock\n    vm-command \"rm -rf *.tsl; critest -runtime-endpoint unix://$cri_endpoint 2>&1 | tsl -uU -F \\\"%(ts) s critest: %(line)s\\\" -o critest.output.tsl\"\n    vm-command-q \"cat *.tsl | sort -n | awk '{if (t_start==0) t_start=\\$1; \\$1=sprintf(\\\"%.6fs\\\", \\$1-t_start); print;}'\" > \"$OUTPUT_DIR/test-containerd.log\"\n}\n\nrequire_cmd() {\n    cmd=$1\n    if ! command -v \"$cmd\" >/dev/null ; then\n        error \"required command missing \\\"${cmd}\\\", make sure it is in PATH\"\n    fi\n}\n\n# Validate parameters\nmode=$1\ntopology=${topology:='[{\"cores\": 2, \"mem\": \"8G\"}]'}\ndistro=${distro:=\"ubuntu-20.04\"}\ncri=${cri:=\"containerd\"}\nvm=${vm:=\"critest-$distro-$cri\"}\ncleanup=${cleanup-0}\nhost-set-vm-config \"$vm\" \"$distro\" \"$cri\"\n\ncd \"${SCRIPT_DIR}\" || error \"failed to cd to \\\"${SCRIPT_DIR}\\\"\"\ntests=${tests-*.cfg}\n\nif [ \"$mode\" == \"test\" ]; then\n    PV=\nelif [ \"$mode\" == \"play\" ] ; then\n    speed=${speed-10}\nelse\n    usage\n    error \"invalid MODE\"\nfi\n\n# Prepare for test/demo\nmkdir -p \"$OUTPUT_DIR\"\nmkdir -p \"$COMMAND_OUTPUT_DIR\"\nrm -f \"$COMMAND_OUTPUT_DIR\"/0*\n( echo x > \"$OUTPUT_DIR/x\" && rm -f \"$OUTPUT_DIR/x\" ) || {\n    error \"output directory outdir=$OUTPUT_DIR is not writable\"\n}\n\nif [ -z \"$VM_IP\" ] || [ -z \"$VM_SSH_USER\" ] || [ -z \"$VM_NAME\" ]; then\n    screen-create-vm\nfi\n\n# always copy new version of the binary to VM\nscreen-copy-cri-resmgr\n\nif ! vm-command-q \"dpkg -l | grep -q containerd\"; then\n    screen-install-containerd\nfi\n\nif ! vm-command-q \"command -v critest | grep -q critest\"; then\n    screen-install-critest\nfi\n\n# Run test/demo\n# 1. Run critest on cri-resmgr with each config file.\nfor config_file in $tests; do\n    screen-critest-crirm-config \"$config_file\"\ndone\n# 2. Run critest without cri-resmgr for reference.\nscreen-critest-containerd\n\n# Cleanup\nif [ \"$cleanup\" == \"0\" ]; then\n    echo \"The VM with critest, cri-resmgr and containerd is left running. Next steps:\"\n    vm-print-usage\nelif [ \"$cleanup\" == \"1\" ]; then\n    host-stop-vm $vm\n    host-delete-vm $vm\nelif [ \"$cleanup\" == \"2\" ]; then\n    host-stop-vm $vm\nfi\n\n# Summarize results\nSUMMARY_FILE=\"$OUTPUT_DIR/summary.txt\"\necho -n \"\" > \"$SUMMARY_FILE\" || error \"cannot write summary to \\\"$SUMMARY_FILE\\\"\"\nfor testlog in \"$OUTPUT_DIR\"/test-*.log; do\n    {\n        echo -n \"$(basename \"$testlog\") \"\n        awk 'BEGIN{s=0;e=0}/critest: /{if(s==0)s=$1;e=$1}END{printf \"(runtime %.2f s): \",e-s}' < \"$testlog\"\n        # remove ansi colors from critest output in the summary\n        grep Pending \"$testlog\" | grep critest: | tail -n 1 | sed -r -e \"s/[[:cntrl:]]\\[[0-9]+m//g\" -e \"s/^.* -- //g\"\n    } >> \"$SUMMARY_FILE\"\ndone\nexit_status=0\n# Declare verdict in test mode\nif [ \"$mode\" == \"test\" ]; then\n    echo \"\" >> \"$SUMMARY_FILE\"\n    # Test is passed if all critest executions had the same passrate,\n    # no matter which cri-resmgr configuration was used.\n    if [ \"$(awk -F: '/Passed/{print $2}' < \"$SUMMARY_FILE\" | sort -u | wc -l)\" == \"1\" ]; then\n        echo \"All critest results are the same.\" >> \"$SUMMARY_FILE\"\n        echo \"Test verdict: PASS\" >> \"$SUMMARY_FILE\"\n    else\n        echo \"Error: critest results are not the same in all configurations.\" >> \"$SUMMARY_FILE\"\n        echo \"Test verdict: FAIL\" >> \"$SUMMARY_FILE\"\n        exit_status=1\n    fi\nfi\necho \"\"\necho \"Summary:\"\ncat \"$SUMMARY_FILE\"\nexit \"$exit_status\"\n"
  },
  {
    "path": "test/critest/topology-aware-policy.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,dump,instrumentation,policy\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*\ninstrumentation:\n  Sampling: disabled\n"
  },
  {
    "path": "test/critest/tsl",
    "content": "#!/usr/bin/python3\n#\n# Copyright 2020 Intel Corporation. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"tsl - timestamp lines\n\nUsage: tsl [options]\n\nOptions:\n  -h, --help      print help.\n  -f TIMEFORMAT   use TIMEFORMAT as output timeformat (man strftime).\n                  The default format is \"%s.%f\".\n  -F LINEFORMAT   use LINEFORMAT as line output format:\n                  - %(ts)s: timestamp\n                  - %(line)s: original line\n                  The default is \"%(ts)s %(line)s\".\n  -o OUTFILE      write output lines to OUTFILE. Supports many -o's.\n                  Special outfiles:\n                  - stdout: standard output\n                  - stderr: standard error\n  -u              unbuffered input: more accurate timestamps, slower throughput.\n  -U              unbuffered output: flush after every line, slower throughput.\n\nExamples:\n  cmd1 | tsl -u -F \"%(ts)s cmd1: %(line)s\" > cmd1.tsl &\n  cmd2 | tsl -u -F \"%(ts)s cmd2: %(line)s\" > cmd2.tsl &\n  wait\n  cat cmd1.tsl cmd2.tsl | sort -n > cmd1_cmd2.output\n\"\"\"\n\nimport getopt\nimport sys\nimport datetime\n\ndef unbuffered_xreadlines(fileobj):\n    \"\"\"like fileobj.xreadlines() but unbuffered\"\"\"\n    ln = []\n    while True:\n        c = fileobj.read(1)\n        if not c:\n            if ln:\n                yield \"\".join(ln)\n            break\n        ln.append(c)\n        if c == \"\\n\":\n            yield \"\".join(ln)\n            ln = []\n\nif __name__ == \"__main__\":\n    opt_timeformat = \"%s.%f\" #\"%Y-%m-%d %H:%M:%S\"\n    opt_lineformat = \"%(ts)s %(line)s\"\n    opt_unbuffered_in = False\n    opt_unbuffered_out = False\n    opt_outfiles = []\n    opts, remainder = getopt.gnu_getopt(\n        sys.argv[1:], 'hf:F:o:uU',\n        ['help', 'format='])\n    for opt, arg in opts:\n        if opt in [\"-h\", \"--help\"]:\n            print(__doc__)\n            sys.exit(0)\n        elif opt in [\"-f\", \"--format\"]:\n            opt_timeformat = arg\n        elif opt in [\"-F\"]:\n            opt_lineformat = arg\n        elif opt in [\"-o\"]:\n            if arg == \"stdout\":\n                opt_outfiles.append(sys.stdout)\n            elif arg == \"stderr\":\n                opt_outfiles.append(sys.stderr)\n            else:\n                opt_outfiles.append(open(arg, \"w\"))\n        elif opt in [\"-u\"]:\n            opt_unbuffered_in = True\n        elif opt in [\"-U\"]:\n            opt_unbuffered_out = True\n    if not opt_outfiles:\n        opt_outfiles.append(sys.stdout)\n    if opt_unbuffered_in:\n        line_iter = unbuffered_xreadlines(sys.stdin)\n    else:\n        line_iter = sys.stdin\n    for line in line_iter:\n        ts = datetime.datetime.now().strftime(opt_timeformat)\n        out_line = opt_lineformat % {'ts': ts, 'line': line}\n        for outfile in opt_outfiles:\n            outfile.write(out_line)\n            if opt_unbuffered_out:\n                outfile.flush()\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/memtier-benchmark-02.yaml.in",
    "content": "apiVersion: batch/v1\nkind: Job\nmetadata:\n  name: memtier-benchmark\nspec:\n  template:\n    metadata:\n      annotations:\n        cri-resource-manager.intel.com/${AFFINITY}: |+\n          memtier-benchmark:\n            - scope:\n                key: pod/name\n                operator: Matches\n                values:\n                  - redis-*\n              match:\n                key: name\n                operator: Equals\n                values:\n                  - redis\n              weight: 10\n    spec:\n      containers:\n      $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n      - name: ${NAME}c$(( contnum - 1 ))\n        image: redislabs/memtier_benchmark:edge\n        imagePullPolicy: IfNotPresent\n        args: ['${ARGS// /\\', \\'}']\n        resources:\n          requests:\n            cpu: ${CPU}\n            memory: '${MEM}'\n          limits:\n            cpu: ${CPULIM}\n            memory: '${MEMLIM}'\n      \"; done )\n      restartPolicy: Never\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/memtier-benchmark.yaml.in",
    "content": "apiVersion: batch/v1\nkind: Job\nmetadata:\n  name: memtier-benchmark\nspec:\n  template:\n    metadata:\n      annotations:\n        cri-resource-manager.intel.com/${AFFINITY}: |+\n          memtier-benchmark:\n            - scope:\n                key: pod/name\n                operator: Matches\n                values:\n                  - redis-*\n              match:\n                key: name\n                operator: Equals\n                values:\n                  - redis\n              weight: 10\n    spec:\n      containers:\n        - name: memtier-benchmark\n          image: redislabs/memtier_benchmark:edge\n          imagePullPolicy: IfNotPresent\n          args: ['${ARGS// /\\', \\'}']\n          $(if [ \"$CPU\" != \"0\" ]; then echo \"\n          resources:\n            requests:\n              cpu: ${CPU}\n              memory: '${MEM}'\n            limits:\n              cpu: ${CPULIM}\n              memory: '${MEMLIM}'\n          \"; fi)\n      restartPolicy: Never\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/n4c16/test01-memtier-stress-ng/code.var.sh",
    "content": "# Redis parameters\nREDIS_PASS=abc123xyz\n\n# Background load parameters\nSTRESS_NG_CPUS=16 # workers per container\nSTRESS_NG_CONTS=8 # number of containers per pod\nSTRESS_NG_PODS=2 # number of pods\n\n# BG_* are background loads\n# CPU turbo licence level 2 (causes big drop on GHz) cannot be reached with stress-ng, but could be implemented with\n# 1. [\"avx-turbo\", \"--test=avx512_vlzcnt_t\", \"--min-threads=1\", \"--max-threads=1\", \"--iters=0\"]\n# 2. [\"avx-turbo\", \"--test=avx512_vlzcnt_t\", \"--min-threads=1\", \"--max-threads=1\", \"--iters=0\"]\n# License level observed with:\n# sudo perf stat --pid $(pidof avx-turbo) -e core_power.lvl0_turbo_license,core_power.lvl1_turbo_license,core_power.lvl2_turbo_license -- sleep 1\n# In the following: \"IPC\" == Instructions Per Cycle\nBG_NOLOAD=\"\"\n# BG_AVX_LL0=\"stress-ng --ipsec-mb $STRESS_NG_CPUS --ipsec-mb-feature avx512\" # AVX, causing CPU turbo license level 0\n# BG_AVX_LL1=\"stress-ng --ipsec-mb $STRESS_NG_CPUS --ipsec-mb-feature avx512\" # AVX, causing CPU turbo license level 1\nBG_SHM=\"stress-ng --shm $STRESS_NG_CPUS\" # shared memory, memory bound (not causing 100% CPU load), IPC ~0.01-0.19\nBG_MEMCPY=\"stress-ng --memcpy $STRESS_NG_CPUS\" # memory bound; IPC =~ 0.15\nBG_STREAM=\"stress-ng --stream $STRESS_NG_CPUS\" # IPC =~ 0.49\nBG_CPUJMP=\"stress-ng --cpu $STRESS_NG_CPUS --cpu-method jmp\" # IPC ~3.6\nBG_CPUALL=\"stress-ng --cpu $STRESS_NG_CPUS\" # IPC ~1.8\n\n# BM_* are benchmarks\nbm_stress_ng_iters=10000\nBM_MEMTIER=\"memtier-benchmark --server=redis-service --authenticate=$REDIS_PASS\" # this is special case\n# BM_MEMCPY=\"stress-ng --memcpy 1 --memcpy-ops $bm_stress_ng_iters\" # IPC ~0.15\n# BM_STREAM=\"stress-ng --stream 1 --stream-ops $bm_stress_ng_iters\" # IPC ~0.49\n# BM_JMP=\"stress-ng --cpu 1 --cpu-method jmp --cpu-ops $bm_stress_ng_iters\" # IPC ~3.6\n# BM_FFT=\"stress-ng --cpu 1 --cpu-method fft --cpu-ops $bm_stress_ng_iters\" # IPC ~2.3\n# BM_AVX_LL0=\"stress-ng --ipsec-mb 1 --ipsec-mb-feature avx2 --ipsec-mb-ops $bm_stress_ng_iters\"\n# BM_AVX_LL2=\"stress-ng --ipsec-mb 1 --ipsec-mb-feature avx512 --ipsec-mb-ops $bm_stress_ng_iters\"\n\n# Clean up\nvm-command \"kubectl delete jobs --all --now; kubectl delete deployment redis; kubectl delete service redis-service; kubectl delete secret redis; kubectl delete pods --all --now; true\"\n\n# Setup Redis\nwait=\"\" create redis-secret\nCPU=4 MEM=32G CPULIM=8 MEMLIM=64G NAME=redis wait=\"Available\" create redis\nNAME=redis-service wait=\"\" create redis-service\n\nfor bg_cmd in \"${!BG_@}\"; do\n    # Reset counters in order to keep creating pod0...\n    reset counters\n\n    benchmark_output_dir=\"$OUTPUT_DIR/benchmark/$bg_cmd\"\n    mkdir -p \"$benchmark_output_dir\"\n\n    # Start background noise\n    if [[ \"${!bg_cmd}\" == \"stress-ng \"* ]]; then\n        n=\"$STRESS_NG_PODS\" ARGS=\"${!bg_cmd#stress-ng }\" CONTCOUNT=\"$STRESS_NG_CONTS\" CPU=50m MEM=50M CPULIM=$STRESS_NG_CPUS MEMLIM=1G wait_t=240s create stress-ng\n        # Stabilize\n        ( vm-run-until --timeout 60 \"sh -c 'uptime; exit 1'\" ) || echo \"expected timeout\"\n    fi\n\n    for bm_cmd in \"${!BM_@}\"; do\n        for CPU in 4; do\n            # Run benchmark\n            if [[ \"${!bm_cmd}\" == \"memtier-benchmark \"* ]]; then\n                AFFINITY=affinity CPU=\"$CPU\" MEM=\"16G\" CPULIM=\"$CPU\" MEMLIM=\"24G\" NAME=memtier-benchmark ARGS=\"${!bm_cmd#memtier-benchmark }\" wait=\"Complete\" wait_t=\"10m\" create memtier-benchmark\n                memtier_benchmark_pod=\"$(kubectl get pods | awk '/memtier-benchmark-/{print $1}')\"\n                kubectl logs \"$memtier_benchmark_pod\" | grep -A7 'ALL STATS' | tee \"$benchmark_output_dir/$bm_cmd-affinity-cpu-$CPU.txt\"\n                kubectl delete jobs --all --now\n\n                # AFFINITY=anti-affinity CPU=\"$CPU\" MEM=\"16G\" NAME=memtier-benchmark ARGS=\"${!bm_cmd#memtier-benchmark }\" wait=\"Complete\" wait_t=\"10m\" create memtier-benchmark\n                # memtier_benchmark_pod=\"$(kubectl get pods | awk '/memtier-benchmark-/{print $1}')\"\n                # kubectl logs \"$memtier_benchmark_pod\" | grep -A7 'ALL STATS' | tee \"$benchmark_output_dir/$bm_cmd-antiaffinity-cpu-$CPU.txt\"\n                # kubectl delete jobs --all --now\n\n            elif [[ \"${!bm_cmd}\" == \"stress-ng \"* ]]; then\n                CPU=\"$CPU\" MEM=\"200M\" CPULIM=\"$STRESS_NG_CPUS\" MEM=\"400M\" NAME=stress-ng-benchmark ARGS=\"${!bm_cmd#stress-ng }\" wait=\"Complete\" wait_t=\"10m\" create stress-ng-benchmark\n                stress_ng_benchmark_pod=\"$(kubectl get pods | awk '/stress-ng-benchmark-/{print $1}')\"\n                kubectl logs \"$stress_ng_benchmark_pod\" | tee \"$benchmark_output_dir/$bm_cmd-cpu-$CPU.txt\"\n                kubectl delete jobs --all --now\n            fi\n        done\n    done\n\n    # Stop background noise\n    ( kubectl delete pods -l e2erole=bgload --now )\ndone\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/n4c16/test01-memtier-stress-ng/post-process.sh",
    "content": "#!/bin/bash\n\n# Usage: VAR=VALUE post-process.sh output-CRICONFIGNAME1 output-CRICONFIGNAME2...\n# VARs:\n#   normalize=1..  # normalizes plotted values so that the smallest is 1.00\n#   normalize=0..1 # normalizes plotted to values between 0.0 and 1.0\n#                  # if normalize=\"\", values are not normalized\n#   maxy=MAXY      # maximum value on the Y axis\n#   ytrans=log2    # logarithmic Y axis, the default ytrans is 'identity'\n#   save=PREFIX    # create PREFIX.svg and PREFIX.csv. The default is 'plot'.\n\nnormalize=\"${normalize:-}\"\nmaxy=\"${maxy:-}\"\nytrans=\"${ytrans:-identity}\"\nsave=\"${save:-plot}\"\n\n(\n    for out_path in \"$@\"; do (\n        benchmark_dir=$out_path/benchmark\n        out_dir=\"$(basename \"$out_path\")\"\n        cd \"$benchmark_dir\" || exit\n        for bgload in *; do (\n            cd \"$bgload\" || exit\n            for memtier_results in BM_MEMTIER-*; do\n                p50latency=\\$6\n                p99latency=\\$7\n                p999latency=\\$8\n                awk \"/Totals/{print \\\"$out_dir $bgload $memtier_results \\\"$p50latency\\\" \\\"$p99latency\\\" \\\"$p999latency}\" < \"$memtier_results\"\n            done\n        ); done\n    ); done\n) > total-latencies.txt\n\nsed -e 's/output-//g' -e 's/BG_//g' -e 's/BM_MEMTIER-//g' -e 's/-cpu-[0-9]*.txt//g' < total-latencies.txt | awk '{print $1\" \"$2\" \"$3\" \"$4\" \"$5\" \"$6}' | grep -v ' antiaffinity' > data.txt\n\ncat > plot.R <<EOF\nlibrary(ggplot2)\nlibrary(svglite)\nd <- read.table(file=\"data.txt\")\nnames(d) <- c(\"cri_conf\", \"bg_load\", \"annotations\", \"p5\", \"p99\", \"p999\")\n\nif (\"$normalize\" == \"1..\") {\n    smallest_value = min(d[c(\"p5\", \"p99\", \"p999\")])\n    d[c(\"p5\", \"p99\", \"p999\")] = d[c(\"p5\", \"p99\", \"p999\")] / smallest_value\n    cat(\"normalized to 1.., the smallest value was\", smallest_value, \"\\n\")\n    latency_unit = \"the fastest is 1.0\"\n} else if (\"$normalize\" == \"0..1\") {\n    smallest_value = min(d[c(\"p5\", \"p99\", \"p999\")])\n    largest_value = max(d[c(\"p5\", \"p99\", \"p999\")])\n    distance = largest_value - smallest_value\n    d[c(\"p5\", \"p99\", \"p999\")] = (d[c(\"p5\", \"p99\", \"p999\")] - smallest_value) / distance\n    cat(\"normalized to 0..1 the smallest value was\", smallest_value, \"and the largest\", largest_value, \"\\n\")\n    latency_unit = \"normalized between 0..1\"\n} else if (\"$normalize\" == \"\") {\n    cat(\"not normalizing values\\n\")\n    latency_unit = \"ms\"\n} else {\n    stop(\"invalid 'normalize' value\")\n}\n\nif (\"$maxy\" == \"\") {\n    maxy=NA\n} else {\n    maxy=as.double(\"$maxy\")\n    cat(\"liming y axis to\", maxy, \"\\n\")\n}\n\nd[c(\"p5\", \"p99\", \"p999\")] = round(d[c(\"p5\", \"p99\", \"p999\")], 2)\nimage = (\n    ggplot(d, aes(x=bg_load, group=cri_conf, shape=cri_conf), ylim=c(0, maxy))\n    + ggtitle(\"Memtier_benchmark total latencies with plain CRI and CRI-RM\")\n    + ylab(paste(\"Latency (\", latency_unit, \")\", sep=\"\"))\n    + xlab(\"Background load (stress-ng)\")\n    + labs(linetype=\"Percentiles\")\n    + labs(color=\"CRI layer\")\n    + geom_line(aes(color=cri_conf, linetype=\"50 %\", y=p5))\n    + geom_line(aes(color=cri_conf, linetype=\"99 %\", y=p99))\n    + geom_line(aes(color=cri_conf, linetype=\"99.9 %\", y=p999))\n    + scale_x_discrete(limits=c(\"NOLOAD\", \"SHM\", \"CPUALL\", \"CPUJMP\", \"MEMCPY\", \"STREAM\"))\n    + scale_y_continuous(limits=c(NA, maxy), trans='$ytrans')\n    )\n# print full data matrix\nd\nprint(\"saving $save.csv\")\nwrite.csv(d, file=\"$save.csv\", row.names=FALSE)\nprint(\"saving $save.svg\")\nggsave(file=\"$save.svg\", plot=image)\nEOF\nRscript plot.R\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/n4c16/test02-multi-memtier/code.var.sh",
    "content": "# Redis parameters\nREDIS_PASS=abc123xyz\n\ntest_time_minutes=3\ndelay_between_rounds=30s\nmemtier_threads=1\ntest_time_seconds=$((test_time_minutes * 60))\ntest_timeout_minutes=$((test_time_minutes + 2))\n\n# BM_* are benchmarks\nBM_MEMTIER=\"memtier-benchmark --server=redis-service --authenticate=$REDIS_PASS --threads=${memtier_threads} --test-time=${test_time_seconds}\"\n\n# Clean up\nvm-command \"kubectl delete jobs --all --now; kubectl delete deployment redis; kubectl delete service redis-service; kubectl delete secret redis; kubectl delete pods --all --now; true\"\n\nvm-command \"systemctl restart dbus\"\nvm-command \"systemctl restart kubelet\"\n\n# Setup Redis\nwait=\"\" create redis-secret\nCPU=4 MEM=8G CPULIM=8 MEMLIM=12G NAME=redis wait=\"Available\" create redis\nNAME=redis-service wait=\"\" create redis-service\n\n# Reset counters in order to keep creating pod0...\nreset counters\n\nbenchmark_output_dir=\"$OUTPUT_DIR/benchmark/multi-memtier\"\nmkdir -p \"$benchmark_output_dir\"\n\nexport NAME=memtier-benchmark\nfor bm_cmd in \"${!BM_@}\"; do\n    for CPU in 1 2; do\n\tfor CONTCOUNT in 2 4 6 8 10 12 14; do\n            # Run benchmark\n            AFFINITY=affinity CPU=\"$CPU\" MEM=\"4G\" CPULIM=\"$CPU\" MEMLIM=\"5G\" ARGS=\"${!bm_cmd#memtier-benchmark }\" wait=\"Complete\" wait_t=\"${test_timeout_minutes}m\" create memtier-benchmark-02\n            memtier_benchmark_pod=\"$(kubectl get pods | awk '/memtier-benchmark-/{print $1}')\"\n\t    outfile=\"$benchmark_output_dir/$bm_cmd-affinity-cpu-$CPU-contcount-$CONTCOUNT.txt\"\n\t    rm -f $outfile\n\t    for contnum in $(seq 0 $((CONTCOUNT-1))); do\n\t\tcontname=${NAME}c${contnum}\n\t\techo \"memtier benchmark CPU=$CPU log for $contname:\"\n\t\tkubectl logs \"$memtier_benchmark_pod\" \"$contname\" | grep -A7 'ALL STATS' | tee -a $outfile\n\t    done\n            kubectl delete jobs --all --now\n\t    # find average from all containers:\n\t    #============================================================================================================================\n\t    #Type         Ops/sec     Hits/sec   Misses/sec    Avg. Latency     p50 Latency     p99 Latency   p99.9 Latency       KB/sec\n\t    #----------------------------------------------------------------------------------------------------------------------------\n\t    sum_ops_sec=$(awk '/Totals/ {sum+=$2} END {print sum}' $outfile)\n\t    avg_ops_sec=$(awk '/Totals/ {sum+=$2;cnt++} END {printf(\"%d\", sum/cnt)}' $outfile)\n\t    avg_latency=$(awk '/Totals/ {sum+=$5;cnt++} END {printf(\"%f\", sum/cnt)}' $outfile)\n\t    avg_p50_lat=$(awk '/Totals/ {sum+=$6;cnt++} END {printf(\"%f\", sum/cnt)}' $outfile)\n\t    avg_p99_lat=$(awk '/Totals/ {sum+=$7;cnt++} END {printf(\"%f\", sum/cnt)}' $outfile)\n\t    printf \"CPU=%2d CONTAINERS=%2d Ops/sec_sum:%7.0f Ops/sec_avg:%7.0f Latency:%f p50Latency:%f p99Latency:%f\\n\" \\\n\t\t   $CPU $CONTCOUNT $sum_ops_sec $avg_ops_sec $avg_latency $avg_p50_lat $avg_p99_lat |tee -a $outfile\n\t    echo \"sleep ${delay_between_rounds} between tests...\"\n\t    echo \"==================================================\"\n\t    sleep ${delay_between_rounds}\n\tdone\n    done\ndone\necho \"Use 'grep CONTAINERS output-of-this-session' to show one-line-per-round output\"\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/redis-secret.yaml.in",
    "content": "apiVersion: v1\nkind: Secret\nmetadata:\n  name: redis\ndata:\n  REDIS_PASS: $(base64 -w0 <<< \"$REDIS_PASS\")\ntype: Opaque\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/redis-service.yaml.in",
    "content": "apiVersion: v1\nkind: Service\nmetadata:\n  name: $NAME\n  labels:\n    app: redis\nspec:\n  ports:\n    - name: redis\n      port: 6379\n      targetPort: 6379\n  selector:\n    app: redis\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/redis.yaml.in",
    "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  labels:\n    app: redis\n  name: redis\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: redis\n  template:\n    metadata:\n      labels:\n        app: redis\n    spec:\n      containers:\n        - name: redis\n          image: redis\n          imagePullPolicy: IfNotPresent\n          args: ['--requirepass', '$REDIS_PASS']\n          ports:\n            - containerPort: 6379\n              name: redis\n          env:\n            - name: MASTER\n              value: 'true'\n            - name: REDIS_PASS\n              valueFrom:\n                secretKeyRef:\n                  name: redis\n                  key: REDIS_PASS\n          resources:\n            requests:\n              cpu: ${CPU}\n              memory: '${MEM}'\n            limits:\n              cpu: ${CPULIM}\n              memory: '${MEMLIM}'\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/stress-ng-benchmark.yaml.in",
    "content": "apiVersion: batch/v1\nkind: Job\nmetadata:\n  name: stress-ng-benchmark\nspec:\n  template:\n    spec:\n      containers:\n        - name: ${NAME}c$(( contnum - 1 ))\n          image: alexeiled/stress-ng\n          imagePullPolicy: IfNotPresent\n          args: ['${ARGS// /\\', \\'}']\n          $(if [ \"$CPU\" != \"0\" ]; then echo \"\n          resources:\n            requests:\n              cpu: ${CPU}\n              memory: '${MEM}'\n            limits:\n              cpu: ${CPULIM}\n              memory: '${MEMLIM}'\n          \"; fi)\n      restartPolicy: Never\n"
  },
  {
    "path": "test/e2e/benchmarks.test-suite/memtier_benchmark/stress-ng.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\n    e2erole: bgload\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: alexeiled/stress-ng\n    imagePullPolicy: IfNotPresent\n    args: ['${ARGS// /\\', \\'}']\n    $(if [ \"$CPU\" != \"0\" ]; then echo \"\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPULIM}\n        memory: '${MEMLIM}'\n    \"; fi)\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/besteffort.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\n    $(for lbl in ${!LABEL*}; do [[ \"$lbl\" == LABEL[0-9]* ]] && echo \"\n    ${!lbl}\n    \"; done)\n  $([ -n \"${!ANN*}\" ] && echo \"\n  annotations:\n    $(for ann in ${!ANN*}; do [[ \"$ann\" == ANN[0-9]* ]] && echo \"\n    ${!ann}\n    \"; done)\n  \")\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/blockio/n4c16/test00-slowreader/code.var.sh",
    "content": "install-bfq() {\n    # Install a kernel with BFQ I/O scheduler.\n    if [[ \"$distro\" == ubuntu* ]]; then\n        vm-command \"uname -a\"\n        if ! grep -q lowlatency <<< \"$COMMAND_OUTPUT\"; then\n            vm-command \"apt install -y linux-image-lowlatency\" ||\n                command-error \"failed to install lowlatency kernel for the BFQ I/O scheduler\"\n\n        fi\n        vm-reboot\n        vm-command-q \"uname -a | grep lowlatency\" || {\n            error \"failed to switch to lowlatency kernel\"\n        }\n        vm-command \"modprobe bfq\"\n    elif [[ \"$distro\" == debian* ]]; then\n        vm-command \"apt install -y linux-image-rt-amd64\"\n        vm-reboot\n        vm-command \"modprobe bfq\"\n    else\n        error \"not implemented: install kernel with BFQ I/O scheduler support to $distro\"\n    fi\n}\n\n# Make sure the BFQ scheduler is available in the system.\nif ! vm-command-q \"grep bfq /sys/block/vda/queue/scheduler\"; then\n    vm-command \"modprobe bfq\"\n    vm-command-q \"grep bfq /sys/block/vda/queue/scheduler\" || {\n        install-bfq\n        vm-command-q \"grep bfq /sys/block/vda/queue/scheduler\" || {\n            error \"failed to make bfq I/O scheduler available in /sys/block/vda/queue/scheduler\"\n        }\n    }\nfi\n\n# Switch to BFQ.\nfor blkdev in vda vdb; do\n    if ! vm-command-q \"grep '[[]bfq[]]' /sys/block/$blkdev/queue/scheduler\"; then\n        vm-command \"echo bfq > /sys/block/$blkdev/queue/scheduler\"\n        vm-command-q \"grep '[[]bfq[]]' /sys/block/$blkdev/queue/scheduler\" || {\n            error \"failed to switch using bfq on /dev/$blkdev\"\n        }\n    fi\ndone\n\nif [[ \"$k8scri\" == *\"containerd\"* ]]; then\n    # Start importing configurations from /etc/containerd/config.d/*.toml.\n    vm-command-q \"[ -f /etc/containerd/config.toml ] || echo \"\" > /etc/containerd/config.toml\"\n    vm-command-q \"grep '^imports' /etc/containerd/config.toml || sed -i '1iimports = [\\\"/etc/containerd/config.d/*.toml\\\"]' /etc/containerd/config.toml\"\n    vm-command-q \"grep -E '^imports.*/etc/containerd/config.d/' || sed -i 's:^\\(imports.*\\)\\]:\\1, \\\"/etc/containerd/config.d/*.toml\\\"\\]:' /etc/containerd/config.toml\"\n    # e2e-specific config: tasks-service plugin loads blockio_config_file.\n    vm-pipe-to-file /etc/containerd/config.d/e2e.toml <<EOF\n[plugins.\"io.containerd.service.v1.tasks-service\"]\nblockio_config_file=\"/etc/containers/blockio.yaml\"\nEOF\n    vm-command \"systemctl restart containerd\"\n    sleep 5\nelse\n    vm-command \"systemctl restart crio\" # make sure to apply latest --blockio-config-file\n    sleep 5\nfi\n\n# Install a test script for viewing cgroups v1 and v2 values of k8s pods/containers.\nvm-put-file \"$HOST_PROJECT_DIR/scripts/testing/kube-cgroups\" \"/usr/local/bin/kube-cgroups\"\n\n# Create pod0, a single-container pod with a pod-level annotation.\nANN0='blockio.resources.beta.kubernetes.io/pod: \"lowprio\"' create besteffort\n\nvm-command \"kube-cgroups -f io\\\\. -p pod0\"\n# Check the blkio controller data when using cgroups v1.\nif \\\n    grep blkio.throttle <<< \"$COMMAND_OUTPUT\" && (\n        ( ! grep -A2 'blkio.throttle.read_bps_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 1000000' ) || \\\n            ( ! grep -A2 'blkio.throttle.read_bps_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 1000000' ) || \\\n            ( ! grep -A2 'blkio.throttle.read_iops_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 2000' ) || \\\n            ( ! grep -A2 'blkio.throttle.read_iops_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 2000' ) || \\\n            ( ! grep -A2 'blkio.throttle.write_bps_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 512000' ) || \\\n            ( ! grep -A2 'blkio.throttle.write_bps_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 512000' ) || \\\n            ( ! grep -A2 'blkio.throttle.write_iops_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 300' ) || \\\n            ( ! grep -A2 'blkio.throttle.write_iops_device:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 300' ) \\\n            )\nthen\n    command-error \"expected blkio.throttle.read_bps_device not found from cgroups v1\"\nfi\n\n# Check the io controller data (successor of blkio) when using cgroups v2.\nif \\\n    grep io.max <<< \"$COMMAND_OUTPUT\" && (\n        ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 .*rbps=1000000' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 .*rbps=1000000' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 .*riops=2000' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 .*riops=2000' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 .*wbps=512000' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 .*wbps=512000' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:16 .*wiops=300' ) || \\\n            ( ! grep -A2 'io.max:' <<< \"$COMMAND_OUTPUT\" | grep -q -E '25[0-5]:0 .*wiops=300' ) \\\n            )\nthen\n    command-error \"expected io.max values not found from cgroups v2\"\nfi\n\n\n# Create pod1, a triple-container pod with all containers in different blockio classes.\nCONTCOUNT=3 CPU=1 MEM=64M \\\n         ANN0='blockio.resources.beta.kubernetes.io/pod: \"lowprio\"' \\\n         ANN1='blockio.resources.beta.kubernetes.io/container.pod1c1: \"normal\"' \\\n         ANN2='blockio.resources.beta.kubernetes.io/container.pod1c2: \"highprio\"' \\\n         create guaranteed\n\nvm-command \"kube-cgroups -f '.*io.bfq.weight.*' -c pod1c0\"\ngrep -q 88 <<< \"$COMMAND_OUTPUT\" || error \"expected blkio.bfq.weight: 88\"\n\n# In some setups besteffort slice has the default bfq weight 10, in some other 100. Accept both.\nvm-command \"kube-cgroups -f '.*io.bfq.weight.*' -c pod1c1\"\ngrep -q -E 'default 10(0)?' <<< \"$COMMAND_OUTPUT\" || error \"expected blkio.bfq.weight: default 10 or 100\"\n\n# Check device-specific bfq I/O scheduler weights.\nvm-command \"kube-cgroups -f '.*io.bfq.weight.*' -c pod1c2\"\nif grep -q blkio.bfq.weight_device <<< \"$COMMAND_OUTPUT\"; then\n    grep -A3 blkio.bfq.weight_device <<< \"$COMMAND_OUTPUT\" | grep -q -E \"25[0-5]:0 444\" || error \"expected 444\"\n    grep -A3 blkio.bfq.weight_device <<< \"$COMMAND_OUTPUT\" | grep -q -E \"25[0-5]:16 555\" || error \"expected 555\"\nelse\n    grep -A3 io.bfq.weight <<< \"$COMMAND_OUTPUT\" | grep -q -E \"25[0-5]:0 444\" || error \"expected 444\"\n    grep -A3 io.bfq.weight <<< \"$COMMAND_OUTPUT\" | grep -q -E \"25[0-5]:16 555\" || error \"expected 555\"\nfi\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/blockio/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/blockio/n4c16/vm-files/etc/containers/blockio.yaml",
    "content": "Classes:\n  lowprio:\n  - Weight: 88\n  - Devices:\n    - /dev/vd[a-z]\n    ThrottleReadBps: 1M\n    ThrottleWriteBps: 512k\n    ThrottleReadIOPS: 2000\n    ThrottleWriteIOPS: 300\n  normal:\n  - ThrottleWriteBps: 16M\n    Devices:\n    - /dev/vda\n  highprio:\n  - Devices:\n    - /dev/vda\n    Weight: 444\n  - Devices:\n    - /dev/vdb\n    Weight: 555\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/blockio/n4c16/vm-files/etc/crio/crio.conf.d/55-blockio",
    "content": "[crio.runtime]\nblockio_config_file=\"/etc/containers/blockio.yaml\"\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/containerd_src.var.in.sh",
    "content": "${containerd_src:-$GOPATH/src/github.com/containerd/containerd}\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/crio_src.var.in.sh",
    "content": "${crio_src:-$GOPATH/src/github.com/cri-o/cri-o}\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/k8scri.var.in.sh",
    "content": "${k8scri:-crio}\n"
  },
  {
    "path": "test/e2e/blockio.test-suite/omit_cri_resmgr.var.sh",
    "content": "1\n"
  },
  {
    "path": "test/e2e/burstable.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\n    $(for lbl in ${!LABEL*}; do [[ \"$lbl\" == LABEL[0-9]* ]] && echo \"\n    ${!lbl}\n    \"; done)\n  $([ -n \"${!ANN*}\" ] && echo \"\n  annotations:\n    $(for ann in ${!ANN*}; do [[ \"$ann\" == ANN[0-9]* ]] && echo \"\n    ${!ann}\n    \"; done)\n  \")\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPUREQ}\n        memory: ${MEMREQ}\n      limits:\n        cpu: ${CPULIM}\n        memory: ${MEMLIM}\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/cri-resmgr-topology-aware.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/guaranteed.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\n    $(for lbl in ${!LABEL*}; do [[ \"$lbl\" == LABEL[0-9]* ]] && echo \"\n    ${!lbl}\n    \"; done)\n  $([ -n \"${!ANN*}\" ] && echo \"\n  annotations:\n    $(for ann in ${!ANN*}; do [[ \"$ann\" == ANN[0-9]* ]] && echo \"\n    ${!ann}\n    \"; done)\n  \")\nspec:\n  $(if [ -n \"$ICONTCOUNT\" ]; then echo \"\n  initContainers:\n  $(for contnum in $(seq 1 ${ICONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))i\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 ))i \\$(sleep ${ICONTSLEEP:-5})\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  \"; fi )\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-11/binsrc.var",
    "content": "packages/debian-11\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-11/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-11/distro.var",
    "content": "debian-11\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-11/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-11/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-11/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-12/binsrc.var",
    "content": "packages/debian-12\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-12/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-12/distro.var",
    "content": "debian-12\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-12/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-12/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-12/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-sid/binsrc.var",
    "content": "packages/debian-sid"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-sid/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-sid/distro.var",
    "content": "debian-sid\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-sid/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-sid/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/debian-sid/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/fedora/binsrc.var",
    "content": "packages/fedora"
  },
  {
    "path": "test/e2e/packages.test-suite/fedora/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/fedora/distro.var",
    "content": "fedora\n"
  },
  {
    "path": "test/e2e/packages.test-suite/fedora/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/fedora/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/fedora/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/opensuse-15.6/binsrc.var",
    "content": "packages/opensuse-leap-15.6\n"
  },
  {
    "path": "test/e2e/packages.test-suite/opensuse-15.6/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/opensuse-15.6/distro.var",
    "content": "opensuse-15.6\n"
  },
  {
    "path": "test/e2e/packages.test-suite/opensuse-15.6/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/opensuse-15.6/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/opensuse-15.6/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-18.04/binsrc.var",
    "content": "packages/ubuntu-18.04"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-18.04/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-18.04/distro.var",
    "content": "ubuntu-18.04\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-18.04/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-18.04/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-18.04/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-20.04/binsrc.var",
    "content": "packages/ubuntu-20.04"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-20.04/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-20.04/distro.var",
    "content": "ubuntu-20.04\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-20.04/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-20.04/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-20.04/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-22.04/binsrc.var",
    "content": "packages/ubuntu-22.04\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-22.04/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-22.04/distro.var",
    "content": "ubuntu-22.04\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-22.04/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-22.04/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-22.04/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-24.04/binsrc.var",
    "content": "packages/ubuntu-24.04\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-24.04/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-24.04/distro.var",
    "content": "ubuntu-24.04\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-24.04/pkgtest/test01-systemd/code.var.sh",
    "content": "# Clear cri-resmgr output from previous runs.\nvm-command \"journalctl --vacuum-time=1s\"\n\n# Create a pod.\ncreate besteffort\n\n# Verify that new pod was created by systemd-managed cri-resource-manager.\nvm-command \"journalctl -xeu cri-resource-manager | grep 'StartContainer: starting container pod0:pod0c0'\" || {\n    command-error \"failed to verify that systemd-managed cri-resource-manager launched the pod\"\n}\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-24.04/pkgtest/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/packages.test-suite/ubuntu-24.04/reinstall_cri_resmgr.var",
    "content": "1"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/balloons-busybox.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"$POD_ANNOTATION\" ]; then echo \"\n  annotations:\n    $POD_ANNOTATION\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - ${WORK}echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    $(if [ -n \"${CPUREQ}\" ]; then echo \"\n    resources:\n      requests:\n        cpu: ${CPUREQ}\n        $(if [ -n \"${MEMREQ}\" ]; then echo \"\n        memory: '${MEMREQ}'\n        \"; fi)\n      $(if [ -n \"${CPULIM}\" ]; then echo \"\n      limits:\n        cpu: ${CPULIM}\n        $(if [ -n \"$MEMLIM\" ]; then echo \"\n        memory: '${MEMLIM}'\n        \"; fi)\n    \"; fi)\n    \"; fi)\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/balloons-configmap.yaml.in",
    "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.default\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: balloons\n    AvailableResources:\n      CPU: ${AVAILABLE_CPU:-cpuset:0-15}\n    ReservedResources:\n      CPU: ${RESERVED_CPU:-1}\n\n    $([ -z \"$IDLECPUCLASS\" ] || echo \"\n    IdleCPUClass: ${IDLECPUCLASS}\n    \")\n\n    balloons:\n      PinCPU: ${PINCPU:-true}\n      PinMemory: ${PINMEMORY:-true}\n      BalloonTypes:\n\n        $([ -n \"$BTYPE0_SKIP\" ] || echo \"\n        - Name: btype0\n          MinCPUs: ${BTYPE0_MINCPUS:-2}\n          MaxCPUs: ${BTYPE0_MAXCPUS:-2}\n          AllocatorPriority: ${BTYPE0_ALLOCATORPRIORITY:-0}\n          CPUClass: ${BTYPE0_CPUCLASS:-classA}\n          PreferNewBalloons: ${BTYPE0_PREFERNEWBALLOONS:-true}\n          PreferSpreadingPods: ${BTYPE0_PREFERSPREADINGPODS:-false}\n        \")\n\n        $([ -n \"$BTYPE1_SKIP\" ] || echo \"\n        - Name: btype1\n          Namespaces:\n            - ${BTYPE1_NAMESPACE0:-btype1ns0}\n          MinCPUs: ${BTYPE1_MINCPUS:-1}\n          MaxCPUs: ${BTYPE1_MAXCPUS:-1}\n          AllocatorPriority: ${BTYPE1_ALLOCATORPRIORITY:-1}\n          CPUClass: ${BTYPE1_CPUCLASS:-classB}\n          PreferNewBalloons: ${BTYPE1_PREFERNEWBALLOONS:-false}\n          PreferSpreadingPods: ${BTYPE1_PREFERSPREADINGPODS:-true}\n        \")\n\n        $([ -n \"$BTYPE2_SKIP\" ] || echo \"\n        - Name: btype2\n          Namespaces:\n            - ${BTYPE2_NAMESPACE0:-btype2ns0}\n            - ${BTYPE2_NAMESPACE1:-btype2ns1}\n          MinCPUs: ${BTYPE2_MINCPUS:-4}\n          MaxCPUs: ${BTYPE2_MAXCPUS:-8}\n          MinBalloons: ${BTYPE2_MINBALLOONS:-1}\n          AllocatorPriority: ${BTYPE2_ALLOCATORPRIORITY:-2}\n          CPUClass: ${BTYPE2_CPUCLASS:-classC}\n          PreferNewBalloons: ${BTYPE2_PREFERNEWBALLOONS:-false}\n          PreferSpreadingPods: ${BTYPE2_PREFERSPREADINGPODS:-false}\n        \")\n\n  instrumentation: |+\n    HTTPEndpoint: :8891\n    PrometheusExport: true\n\n  logger: |+\n    Debug: policy\n\n  cpu: |+\n    classes:\n      default:\n        minFreq: ${CPU_DEFAULT_MIN:-800}\n        maxFreq: ${CPU_DEFAULT_MAX:-2800}\n      classA:\n        minFreq: ${CPU_CLASSA_MIN:-900}\n        maxFreq: ${CPU_CLASSA_MAX:-2900}\n      classB:\n        minFreq: ${CPU_CLASSB_MIN:-1000}\n        maxFreq: ${CPU_CLASSB_MAX:-3000}\n      classC:\n        minFreq: ${CPU_CLASSC_MIN:-1100}\n        maxFreq: ${CPU_CLASSC_MAX:-3100}\n        energyPerformancePreference: ${CPU_CLASSC_EPP:-1}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/cri-resmgr.cfg",
    "content": "policy:\n  Active: balloons\n  # Use only 15 CPUs in total, leave cpu0 for other than Kubernetes\n  # processes.\n  AvailableResources:\n    CPU: cpuset:1-15\n  # Reserve one of our CPUs (cpu15) for kube-system tasks.\n  ReservedResources:\n    CPU: 1\n  balloons:\n    # PinCPU: allow containers to use only the CPUs in their balloons.\n    PinCPU: true\n    # PinMemory: allow containers to use only the closest memory to\n    # the CPUs in their balloons.\n    PinMemory: true\n    BalloonTypes:\n      - Name: two-cpu\n        MinCPUs: 2\n        MaxCPUs: 2\n        AllocatorPriority: 0\n        CPUClass: class2\n        PreferNewBalloons: true\n\n      - Name: three-cpu\n        Namespaces:\n          - \"three\"\n        MinCPUs: 3\n        AllocatorPriority: 1\n        CPUClass: class3\n        PreferSpreadingPods: true\n\n      - Name: four-cpu\n        MinCPUs: 4\n        MaxCPUs: 8\n        MinBalloons: 1\n        AllocatorPriority: 2\n        CPUClass: class4\n\n      - Name: five-cpu\n        MaxCPUs: 5\n        AllocatorPriority: 3\n        PreferSpreadingPods: true\n        PreferNewBalloons: true\n        CPUClass: class5\n\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n\ncpu:\n  classes:\n    default:\n      minFreq: 800\n      maxFreq: 2800\n    class2:\n      minFreq: 900\n      maxFreq: 2900\n    class3:\n      minFreq: 1000\n      maxFreq: 3000\n    class4:\n      minFreq: 1100\n      maxFreq: 3100\n      energyPerformancePreference: 1\n    class5:\n      minFreq: 1200\n      maxFreq: 3200\n      energyPerformancePreference: 2\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test01-basic-placement/code.var.sh",
    "content": "# Test placing containers with and without annotations to correct balloons\n# reserved and shared CPUs.\n\ncleanup() {\n    vm-command \"kubectl delete pods -n kube-system pod0; kubectl delete pods --all --now --wait; kubectl delete namespace three --now --wait --ignore-not-found\"\n    return 0\n}\n\ncleanup\n\n# pod0: run on reserved CPUs\nnamespace=kube-system CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 1'\n\n# pod1: run on the same two-cpu balloon (running containers of a pod\n# on the same balloon takes precedence creating new balloons).\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: two-cpu\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'len(cpus[\"pod1c0\"]) == 2'\n\n# pod2: run on a different two-cpu balloon than pod1 (new balloon\n# creation is preferred).\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: two-cpu\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod2c0\"]) == 2' \\\n       'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod1c0\"])'\n\n# pod3: fits exactly on a single three-cpu instance. No need to create\n# new balloon even if spreading pods is preferred.\nCPUREQ=\"1500m\" MEMREQ=\"100M\" CPULIM=\"1500m\" MEMLIM=\"100M\"\nkubectl create namespace \"three\"\nnamespace=\"three\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod3c0\"] == cpus[\"pod3c1\"]' \\\n       'len(cpus[\"pod3c0\"]) == 3'\n\ncleanup\n\n# pod4: first two containers to the first instance, 3rd to new four-cpu instance\nCPUREQ=\"3\" MEMREQ=\"\" CPULIM=\"3\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: four-cpu\" CONTCOUNT=3 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod4c0\"] == cpus[\"pod4c1\"]' \\\n       'disjoint_sets(cpus[\"pod4c2\"], cpus[\"pod4c0\"])' \\\n       'len(cpus[\"pod4c0\"]) == 6' \\\n       'len(cpus[\"pod4c2\"]) == 4'\n\ncleanup\n\n# pod5: all spread containers to their own balloon instances\nCPUREQ=\"1250m\" MEMREQ=\"\" CPULIM=\"\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: five-cpu\" CONTCOUNT=3 create balloons-busybox\nreport allowed\nverify 'disjoint_sets(cpus[\"pod5c0\"], cpus[\"pod5c1\"], cpus[\"pod5c2\"])' \\\n       'len(cpus[\"pod5c0\"]) == 2' \\\n       'len(cpus[\"pod5c1\"]) == 2' \\\n       'len(cpus[\"pod5c2\"]) == 2'\n\ncleanup\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/balloons-metrics.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: 1\n  balloons:\n    BalloonTypes:\n      - Name: full-core\n        MinCPUs: 2\n        MaxCPUs: 2\n        CPUClass: normal\n\n      - Name: fast-dualcore\n        MinCPUs: 4\n        MaxCPUs: 4\n        CPUClass: turbo\n        PreferNewBalloons: true\n\n      - Name: flex\n        MaxCPUs: 8\n        CPUClass: slow\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh",
    "content": "# This test verifies prometheus metrics from the balloons policy.\n\ncleanup() {\n    vm-command \"kubectl delete pods --all --now --wait\"\n    return 0\n}\n\ncleanup\n\n# Launch cri-resmgr with wanted metrics update interval and a\n# configuration that opens the instrumentation http server.\nterminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-metrics.cfg  cri_resmgr_extra_args=\"-metrics-interval 4s\" launch cri-resmgr\nverify-metrics-has-line 'balloon=\"default\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"reserved\\[0\\]\"'\nverify-metrics-has-no-line 'balloon=\"full-core\\[0\\]\"'\nverify-metrics-has-no-line 'balloon_type=\"full-core\"'\nverify-metrics-has-no-line 'balloon_type=\"fast-dualcore\"'\nverify-metrics-has-no-line 'balloon_type=\"flex\"'\n\n# pod0 in full-core[0]\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: full-core\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify-metrics-has-line 'balloon=\"default\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"reserved\\[0\\]\"'\nverify-metrics-has-line 'balloons{balloon=\"full-core\\[0\\]\",balloon_type=\"full-core\",containers=\"pod0:pod0c0,pod0:pod0c1\",cpu_class=\"normal\",cpus=\".*\",cpus_allowed=\".*\",cpus_allowed_count=\"2\",cpus_count=\"2\",cpus_max=\"2\",cpus_min=\"2\",dies=\"p[01]d0\",dies_count=\"1\",mems=\"0\",numas=\"p[01]d0n[0-3]\",numas_count=\"1\",packages=\"p[01]\",packages_count=\"1\",sharedidlecpus=\"\",sharedidlecpus_count=\"0\",tot_req_millicpu=\"(199|200)\"} 2'\n\n# pod1 in fast-dualcore[0]\nCPUREQ=\"200m\" MEMREQ=\"\" CPULIM=\"200m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: fast-dualcore\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify-metrics-has-line 'balloon=\"fast-dualcore\\[0\\]\".*tot_req_millicpu=\"(199|200)\".* 4'\nverify-metrics-has-no-line 'balloon=\"fast-dualcore\\[1\\]\"'\n\n# pod2 in fast-dualcore[1] (FillChain prefers new-balloon)\nCPUREQ=\"500m\" MEMREQ=\"\" CPULIM=\"500m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: fast-dualcore\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify-metrics-has-line 'balloon=\"fast-dualcore\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"fast-dualcore\\[1\\]\".*tot_req_millicpu=\"500\".* 4'\nverify-metrics-has-no-line 'balloon_type=\"flex\"'\n\n# pod3 in flex[0]\nCPUREQ=\"3500m\" MEMREQ=\"\" CPULIM=\"3500m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: flex\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify-metrics-has-line 'balloon_type=\"flex\".* 4'\n\n# pod4 in flex[0], balloon inflated to fit pod3 + pod4\nCPUREQ=\"1200m\" MEMREQ=\"\" CPULIM=\"1200m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: flex\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify-metrics-has-line 'balloon_type=\"flex\"'\nverify-metrics-has-line 'balloon_type=\"flex\".* 5'\n\n# check deflating a balloon in metrics\nkubectl delete pods --now --wait --ignore-not-found pod3\nverify-metrics-has-line 'balloon_type=\"flex\"'\nverify-metrics-has-line 'balloon_type=\"flex\".* 2'\n\nkubectl delete pods --now --wait --ignore-not-found pod4\nsleep 5\n# check popping a balloon from metrics\nverify-metrics-has-no-line 'balloon_type=\"flex\"'\n\n# pop fast-dualcore[0], keep fast-dualcore[1]\nkubectl delete pods --now --wait --ignore-not-found pod1\nverify-metrics-has-line 'balloon=\"fast-dualcore\\[1\\]\"'\nsleep 5\nverify-metrics-has-no-line 'balloon=\"fast-dualcore\\[0\\]\"'\n\n# re-create balloon instance fast-dualcore[0] that was just popped.\n# pod5 in fast-dualcore[0], pod2 keeps running in fast-dualcore[1]\nCPUREQ=\"4000m\" MEMREQ=\"100M\" CPULIM=\"4000m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: fast-dualcore\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify-metrics-has-line 'balloon=\"fast-dualcore\\[1\\]\".*pod2c0.* 4'\nverify-metrics-has-line 'balloon=\"fast-dualcore\\[0\\]\".*pod5c0.* 4'\n\n# # Re-launch cri-resmgr with test suite default parameters\n# terminate cri-resmgr\n# launch cri-resmgr\n\ncleanup\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/balloons-reserved.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: cpuset:0-2\n  balloons:\n    IdleCPUClass: idle-class\n    ReservedPoolNamespaces:\n      - \"monitor-*\"\n      - \"*-log*\"\n    BalloonTypes:\n      - Name: reserved\n        Namespaces:\n          - my-exact-name\n        CPUClass: reserved-class\n      - Name: default\n        MinCPUs: 1\n      - Name: full-core\n        MinCPUs: 2\n        MaxCPUs: 2\n        CPUClass: turbo\n        MinBalloons: 2\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test03-reserved/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-reserved.cfg launch cri-resmgr\n\ncleanup() {\n    vm-command \\\n        \"kubectl delete pod -n kube-system --now --wait --ignore-not-found pod0\n         kubectl delete pod -n monitor-mypods --now --wait --ignore-not-found pod1\n         kubectl delete pod -n system-logs --now --wait --ignore-not-found pod2\n         kubectl delete pod -n kube-system --now --wait --ignore-not-found pod3\n         kubectl delete pods --now --wait --ignore-not-found pod4 pod5 pod6\n         kubectl delete pod -n kube-system --now --wait --ignore-not-found pod7\n         kubectl delete namespace monitor-mypods --wait --ignore-not-found\n         kubectl delete namespace system-logs --wait --ignore-not-found\n         kubectl delete namespace my-exact-name --wait --ignore-not-found\"\n    return 0\n}\n\ncleanup\n\nkubectl create namespace monitor-mypods\nkubectl create namespace system-logs\nkubectl create namespace my-exact-name\n\n# pod0: kube-system\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=kube-system create balloons-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == {\"cpu00\", \"cpu01\", \"cpu02\"}'\n\n# pod1: match first ReservedPoolNamespaces glob, multicontainer\nCPUREQ=\"1\" MEMREQ=\"\" CPULIM=\"1\" MEMLIM=\"\"\nnamespace=monitor-mypods CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod1c0\"] == cpus[\"pod0c0\"]' \\\n       'cpus[\"pod1c1\"] == cpus[\"pod0c0\"]'\n\n# pod2: match last ReservedPoolNamespaces glob, slightly overbook reserved CPU\nCPUREQ=\"1\" MEMREQ=\"\" CPULIM=\"1\" MEMLIM=\"\"\nnamespace=system-logs create balloons-busybox\nreport allowed\nverify 'cpus[\"pod2c0\"] == cpus[\"pod0c0\"]'\n\n# pod3: force a kube-system pod to full-core using an annotation\nCPUREQ=\"2\" MEMREQ=\"\" CPULIM=\"2\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: full-core\" namespace=kube-system create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod3c0\"]) == 2' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod3c0\"])'\n\n# pod4: a default pod, should go to the default balloon and inflate it\n# (configuration does not set MaxCPUs limit on the default balloon)\nCPUREQ=\"2500m\" MEMREQ=\"\" CPULIM=\"2500m\" MEMLIM=\"\"\ncreate balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod4c0\"]) == 3' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod3c0\"], cpus[\"pod4c0\"])'\n\n# pod5: annotate otherwise a default pod to the reserved CPUs,\n# severely overbook reserved CPUs\nCPUREQ=\"2500m\" MEMREQ=\"\" CPULIM=\"2500m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: reserved\" create balloons-busybox\nreport allowed\nverify 'cpus[\"pod5c0\"] == {\"cpu00\", \"cpu01\", \"cpu02\"}' \\\n       'disjoint_sets(cpus[\"pod5c0\"], cpus[\"pod3c0\"], cpus[\"pod4c0\"])'\n\ncleanup\n\n# Now that all pods are deleted, make sure that cpus of reserved and\n# default pools are as expected. They could be now wrong if emptying\n# the pools would have let to deflating or popping the balloons.\n\n# pod6: a default pod, should go to the default balloon, and fit in\n# the single CPU\nCPUREQ=\"999m\" MEMREQ=\"\" CPULIM=\"999m\" MEMLIM=\"\"\ncreate balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod6c0\"]) == 1'\n\n# pod7: kube-system\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=kube-system create balloons-busybox\nreport allowed\nverify 'cpus[\"pod7c0\"] == {\"cpu00\", \"cpu01\", \"cpu02\"}'\n\ncleanup\n\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/balloons-namespace.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: 1\n  balloons:\n    PinCPU: true\n    PinMemory: true\n    BalloonTypes:\n      - Name: nsballoon\n        Namespaces:\n          - \"*\"\n        MinCPUs: 2\n        MaxCPUs: 4\n        PreferPerNamespaceBalloon: true\nlogger:\n  Debug: policy\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test05-namespace/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-namespace.cfg launch cri-resmgr\n\ncleanup() {\n    vm-command \\\n        \"kubectl delete pods --all --now --wait\n         kubectl delete namespace e2e-a --wait --ignore-not-found\n         kubectl delete namespace e2e-b --wait --ignore-not-found\n         kubectl delete namespace e2e-c --wait --ignore-not-found\n         kubectl delete namespace e2e-d --wait --ignore-not-found\"\n    return 0\n}\ncleanup\n\nkubectl create namespace e2e-a\nkubectl create namespace e2e-b\nkubectl create namespace e2e-c\nkubectl create namespace e2e-d\n\n# pod0: create in the default namespace, both containers go to nsballoon[0]\nCPUREQ=\"\"\nCONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 2'\n\n# pod1: create in the e2e-a namespace, both containers go nsballoon[1] because\n# nsballoon[0] does not contain any containers in this namespace.\nCPUREQ=\"\"\nnamespace=\"e2e-a\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 2' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"])'\n\n# pod2: create in the default namespace, should go to nsballoon[0] as\n# pod0, and the balloon should inflate to 4 CPUs. cpusets with pod1\n# should not overlap after inflation.\nCPUREQ=\"2\" MEMREQ=\"100M\" CPULIM=\"2\" MEMLIM=\"100M\"\nCONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]' \\\n       'len(cpus[\"pod2c0\"]) == 4' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod0c0\"]' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod0c1\"]' \\\n       'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod1c0\"])'\n\n# pod3: create again in the default namespace. nsballoon[0] has\n# reached the maximum capacity, nsballoon[2] should be created for\n# this pod.\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nCONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod3c0\"] == cpus[\"pod3c1\"]' \\\n       'len(cpus[\"pod3c0\"]) == 2' \\\n       'disjoint_sets(cpus[\"pod3c0\"], cpus[\"pod2c0\"], cpus[\"pod1c0\"])'\n\n# pod4: new namespace => nsballoon[3]\nCPUREQ=\"2\" MEMREQ=\"100M\" CPULIM=\"2\" MEMLIM=\"100M\"\nnamespace=\"e2e-b\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod4c0\"] == cpus[\"pod4c1\"]' \\\n       'len(cpus[\"pod4c0\"]) == 4' \\\n       'disjoint_sets(cpus[\"pod4c0\"], cpus[\"pod3c0\"], cpus[\"pod2c0\"], cpus[\"pod1c0\"])'\n\n# pod5: new namespace => nsballoon[5]\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=\"e2e-c\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod5c0\"] == cpus[\"pod5c1\"]' \\\n       'len(cpus[\"pod5c0\"]) == 2' \\\n       'disjoint_sets(cpus[\"pod5c0\"], cpus[\"pod4c0\"], cpus[\"pod3c0\"], cpus[\"pod2c0\"], cpus[\"pod1c0\"])'\n\n# pod6: new namespace, but nsballoon[6] cannot be created because all\n# CPUs are already allocated to balloons. Cannot honor the preference\n# of spreading different namespaces to different balloon instances\n# anymore, should fallback to balanced assignment.\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=\"e2e-d\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod6c0\"] == cpus[\"pod6c1\"]'\n\ncleanup\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test06-update-configmap/code.var.sh",
    "content": "# This test verifies that configuration updates via cri-resmgr-agent\n# are handled properly in the balloons policy.\n\ntestns=e2e-balloons-test06\n\ncleanup() {\n    vm-command \"kubectl delete pods --all --now --wait; \\\n        kubectl delete namespace $testns --now --wait --ignore-not-found || :; \\\n        kubectl delete namespace btype1ns0 --now --wait --ignore-not-found || :\"\n    terminate cri-resmgr\n    terminate cri-resmgr-agent\n    vm-command \"cri-resmgr -reset-policy; cri-resmgr -reset-config\"\n}\n\napply-configmap() {\n    vm-put-file $(instantiate balloons-configmap.yaml) balloons-configmap.yaml\n    vm-command \"cat balloons-configmap.yaml\"\n    kubectl apply -f balloons-configmap.yaml\n}\n\ncleanup\ncri_resmgr_extra_args=\"-metrics-interval 1s\" cri_resmgr_config=fallback launch cri-resmgr\nlaunch cri-resmgr-agent\n\nkubectl create namespace $testns\nkubectl create namespace btype1ns0\n\nAVAILABLE_CPU=\"cpuset:0,4-15\" BTYPE2_NAMESPACE0='\"*\"' BTYPE1_MAXCPUS='0' apply-configmap\nsleep 3\n\n# pod0 in btype0, annotation\nCPUREQ=1 MEMREQ=\"100M\" CPULIM=1 MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: btype0\" create balloons-busybox\n# pod1 in btype1, namespace\nCPUREQ=1 MEMREQ=\"100M\" CPULIM=1 MEMLIM=\"100M\"\nnamespace=\"btype1ns0\" create balloons-busybox\n# pod2 in btype2, wildcard namespace\nCPUREQ=1 MEMREQ=\"100M\" CPULIM=1 MEMLIM=\"100M\"\nnamespace=\"e2e-balloons-test06\" create balloons-busybox\nvm-command \"curl -s $verify_metrics_url\"\nverify-metrics-has-line 'btype0\\[0\\].*containers=\".*pod0:pod0c0'\nverify-metrics-has-line 'btype1\\[0\\].*containers=\".*pod1:pod1c0'\nverify-metrics-has-line 'btype2\\[0\\].*containers=\".*pod2:pod2c0'\n\n# Remove first two balloon types, change btype2 to match all\n# namespaces.\nBTYPE0_SKIP=1 BTYPE1_SKIP=1 BTYPE2_NAMESPACE0='\"*\"' apply-configmap\n# Note:\n\n# pod0 was successfully assigned to and running in balloon of btype0.\n# Now btype0 was completely removed from the node.\n# Currently this behavior is undefined.\n# Possible behaviors: evict pod0, continue assign chain, refuse config...\n# For now, skip pod0c0 balloon validation:\n# verify-metrics-has-line '\"btype2\\[0\\]\".*pod0:pod0c0'\nverify-metrics-has-line '\"btype2\\[0\\]\".*pod1:pod1c0'\nverify-metrics-has-line '\"btype2\\[0\\]\".*pod2:pod2c0'\n\n# Bring back btype0 where pod0 belongs to by annotation.\nBTYPE1_SKIP=1 BTYPE2_NAMESPACE0='\"*\"' apply-configmap\nverify-metrics-has-line '\"btype0\\[0\\]\".*pod0:pod0c0'\nverify-metrics-has-line '\"btype2\\[0\\]\".*pod1:pod1c0'\nverify-metrics-has-line '\"btype2\\[0\\]\".*pod2:pod2c0'\n\n# Change only CPU classes, no reassigning.\nverify-metrics-has-line 'btype0\\[0\\].*pod0:pod0c0.*cpu_class=\"classA\"'\nverify-metrics-has-line 'btype2\\[0\\].*pod1:pod1c0.*cpu_class=\"classC\"'\nverify-metrics-has-line 'btype2\\[0\\].*pod2:pod2c0.*cpu_class=\"classC\"'\nBTYPE0_CPUCLASS=\"classC\" BTYPE1_SKIP=1 BTYPE2_CPUCLASS=\"classB\" BTYPE2_NAMESPACE0='\"*\"'  apply-configmap\nverify-metrics-has-line 'btype0\\[0\\].*pod0:pod0c0.*cpu_class=\"classC\"'\nverify-metrics-has-line 'btype2\\[0\\].*pod1:pod1c0.*cpu_class=\"classB\"'\nverify-metrics-has-line 'btype2\\[0\\].*pod2:pod2c0.*cpu_class=\"classB\"'\n\ncleanup\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/balloons-maxballoons-impossible.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: 1\n  balloons:\n    PinCPU: true\n    PinMemory: true\n    BalloonTypes:\n      - Name: singleton\n        MinCPUs: 2\n        MaxCPUs: 2\n        MinBalloons: 1\n        MaxBalloons: 1\n      - Name: impossible\n        MinBalloons: 2\n        MaxBalloons: 1\nlogger:\n  Debug: policy\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/balloons-maxballoons.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: 1\n  balloons:\n    PinCPU: true\n    PinMemory: true\n    BalloonTypes:\n      - Name: singleton\n        MinCPUs: 2\n        MaxCPUs: 2\n        MinBalloons: 1\n        MaxBalloons: 1\n      - Name: dynamictwo\n        MaxCPUs: 1\n        MaxBalloons: 2\n        PreferNewBalloon: true\nlogger:\n  Debug: policy\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test07-maxballoons/code.var.sh",
    "content": "cleanup() {\n    vm-command \"kubectl delete pods --all --now --wait\"\n    return 0\n}\n\ncleanup\n\nterminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-maxballoons.cfg launch cri-resmgr\n\n# pod0: allocate 1500/2000 mCPUs of the singleton balloon\nCPUREQ=\"1500m\" CPULIM=\"1500m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: singleton\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 2'\n\n# pod1: allocate the rest 500/2000 mCPUs of the singleton balloon\nCPUREQ=\"500m\" CPULIM=\"500m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: singleton\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod1c0\"]'\n\n# pod2: try to fit in the already full singleton balloon\nCPUREQ=\"100m\" CPULIM=\"100m\"\n( POD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: singleton\" CONTCOUNT=1 wait_t=5s create balloons-busybox ) && {\n    error \"creating pod2 succeeded but was expected to fail with balloon allocation error\"\n}\necho \"pod2 creation failed with an error as expected\"\nvm-command \"kubectl describe pod pod2\"\nif ! grep -q 'no suitable balloon instance available' <<< \"$COMMAND_OUTPUT\"; then\n    error \"could not find 'no suitable balloon instance available' in pod2 description\"\nfi\nvm-command \"kubectl delete pod pod2 --now --wait --ignore-not-found\"\n\n# pod2: create dynamically the first dynamictwo balloon\nCPUREQ=\"800m\" CPULIM=\"800m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamictwo\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod2c0\"]) == 1'\n\n# pod3: create dynamically the second dynamictwo balloon\nCPUREQ=\"600m\" CPULIM=\"600m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamictwo\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod3c0\"])'\n\n# pod4: prefering new balloon fails, but this fits in the second dynamictwo balloon\nCPUREQ=\"300m\" CPULIM=\"300m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamictwo\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'cpus[\"pod4c0\"] == cpus[\"pod3c0\"]'\n\n# pod5: prefering new balloon fails, and fitting to existing dynamictwo balloons fails\nCPUREQ=\"300m\" CPULIM=\"300m\"\n( POD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamictwo\" CONTCOUNT=1 wait_t=5s create balloons-busybox ) && {\n    error \"creating pod6 succeeded but was expected to fail with balloon allocation error\"\n}\nvm-command \"kubectl describe pod pod5\"\nif ! grep -q 'no suitable balloon instance available' <<< \"$COMMAND_OUTPUT\"; then\n    error \"could not find 'no suitable balloon instance available' in pod6 description\"\nfi\nvm-command \"kubectl delete pod pod5 --now --wait --ignore-not-found\"\n\ncleanup\n\n# Try starting cri-resmgr with a configuration where MinBalloons and\n# MaxBalloons of the same balloon type contradict.\nterminate cri-resmgr\n( cri_resmgr_cfg=${TEST_DIR}/balloons-maxballoons-impossible.cfg launch cri-resmgr ) && {\n    error \"starting cri-resmgr succeeded, but was expected to fail due to impossible static balloons\"\n}\necho \"starting cri-resmgr with impossible static balloons configuration failed as expected\"\n\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test08-numa/balloons-numa.cfg",
    "content": "policy:\n  Active: balloons\n  AvailableResources:\n    CPU: cpuset:0-15\n  # Reserve one of our CPUs (cpu15) for kube-system tasks.\n  ReservedResources:\n    CPU: 1\n  balloons:\n    PinCPU: true\n    PinMemory: true\n    BalloonTypes:\n      - Name: fit-in-numa\n        # All (non-system) containers are assigned to this balloon\n        # type\n        Namespaces:\n          - \"*\"\n        # Prevent a balloon to be inflated larger than a NUMA node\n        MinCPUs: 0\n        MaxCPUs: 4\n        AllocatorPriority: 0\n        PreferNewBalloons: false\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test08-numa/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-numa.cfg launch cri-resmgr\n\n# pod0: besteffort, make sure it still gets at least 1 CPU\nCPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1'\n\n# pod1: guaranteed, make sure it gets the CPU it requested.\n# The configuration does not prefer creating new balloons,\n# so pod0 and pod1 should be placed in the same balloon.\n# Sum of their CPU requests is 1, so they should actually\n# run on the same CPU.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod1c0\"]'\n\n# pod2: guaranteed, make sure it gets the CPU it requested.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 2' \\\n       'len(cpus[\"pod1c0\"]) == 2' \\\n       'len(cpus[\"pod2c0\"]) == 2' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"]'\n\n# pod3: guaranteed, make sure it gets the CPU it requested.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 3' \\\n       'len(cpus[\"pod1c0\"]) == 3' \\\n       'len(cpus[\"pod2c0\"]) == 3' \\\n       'len(cpus[\"pod3c0\"]) == 3' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"]'\n\n# pod4: guaranteed, fill up a balloon to the MaxCPU\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 4' \\\n       'len(cpus[\"pod1c0\"]) == 4' \\\n       'len(cpus[\"pod2c0\"]) == 4' \\\n       'len(cpus[\"pod3c0\"]) == 4' \\\n       'len(cpus[\"pod4c0\"]) == 4' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"] == cpus[\"pod4c0\"]'\n\n# pod5: besteffort, no CPU request, should fit into the full balloon\nCPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 4' \\\n       'len(cpus[\"pod1c0\"]) == 4' \\\n       'len(cpus[\"pod2c0\"]) == 4' \\\n       'len(cpus[\"pod3c0\"]) == 4' \\\n       'len(cpus[\"pod4c0\"]) == 4' \\\n       'len(cpus[\"pod5c0\"]) == 4' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"] == cpus[\"pod4c0\"] == cpus[\"pod5c0\"]'\n\n# pod6: guaranteed, start filling new balloon\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 4' \\\n       'len(cpus[\"pod1c0\"]) == 4' \\\n       'len(cpus[\"pod2c0\"]) == 4' \\\n       'len(cpus[\"pod3c0\"]) == 4' \\\n       'len(cpus[\"pod4c0\"]) == 4' \\\n       'len(cpus[\"pod5c0\"]) == 4' \\\n       'len(cpus[\"pod6c0\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"] == cpus[\"pod4c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod6c0\"])'\n\n# Leave only one guaranteed container to the first balloon.\nkubectl delete pods pod1 pod2 pod3 --now --wait --ignore-not-found\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod4c0\"]) == 1' \\\n       'len(cpus[\"pod5c0\"]) == 1' \\\n       'len(cpus[\"pod6c0\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod4c0\"] == cpus[\"pod5c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod6c0\"])'\n\n# Leave only bestefforts to the first balloon. Make sure they still\n# have a CPU.\nkubectl delete pods pod4 --now --wait --ignore-not-found\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod5c0\"]) == 1' \\\n       'len(cpus[\"pod6c0\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod5c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod6c0\"])'\n\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test09-isolated/balloons-isolated.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: cpuset:0\n  balloons:\n    BalloonTypes:\n      - Name: isolated-pods\n        MinCPUs: 0\n        MaxCPUs: 2\n        CPUClass: turbo\n        MinBalloons: 2\n        PreferNewBalloons: true\n        PreferSpreadingPods: false\n      - Name: isolated-ctrs\n        MinCPUs: 1\n        MaxCPUs: 1\n        CPUClass: turbo\n        MinBalloons: 2\n        PreferNewBalloons: true\n        PreferSpreadingPods: true\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test09-isolated/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-isolated.cfg cri_resmgr_extra_args=\"-metrics-interval 4s\" launch cri-resmgr\n\nverify-metrics-has-line 'balloon=\"isolated-pods\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[1\\]\"'\nverify-metrics-has-no-line 'balloon=\"isolated-pods\\[2\\]\"'\n\n# pod0: besteffort\nCPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: isolated-pods\"\nCONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod0c1\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]'\n# Even if the isolated balloon type has PreferNewBalloons=1, adding\n# this pod0 or pod1 must not create a new balloon because existing\n# empty balloons should be filled first.\nverify-metrics-has-line 'balloon=\"isolated-pods\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[1\\]\"'\nverify-metrics-has-no-line 'balloon=\"isolated-pods\\[2\\]\"'\n\n# pod1: guaranteed\nCPUREQ=\"600m\" CPULIM=\"600m\" MEMREQ=\"100M\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: isolated-pods\"\nCONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod0c1\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 2' \\\n       'len(cpus[\"pod1c1\"]) == 2' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"])'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[1\\]\"'\nverify-metrics-has-no-line 'balloon=\"isolated-pods\\[2\\]\"'\n\n# pod2: burstable\nCPUREQ=\"100m\" CPULIM=\"200m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: isolated-pods\"\nCONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod0c1\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 2' \\\n       'len(cpus[\"pod1c1\"]) == 2' \\\n       'len(cpus[\"pod2c0\"]) == 1' \\\n       'len(cpus[\"pod2c1\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"])'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[1\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[2\\]\"'\nverify-metrics-has-no-line 'balloon=\"isolated-pods\\[3\\]\"'\n\n# pod3: isolated containers\nCPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: isolated-ctrs\"\nCONTCOUNT=4 create balloons-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod0c1\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 2' \\\n       'len(cpus[\"pod1c1\"]) == 2' \\\n       'len(cpus[\"pod2c0\"]) == 1' \\\n       'len(cpus[\"pod2c1\"]) == 1' \\\n       'len(cpus[\"pod3c0\"]) == 1' \\\n       'len(cpus[\"pod3c1\"]) == 1' \\\n       'len(cpus[\"pod3c2\"]) == 1' \\\n       'len(cpus[\"pod3c3\"]) == 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"])' \\\n       'disjoint_sets(cpus[\"pod3c0\"], cpus[\"pod3c1\"], cpus[\"pod3c2\"], cpus[\"pod3c3\"])' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"], cpus[\"pod3c0\"], cpus[\"pod3c1\"], cpus[\"pod3c2\"], cpus[\"pod3c3\"])'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[1\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-pods\\[2\\]\"'\nverify-metrics-has-no-line 'balloon=\"isolated-pods\\[3\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-ctrs\\[0\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-ctrs\\[1\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-ctrs\\[2\\]\"'\nverify-metrics-has-line 'balloon=\"isolated-ctrs\\[3\\]\"'\n\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/balloons-allocator-opts.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    CPU: 1\n  balloons:\n    AllocatorTopologyBalancing: true\n    PreferSpreadOnPhysicalCores: true\n    BalloonTypes:\n      - Name: policydefaults\n        MinCPUs: 2\n        MinBalloons: 2\n      - Name: topo1cores0\n        MinCPUs: 2\n        MinBalloons: 2\n        PreferSpreadOnPhysicalCores: false\n      - Name: topo0cores1\n        AllocatorTopologyBalancing: false\n        PreferSpreadOnPhysicalCores: true\n      - Name: topo0cores0\n        AllocatorTopologyBalancing: false\n        PreferSpreadOnPhysicalCores: false\n      - Name: topo1cores1\n        AllocatorTopologyBalancing: true\n        PreferSpreadOnPhysicalCores: true\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/test10-allocator-opts/code.var.sh",
    "content": "cleanup() {\n    vm-command \"kubectl delete pods --all --now --wait\"\n    return 0\n}\n\ncleanup\n\n# Launch cri-resmgr with wanted metrics update interval and a\n# configuration that opens the instrumentation http server.\nterminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-allocator-opts.cfg launch cri-resmgr\n\n# pod0 in a 2-CPU balloon\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: policydefaults\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cores[\"pod0c0\"]) == 2' \\\n       'len(cpus[\"pod0c0\"]) == 2'\n\n\n# pod1 in a 2-CPU balloon\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: topo1cores0\" CONTCOUNT=1 create balloons-busybox\nreport allowed\nverify 'len(cores[\"pod1c0\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 2'\n\n# pod2: container 0 resizes first from 0 to 1, container 2 from 1 to 2 CPUs,\n# use more cores\nCPUREQ=\"1\" MEMREQ=\"100M\" CPULIM=\"1\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: topo1cores1\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'len(cores[\"pod2c0\"]) == 2' \\\n       'len(cpus[\"pod2c0\"]) == 2' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]'\n\n# pod3: container 0 resizes first from 0 to 1, container 2 from 1 to 2 CPUs,\n# pack tightly\nCPUREQ=\"1\" MEMREQ=\"100M\" CPULIM=\"1\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: topo0cores0\" CONTCOUNT=2 create balloons-busybox\nreport allowed\nverify 'len(cores[\"pod3c0\"]) == 1' \\\n       'len(cpus[\"pod3c0\"]) == 2' \\\n       'cpus[\"pod3c0\"] == cpus[\"pod3c1\"]'\n\ncleanup\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c32/test01-dynamic-baloons/balloons-dynamic.cfg",
    "content": "policy:\n  Active: balloons\n  ReservedResources:\n    cpu: cpuset:31\n  balloons:\n    AllocatorTopologyBalancing: true\n    BalloonTypes:\n      - Name: dynamic\n        MaxCPUs: 32\n        MaxBalloons: 8\n        PreferNewBalloons: true\n        ShareIdleCpusInSame: numa\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c32/test01-dynamic-baloons/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/balloons-dynamic.cfg cri_resmgr_extra_args=\"-metrics-interval 4s\" launch cri-resmgr\n\n# pod0-pod7: create 8 balloons, where each lands on a different NUMA node.\n# Each balloon (except one that lands on the NUMA node with reserved CPUs)\n# has 1 shared CPU at the most since a NUMA node has 4 CPUs and a pod is\n# requesting 1 CPU. Only one of the balloon that using NUMA node with\n#reserved CPU has 0 shared CPUs.\nCPUREQLIM=\"3\"\nINITCPUREQLIM=\"100m-100m 100m-100m 100m-100m\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamic\"\nn=8 create multicontainerpod\nverify-metrics-has-line 'balloon=\"dynamic\\[0\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[1\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[2\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[3\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[4\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[5\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[6\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-line 'balloon=\"dynamic\\[7\\]\".*cpus_count=\"3\"*'\nverify-metrics-has-no-line 'cpus_count=\"4\"'\nverify-metrics-has-line 'sharedidlecpus_count=\"1\"'\nverify-metrics-has-line 'cpus_allowed_count=\"4\"'\nverify-metrics-has-line 'sharedidlecpus_count=\"0\"'\nverify-metrics-has-line 'cpus_allowed_count=\"3\"'\nverify-metrics-has-no-line 'sharedidlecpus_count=\"2\"'\nverify-metrics-has-no-line 'cpus_allowed_count=\"5\"'\nverify 'disjoint_sets(nodes[\"pod0c0\"], nodes[\"pod1c0\"], nodes[\"pod2c0\"], nodes[\"pod3c0\"], nodes[\"pod4c0\"], nodes[\"pod5c0\"], nodes[\"pod6c0\"], nodes[\"pod7c0\"])' \\\n       'len(nodes[\"pod0c0\"]) == len(nodes[\"pod1c0\"]) == len(nodes[\"pod2c0\"]) == \\\n        len(nodes[\"pod3c0\"]) == len(nodes[\"pod4c0\"]) == len(nodes[\"pod5c0\"]) == \\\n        len(nodes[\"pod6c0\"]) == len(nodes[\"pod7c0\"]) == 1'\n\n# pod8: Add one more pod with 2 CPUs to inflate over NUMAs nodes, which should cross\n# the NUMA node boundaries but not the die boundaries. Because two NUMA nodes can offer\n# 2 CPUs in total. \nCPUREQLIM=\"2\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamic\"\ncreate multicontainerpod\nverify-metrics-has-line 'cpus_count=\"5\"'\nverify-metrics-has-line 'sharedidlecpus=\"\",sharedidlecpus_count=\"0\"'\nverify-metrics-has-line 'sharedidlecpus_count=\"1\"'\nverify 'len(nodes[\"pod8c0\"])==2' \\\n       'len(dies[\"pod8c0\"])==1' \\\n       'len(packages[\"pod8c0\"])==1'\nkubectl delete pod pod8 --now --wait --ignore-not-found\nverify-metrics-has-no-line 'cpus_count=\"5\"'\n\n# pod9: Add one more pod with 4 CPUs to inflate over dies, which should cross\n# the NUMA node boundaries as well as dies boundaries. Since 2 dies under the\n# same package can offer 4 CPUs, we should not cross the package boundaries.\nCPUREQLIM=\"4\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamic\"\ncreate multicontainerpod\nverify 'len(nodes[\"pod9c0\"])==4' \\\n       'len(dies[\"pod9c0\"])==2' \\\n       'len(packages[\"pod9c0\"])==1'\nkubectl delete pod pod9 --now --wait --ignore-not-found\nverify 'disjoint_sets(nodes[\"pod0c0\"], nodes[\"pod1c0\"], nodes[\"pod2c0\"], nodes[\"pod3c0\"], nodes[\"pod4c0\"], nodes[\"pod5c0\"], nodes[\"pod6c0\"], nodes[\"pod7c0\"])' \\\n\n# pod9: Add one more pod with 7 CPUs to inflate over packages, which should cross\n# NUMA node, dies and package boundaries. At this point, there is no free CPUs\n# left on the host, so no shared CPUs.\nCPUREQLIM=\"6 1\"\nPOD_ANNOTATION=\"balloon.balloons.cri-resource-manager.intel.com: dynamic\"\ncreate multicontainerpod\nverify 'len(nodes[\"pod10c0\"])==7' \\\n       'len(dies[\"pod10c0\"])==4' \\\n       'len(packages[\"pod10c0\"])==2'\nverify-metrics-has-line 'sharedidlecpus=\"\",sharedidlecpus_count=\"0\"'\nverify-metrics-has-no-line 'sharedidlecpus_count=\"1\"'\n\n# pod0, pod9 deflate. This should free up 10 CPUs that will cause having\n# shared CPUs available again.\nkubectl delete pod pod10 --now --wait --ignore-not-found\nkubectl delete pod pod0 --now --wait --ignore-not-found\nverify-metrics-has-line 'sharedidlecpus_count=\"1\"'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c32/test01-dynamic-baloons/multicontainerpod.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"$POD_ANNOTATION\" ]; then echo \"\n  annotations:\n    $POD_ANNOTATION\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(contnum=0; for reqlim in ${CPUREQLIM}; do echo \"\n  - name: ${NAME}c${contnum}\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - ${WORK}echo ${NAME}c${contnum} \\$(sleep inf)\n    $(if [ -n \"${reqlim}\" ]; then echo \"\n    resources:\n      $(if [ -n \"${reqlim/-*}\" ]; then echo \"\n      requests:\n        cpu: ${reqlim/-*/}\n      \"; fi)\n      $(if [ -n \"${reqlim/*-/}\" ]; then echo \"\n      limits:\n        cpu: ${reqlim/*-}\n      \"; fi)\n    \"; fi)\n  \"; contnum=$((contnum + 1)); done )\n  $(if [ -n \"$INITCPUREQLIM\" ]; then echo \"\n  initContainers:\n  $(contnum=0; for initreqlim in ${INITCPUREQLIM}; do echo \"\n  - name: ${NAME}c${contnum}-init\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - ${WORK}echo ${NAME}c${contnum}-init \\$(sleep 1)\n    $(if [ -n \"${initreqlim}\" ]; then echo \"\n    resources:\n      $(if [ -n \"${initreqlim/-*}\" ]; then echo \"\n      requests:\n        cpu: ${initreqlim/-*/}\n      \"; fi)\n      $(if [ -n \"${initreqlim/*-/}\" ]; then echo \"\n      limits:\n        cpu: ${initreqlim/*-}\n      \"; fi)\n    \"; fi)\n  \"; contnum=$((contnum + 1)); done )\n  \"; fi)\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/n4c32/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"dies\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/policies.test-suite/balloons/verify.source.sh",
    "content": "# Utilities to verify data from metrics\n\nverify_metrics_url=\"http://localhost:8891/metrics\"\n\nverify-metrics-has-line() {\n    local expected_line=\"$1\"\n    vm-run-until --timeout 10 \"echo 'waiting for metrics line: $expected_line' >&2; curl --silent $verify_metrics_url | grep -E '$expected_line'\" || {\n        command-error \"expected line '$1' missing from the output\"\n    }\n}\n\nverify-metrics-has-no-line() {\n    local unexpected_line=\"$1\"\n    vm-run-until --timeout 10 \"echo 'checking absense of metrics line: $unexpected_line' >&2; ! curl --silent $verify_metrics_url | grep -Eq '$unexpected_line'\" || {\n        command-error \"unexpected line '$1' found from the output\"\n    }\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/check-correct-policy.source.sh",
    "content": "# This script does a policy check before the real test code is started.\n\ncache_policy=\"$(vm-command-q \"cat /var/lib/cri-resmgr/cache\" | jq -r .PolicyName)\"\n\ncfg_policy=$(awk '/Active:/{print $2}' < \"$cri_resmgr_cfg\")\n\nif [ -n \"$cache_policy\" ] && [ -n \"$cfg_policy\" ] && [ \"$cache_policy\" != \"$cfg_policy\" ]; then\n    echo \"cri-resmgr is been started with policy \\\"$cache_policy\\\", switching to \\\"$cfg_policy\\\"\"\n    terminate cri-resmgr\n    echo \"destroying cri-resmgr cache with previous policy\"\n    vm-command \"rm -rf /var/lib/cri-resmgr\"\n    launch cri-resmgr\nfi\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/cri-resmgr.cfg",
    "content": "policy:\n  Active: dynamic-pools\n  # Use only 15 CPUs in total, leave cpu0 for other than Kubernetes\n  # processes.\n  AvailableResources:\n    CPU: cpuset:1-15\n  # Reserve one of our CPUs for kube-system tasks.\n  ReservedResources:\n    CPU: 1\n  dynamic-pools:\n    PinCPU: true\n    PinMemory: true\n    DynamicPoolTypes:\n      - Name: \"pool1\"\n        Namespaces:\n          - \"pool1\"\n        CPUClass: \"pool1-cpuclass\"\n      - Name: \"pool2\"\n        Namespaces:\n          - \"pool2\"\n        CPUClass: \"pool2-cpuclass\"\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\ncpu:\n  classes:\n    default:\n      minFreq: 800\n      maxFreq: 2800\n    pool1-cpuclass:\n      minFreq: 900\n      maxFreq: 2900\n    pool2-cpuclass:\n      minFreq: 1000\n      maxFreq: 3000"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/dyp-busybox.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"$POD_ANNOTATION\" ]; then echo \"\n  annotations:\n    $POD_ANNOTATION\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - ${WORK}echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    $(if [ -n \"${CPUREQ}\" ]; then echo \"\n    resources:\n      requests:\n        cpu: ${CPUREQ}\n        $(if [ -n \"${MEMREQ}\" ]; then echo \"\n        memory: '${MEMREQ}'\n        \"; fi)\n      $(if [ -n \"${CPULIM}\" ]; then echo \"\n      limits:\n        cpu: ${CPULIM}\n        $(if [ -n \"$MEMLIM\" ]; then echo \"\n        memory: '${MEMLIM}'\n        \"; fi)\n    \"; fi)\n    \"; fi)\n  \"; done )\n  terminationGracePeriodSeconds: 1"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/dyp-configmap.yaml.in",
    "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.default\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: dynamic-pools\n    AvailableResources:\n      CPU: ${AVAILABLE_CPU:-cpuset:0-15}\n    ReservedResources:\n      CPU: ${RESERVED_CPU:-1}\n\n    dynamic-pools:\n      PinCPU: ${PINCPU:-true}\n      PinMemory: ${PINMEMORY:-true}\n      DynamicPoolTypes:\n\n        $([ -n \"$DYPTYPE0_SKIP\" ] || echo \"\n        - Name: dyptype0\n          AllocatorPriority: ${DYPTYPE0_ALLOCATORPRIORITY:-0}\n          CPUClass: ${DYPTYPE0_CPUCLASS:-classA}\n        \")\n\n        $([ -n \"$DYPTYPE1_SKIP\" ] || echo \"\n        - Name: dyptype1\n          Namespaces:\n            - ${DYPTYPE1_NAMESPACE0:-dyptype1ns0}\n          AllocatorPriority: ${DYPTYPE1_ALLOCATORPRIORITY:-1}\n          CPUClass: ${DYPTYPE1_CPUCLASS:-classB}\n        \")\n\n        $([ -n \"$DYPTYPE2_SKIP\" ] || echo \"\n        - Name: dyptype2\n          Namespaces:\n            - ${DYPTYPE2_NAMESPACE0:-dyptype2ns0}\n            - ${DYPTYPE2_NAMESPACE1:-dyptype2ns1}\n          AllocatorPriority: ${DYPTYPE2_ALLOCATORPRIORITY:-2}\n          CPUClass: ${DYPTYPE2_CPUCLASS:-classC}\n        \")\n\n  instrumentation: |+\n    HTTPEndpoint: :8891\n    PrometheusExport: true\n\n  logger: |+\n    Debug: policy\n\n  cpu: |+\n    classes:\n      default:\n        minFreq: ${CPU_DEFAULT_MIN:-800}\n        maxFreq: ${CPU_DEFAULT_MAX:-2800}\n      classA:\n        minFreq: ${CPU_CLASSA_MIN:-900}\n        maxFreq: ${CPU_CLASSA_MAX:-2900}\n      classB:\n        minFreq: ${CPU_CLASSB_MIN:-1000}\n        maxFreq: ${CPU_CLASSB_MAX:-3000}\n      classC:\n        minFreq: ${CPU_CLASSC_MIN:-1100}\n        maxFreq: ${CPU_CLASSC_MAX:-3100}\n        energyPerformancePreference: ${CPU_CLASSC_EPP:-1}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test01-basic-placement/code.var.sh",
    "content": "# Test placing containers with and without annotations to correct dynamic pools\n# reserved and shared CPUs.\n\ncleanup() {\n    vm-command \"kubectl delete pods pod0 -n kube-system; kubectl delete pods -n pool1 --all --now; kubectl delete pods --all --now; kubectl delete namespace pool1\"\n    return 0\n}\n\ncleanup\n\nterminate cri-resmgr\n\nlaunch cri-resmgr\n\n# pod0: run on reserved CPUs.\nnamespace=kube-system CONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 1'\n\n# pod1: run in shared dynamic pool.\n# We do not add annotations to this pod, and we do not set any\n# namespace, so this pod is expected to be created to the shared pool.\ncreate dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod1c0\"]) == 14'\n\n# The size of each dynamic pool is obtained by adding the requests of the containers in this pool and the CPUs allocated based on cpu utilization,\n# so the size of each dynamic pool is greater than or equal to the sum of the requests of the containers in the pool.\n\n# pod2: run in the pool1.\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: pool1\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod2c0\"]) >= 1' \\\n      'len(cpus[\"pod1c0\"]) + len(cpus[\"pod2c0\"]) == 14' \\\n      'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod1c0\"])'\n\n# pod3: run in the pool1.\nCPUREQ=\"1500m\" MEMREQ=\"100M\" CPULIM=\"1500m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: pool1\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod2c0\"] == cpus[\"pod3c0\"]' \\\n      'len(cpus[\"pod3c0\"]) >= 2' \\\n      'len(cpus[\"pod1c0\"]) + len(cpus[\"pod3c0\"]) == 14' \\\n      'disjoint_sets(cpus[\"pod1c0\"], cpus[\"pod3c0\"])'\n\n# pod4: run in the pool2.\nCPUREQ=\"1500m\" MEMREQ=\"100M\" CPULIM=\"1500m\" MEMLIM=\"100M\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: pool2\" CONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod4c0\"] == cpus[\"pod4c1\"]' \\\n      'len(cpus[\"pod4c0\"]) >= 3' \\\n      'len(cpus[\"pod3c0\"]) >= 2' \\\n      'len(cpus[\"pod1c0\"]) + len(cpus[\"pod3c0\"]) + len(cpus[\"pod4c0\"]) == 14' \\\n      'disjoint_sets(cpus[\"pod4c0\"], cpus[\"pod3c0\"], cpus[\"pod1c0\"])'\n\n# pod5: run in the pool1.\nCPUREQ=\"1500m\" MEMREQ=\"100M\" CPULIM=\"1500m\" MEMLIM=\"100M\"\nkubectl create namespace \"pool1\"\nnamespace=\"pool1\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod5c0\"] == cpus[\"pod2c0\"]'\\\n      'len(cpus[\"pod5c0\"]) >= 4' \\\n      'len(cpus[\"pod4c0\"]) >= 3' \\\n      'len(cpus[\"pod1c0\"]) + len(cpus[\"pod3c0\"]) + len(cpus[\"pod4c0\"]) == 14'\n\ncleanup\n\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test02-prometheus-metrics/code.var.sh",
    "content": "# This test verifies prometheus metrics from the dynamic-pools policy.\n\ncleanup() {\n    vm-command \"kubectl delete pods --all --now\"\n    terminate cri-resmgr\n    terminate cri-resmgr-agent\n    vm-command \"cri-resmgr -reset-policy; cri-resmgr -reset-config\"\n    return 0\n}\n\ncleanup\n\n# Launch cri-resmgr with wanted metrics update interval and a\n# configuration that opens the instrumentation http server.\ncri_resmgr_cfg=${TEST_DIR}/dyp-metrics.cfg  cri_resmgr_extra_args=\"-metrics-interval 1s\" launch cri-resmgr\nsleep 10\nverify-metrics-has-line 'dynamicPool=\"shared\"'\nverify-metrics-has-line 'dynamicPool=\"reserved\"'\nverify-metrics-has-line 'dynamicPool=\"full-core\"'\nverify-metrics-has-line 'dynamicPool=\"flex\"'\nverify-metrics-has-line 'dynamicPool=\"fast-dualcore\"'\n\n# pod0: run in shared dynamic pool.\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nCONTCOUNT=2 create dyp-busybox\nreport allowed\nverify-metrics-has-line 'dynamicPool=\"reserved\"'\nverify-metrics-has-line 'dynamicPool=\"full-core\"'\nverify-metrics-has-line 'dynamicPool=\"flex\"'\nverify-metrics-has-line 'dynamicPool=\"fast-dualcore\"'\nverify-metrics-has-line 'DynamicPools{containers=\"pod0:pod0c0,pod0:pod0c1\",cpu_class=\"\",cpus=\".*\",dynamicPool=\"shared\",dynamicPool_type=\"shared\",mems=\".*\",tot_limit_millicpu=\"200\",tot_req_millicpu=\"200\"} 15'\n\n# pod1: run in fast-dualcore dynamic pool.\nCPUREQ=\"200m\" MEMREQ=\"\" CPULIM=\"200m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: fast-dualcore\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify-metrics-has-line 'containers=\"pod1:pod1c0\".*dynamicPool=\"fast-dualcore\",dynamicPool_type=\"fast-dualcore\".*tot_req_millicpu=\"(199|200)\"'\nverify 'len(cpus[\"pod1c0\"]) >= 1'\n\n# pod2: run in flex dynamic pool.\nCPUREQ=\"3500m\" MEMREQ=\"\" CPULIM=\"3500m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: flex\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify-metrics-has-line 'containers=\"pod2:pod2c0\".*dynamicPool=\"flex\",dynamicPool_type=\"flex\"'\nverify 'len(cpus[\"pod2c0\"]) >= 4'\n\n# pod3: run in flex dynamic pool.\nCPUREQ=\"1200m\" MEMREQ=\"\" CPULIM=\"1200m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: flex\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify-metrics-has-line 'containers=\"pod2:pod2c0,pod3:pod3c0\".*dynamicPool=\"flex\",dynamicPool_type=\"flex\"'\nverify 'len(cpus[\"pod2c0\"]) >= 5'\n\n# Resize flex dynamic pool in metrics.\nkubectl delete pods --now pod3\nverify-metrics-has-line 'containers=\"pod2:pod2c0\".*dynamicPool=\"flex\",dynamicPool_type=\"flex\"'\nverify 'len(cpus[\"pod2c0\"]) >= 4'\n\nkubectl delete pods --now pod2\nsleep 5\nverify-metrics-has-line 'containers=\"\".*dynamicPool=\"flex\",dynamicPool_type=\"flex\".*0'\n\n# Delete all pods in shared dynamic pool.\nkubectl delete pods --now pod0\n# pod4: run in fast-dualcore dynamic pool, all CPUs are allocated to fast-dualcore dynamic pool.\nCPUREQ=\"14\" MEMREQ=\"\" CPULIM=\"14\" MEMLIM=\"\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: fast-dualcore\" CONTCOUNT=1 create dyp-busybox\nreport allowed\nverify-metrics-has-line 'containers=\"pod1:pod1c0,pod4:pod4c0\".*dynamicPool=\"fast-dualcore\",dynamicPool_type=\"fast-dualcore\".*15'\nverify 'len(cpus[\"pod1c0\"]) == 15'\n\ncleanup\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test02-prometheus-metrics/dyp-metrics.cfg",
    "content": "policy:\n  Active: dynamic-pools\n  AvailableResources:\n    CPU: cpuset:0-15\n  # Reserve one of our CPUs for kube-system tasks.\n  ReservedResources:\n    CPU: cpuset:0\n  dynamic-pools:\n    DynamicPoolTypes:\n      - Name: full-core\n        CPUClass: normal\n\n      - Name: fast-dualcore\n        CPUClass: turbo\n\n      - Name: flex\n        CPUClass: slow\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test03-rebalancing/code.var.sh",
    "content": "# Re-launch cri-resmgr with the rebalancing parameter in order to\n# enable rebalancing calls. (See help of the \"launch\" function for\n# more options.)\n\ncleanup() {\n    vm-command \"kubectl delete pods --all --now\"\n    return 0\n}\n\ncleanup\nterminate cri-resmgr\ncri_resmgr_extra_args=\"-metrics-interval 1s -rebalance-interval 2s\" launch cri-resmgr\nsleep 10\n\n# Create three pods:\n# - pod0 to \"shared\"\n# - pod1 to \"pool1\"\n# - pod2 to \"pool2\"\ncreate dyp-busybox\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: pool1\"\ncreate dyp-busybox\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: pool2\"\ncreate dyp-busybox\n# Print initial CPU pinning.\nreport allowed\n# Wait at least one rebalancing round.\nsleep 3\nverify 'len(cpus[\"pod0c0\"]) >= 1'\nverify 'len(cpus[\"pod1c0\"]) >= 1'\nverify 'len(cpus[\"pod2c0\"]) >= 1'\nverify-metrics-has-line 'containers=\"pod0:pod0c0\".*dynamicPool=\"shared\",dynamicPool_type=\"shared\"'\nverify-metrics-has-line 'containers=\"pod1:pod1c0\".*dynamicPool=\"pool1\",dynamicPool_type=\"pool1\"'\nverify-metrics-has-line 'containers=\"pod2:pod2c0\".*dynamicPool=\"pool2\",dynamicPool_type=\"pool2\"'\n\n# Increase CPU usage of pod1 to 200%\nvm-command \"nohup kubectl exec pod1 -- /bin/sh -c 'gzip </dev/zero >/dev/null' </dev/null >&/dev/null &\"\nvm-command \"nohup kubectl exec pod1 -- /bin/sh -c 'gzip </dev/zero >/dev/null' </dev/null >&/dev/null &\"\n# Wait at least one rebalancing round and print CPU pinning.\nsleep 10\nreport allowed\n# Now \"pool1\" has 200% CPU load, \"shared\" and \"pool2\" have 0%.\n# Verify that the number of CPUs in pool1 is the largest.\nverify 'len(cpus[\"pod1c0\"]) > len(cpus[\"pod0c0\"])'\nverify 'len(cpus[\"pod1c0\"]) > len(cpus[\"pod2c0\"])'\nverify 'len(cpus[\"pod0c0\"]) + len(cpus[\"pod1c0\"]) + len(cpus[\"pod2c0\"]) == 14'\n\n# Remove CPU load from pool1 and put 100% CPU load to pool2.\nvm-command \"pkill gzip\"\nvm-command \"nohup kubectl exec pod2 -- /bin/sh -c 'gzip </dev/zero >/dev/null' </dev/null >&/dev/null &\"\n# Wait at least one rebalancing round and print CPU pinning.\nsleep 10\nreport allowed\n# Verify that the number of CPUs in pool2 is the largest.\nverify 'len(cpus[\"pod2c0\"]) > len(cpus[\"pod0c0\"])'\nverify 'len(cpus[\"pod2c0\"]) > len(cpus[\"pod1c0\"])'\nverify 'len(cpus[\"pod0c0\"]) + len(cpus[\"pod1c0\"]) + len(cpus[\"pod2c0\"]) == 14'\n\n# Remove CPU load from pool1 and put 100% CPU load to pool2 and pool1.\nvm-command \"pkill gzip\"\nvm-command \"nohup kubectl exec pod1 -- /bin/sh -c 'gzip </dev/zero >/dev/null' </dev/null >&/dev/null &\"\nvm-command \"nohup kubectl exec pod2 -- /bin/sh -c 'gzip </dev/zero >/dev/null' </dev/null >&/dev/null &\"\n# Takes time to reach a state of balance\nsleep 10\nreport allowed\n# Verify that the number of CPUs in pool1 is greater than or equal to 6 and less than or equal to 8.\n# Verify that the number of CPUs in pool2 is greater than or equal to 6 and less than or equal to 8.\nverify 'len(cpus[\"pod0c0\"]) == 1'\nverify 'len(cpus[\"pod1c0\"]) >= 6'\nverify 'len(cpus[\"pod1c0\"]) <= 8'\nverify 'len(cpus[\"pod2c0\"]) >= 6'\nverify 'len(cpus[\"pod2c0\"]) <= 8'\n\n# Remove CPU load from pool1 and pool2\nvm-command \"pkill gzip\"\n\ncleanup"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test04-reserved/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/dyp-reserved.cfg launch cri-resmgr\n\ncleanup() {\n    vm-command \\\n        \"kubectl delete pod -n kube-system --now pod0\n         kubectl delete pod -n monitor-mypods --now pod1\n         kubectl delete pod -n system-logs --now pod2\n         kubectl delete pod -n kube-system --now pod3\n         kubectl delete pods --now pod4 pod5 pod6\n         kubectl delete pod -n kube-system --now pod7\n         kubectl delete namespace monitor-mypods\n         kubectl delete namespace system-logs\n         kubectl delete namespace my-exact-name\"\n    return 0\n}\n\ncleanup\n\nkubectl create namespace monitor-mypods\nkubectl create namespace system-logs\nkubectl create namespace my-exact-name\n\n# pod0: kube-system\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=kube-system create dyp-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == {\"cpu00\", \"cpu01\", \"cpu02\"}'\n\n# pod1: match first ReservedPoolNamespaces glob, multicontainer\nCPUREQ=\"1\" MEMREQ=\"\" CPULIM=\"1\" MEMLIM=\"\"\nnamespace=monitor-mypods CONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod1c0\"] == cpus[\"pod0c0\"]' \\\n       'cpus[\"pod1c1\"] == cpus[\"pod0c0\"]'\n\n# pod2: match last ReservedPoolNamespaces glob, slightly overbook reserved CPU\nCPUREQ=\"1\" MEMREQ=\"\" CPULIM=\"1\" MEMLIM=\"\"\nnamespace=system-logs create dyp-busybox\nreport allowed\nverify 'cpus[\"pod2c0\"] == cpus[\"pod0c0\"]'\n\n# pod3: force a kube-system pod to full-core dynamic pool using an annotation\nCPUREQ=\"2\" MEMREQ=\"\" CPULIM=\"2\" MEMLIM=\"\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: full-core\" namespace=kube-system create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod3c0\"]) >= 2' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod3c0\"])'\n\n# pod4: run in shared dynamic pool\nCPUREQ=\"2500m\" MEMREQ=\"\" CPULIM=\"2500m\" MEMLIM=\"\"\ncreate dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod4c0\"]) >= 3' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod3c0\"], cpus[\"pod4c0\"])'\n\n# pod5: annotate otherwise a default pod to the reserved CPUs,\n# severely overbook reserved CPUs\nCPUREQ=\"2500m\" MEMREQ=\"\" CPULIM=\"2500m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: reserved\" create dyp-busybox\nreport allowed\nverify 'cpus[\"pod5c0\"] == {\"cpu00\", \"cpu01\", \"cpu02\"}' \\\n       'disjoint_sets(cpus[\"pod5c0\"], cpus[\"pod3c0\"], cpus[\"pod4c0\"])'\n\ncleanup\n\n# Now that all pods are deleted, make sure that cpus of reserved and\n# default dynamic pools are as expected.\n\n# pod6: run in shared dynamic pool\nCPUREQ=\"999m\" MEMREQ=\"\" CPULIM=\"999m\" MEMLIM=\"\"\ncreate dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod6c0\"]) >= 1'\n\n# pod7: kube-system\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=kube-system create dyp-busybox\nreport allowed\nverify 'cpus[\"pod7c0\"] == {\"cpu00\", \"cpu01\", \"cpu02\"}'\n\ncleanup\n\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test04-reserved/dyp-reserved.cfg",
    "content": "policy:\n  Active: dynamic-pools\n  ReservedResources:\n    CPU: cpuset:0-2\n  dynamic-pools:\n    PinCPU: true\n    PinMemory: true\n    ReservedPoolNamespaces:\n      - \"monitor-*\"\n      - \"*-log*\"\n    DynamicPoolTypes:\n      - Name: reserved\n        Namespaces:\n          - my-exact-name\n        CPUClass: reserved-class\n      - Name: default\n      - Name: full-core\n        CPUClass: turbo\nlogger:\n  Debug: policy\n  Klog:\n    skip_headers: true"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test05-namespace/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/dyp-namespace.cfg launch cri-resmgr\n\ncleanup() {\n    vm-command \\\n        \"kubectl delete pods -n e2e-a --all --now\n         kubectl delete pods -n e2e-b --all --now\n         kubectl delete pods -n e2e-c --all --now\n         kubectl delete pods -n e2e-d --all --now\n         kubectl delete pods --all --now\n         kubectl delete namespace e2e-a\n         kubectl delete namespace e2e-b\n         kubectl delete namespace e2e-c\n         kubectl delete namespace e2e-d\"\n    return 0\n}\ncleanup\n\nkubectl create namespace e2e-a\nkubectl create namespace e2e-b\nkubectl create namespace e2e-c\nkubectl create namespace e2e-d\n\n# pod0: create in the default namespace, CPUREQ is nil, both containers go to shared dynamic pool.\nCPUREQ=\"\"\nCONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 15'\n\n# pod1: create in the e2e-a namespace, CPUREQ is nil, both containers go to shared dynamic pool.\nCPUREQ=\"\"\nnamespace=\"e2e-a\" CONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod1c0\"] == cpus[\"pod1c1\"] == cpus[\"pod0c0\"]' \\\n       'len(cpus[\"pod1c0\"]) == 15' \\\n\n# pod2: create in the default namespace, CPUREQ is 2*2, both containers go to nsdyp dynamic pool.\nCPUREQ=\"2\" MEMREQ=\"100M\" CPULIM=\"2\" MEMLIM=\"100M\"\nCONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]' \\\n       'len(cpus[\"pod2c0\"]) >= 4' \\\n       'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod1c0\"])' \\\n       'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod0c0\"])'\n\n# pod3: create again in the default namespace, CPUREQ is 200m*2, both containers go to nsdyp dynamic pool.\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nCONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod3c0\"] == cpus[\"pod3c1\"] == cpus[\"pod2c0\"]' \\\n       'len(cpus[\"pod3c0\"]) >= 5'\n\n# pod4: create in the e2e-b namespace, CPUREQ is 2*2, both containers go to nsdyp dynamic pool.\nCPUREQ=\"2\" MEMREQ=\"100M\" CPULIM=\"2\" MEMLIM=\"100M\"\nnamespace=\"e2e-b\" CONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod4c0\"] == cpus[\"pod4c1\"] == cpus[\"pod3c0\"] == cpus[\"pod2c0\"]' \\\n       'len(cpus[\"pod4c0\"]) >= 9'\n\n# pod5: create in the e2e-c namespace, CPUREQ is 100m*2, both containers go to nsdyp dynamic pool.\nCPUREQ=\"100m\" MEMREQ=\"100M\" CPULIM=\"100m\" MEMLIM=\"100M\"\nnamespace=\"e2e-c\" CONTCOUNT=2 create dyp-busybox\nreport allowed\nverify 'cpus[\"pod5c0\"] == cpus[\"pod5c1\"] == cpus[\"pod4c0\"] == cpus[\"pod3c0\"] == cpus[\"pod2c0\"]' \\\n       'len(cpus[\"pod5c0\"]) >= 9'\n\ncleanup\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test05-namespace/dyp-namespace.cfg",
    "content": "policy:\n  Active: dynamic-pools\n  ReservedResources:\n    CPU: 1\n  dynamic-pools:\n    PinCPU: true\n    PinMemory: true\n    DynamicPoolTypes:\n      - Name: nsdyp\n        Namespaces:\n          - \"*\"\nlogger:\n  Debug: policy\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test06-update-configmap/code.var.sh",
    "content": "# This test verifies that configuration updates via cri-resmgr-agent\n# are handled properly in the dynamic-pools policy.\n\ntestns=e2e-dyp-test06\n\ncleanup() {\n    vm-command \"kubectl delete pods --all --now; \\\n        kubectl delete pods -n $testns --all --now; \\\n        kubectl delete pods -n dyptype1ns0 --all --now; \\\n        kubectl delete namespace $testns || :; \\\n        kubectl delete namespace dyptype1ns0 || :\"\n    terminate cri-resmgr\n    terminate cri-resmgr-agent\n    vm-command \"cri-resmgr -reset-policy; cri-resmgr -reset-config\"\n}\n\napply-configmap() {\n    vm-put-file $(instantiate dyp-configmap.yaml) dyp-configmap.yaml\n    vm-command \"cat dyp-configmap.yaml\"\n    kubectl apply -f dyp-configmap.yaml\n}\n\ncleanup\ncri_resmgr_extra_args=\"-metrics-interval 1s\" cri_resmgr_config=fallback launch cri-resmgr\nlaunch cri-resmgr-agent\n\nkubectl create namespace $testns\nkubectl create namespace dyptype1ns0\n\nAVAILABLE_CPU=\"cpuset:1,4-15\" DYPTYPE2_NAMESPACE0='\"*\"' apply-configmap\nsleep 3\n\n# pod0 run in dyptype0, annotation\nCPUREQ=1 MEMREQ=\"100M\" CPULIM=1 MEMLIM=\"100M\"\nPOD_ANNOTATION=\"dynamic-pool.dynamic-pools.cri-resource-manager.intel.com/pod: dyptype0\" create dyp-busybox\n# pod1 run in dyptype1, namespace\nCPUREQ=1 MEMREQ=\"100M\" CPULIM=1 MEMLIM=\"100M\"\nnamespace=\"dyptype1ns0\" create dyp-busybox\n# pod2 run in dyptype2, wildcard namespace\nCPUREQ=1 MEMREQ=\"100M\" CPULIM=1 MEMLIM=\"100M\"\nnamespace=\"e2e-dyp-test06\" create dyp-busybox\nsleep 3\nvm-command \"curl -s $verify_metrics_url\"\nverify-metrics-has-line 'pod0:pod0c0.*\"dyptype0\"'\nverify-metrics-has-line 'pod1:pod1c0.*\"dyptype1\"'\nverify-metrics-has-line 'pod2:pod2c0.*\"dyptype2\"'\n\n# Remove first two dynamic pool types, change dyptype2 to match all\n# namespaces.\nDYPTYPE0_SKIP=1 DYPTYPE1_SKIP=1 DYPTYPE2_NAMESPACE0='\"*\"' apply-configmap\n# Note:\n\n# pod0 was successfully assigned to and running in dyptype0 dynamic pool.\n# Now dyptype0 was completely removed from the node.\n# Currently this behavior is undefined.\n# Possible behaviors: evict pod0, continue assign chain, refuse config...\n# For now, skip pod0c0 dynamic pool validation:\n# verify-metrics-has-line '\"dyptype2\".*pod0:pod0c0'\nverify-metrics-has-line 'pod1:pod1c0.*\"dyptype2\"'\nverify-metrics-has-line 'pod2:pod2c0.*\"dyptype2\"'\n\n# Bring back dyptype0 where pod0 belongs to by annotation.\nDYPTYPE1_SKIP=1 DYPTYPE2_NAMESPACE0='\"*\"' apply-configmap\nverify-metrics-has-line 'pod0:pod0c0.*\"dyptype0\"'\nverify-metrics-has-line 'pod1:pod1c0.*\"dyptype2\"'\nverify-metrics-has-line 'pod2:pod2c0.*\"dyptype2\"'\n\n# Change only CPU classes, no reassigning.\nverify-metrics-has-line 'pod0:pod0c0.*cpu_class=\"classA\".*\"dyptype0\"'\nverify-metrics-has-line 'pod1:pod1c0.*cpu_class=\"classC\".*\"dyptype2\"'\nverify-metrics-has-line 'pod2:pod2c0.*cpu_class=\"classC\".*\"dyptype2\"'\nDYPTYPE0_CPUCLASS=\"classC\" DYPTYPE1_SKIP=1 DYPTYPE2_CPUCLASS=\"classB\" DYPTYPE2_NAMESPACE0='\"*\"'  apply-configmap\nverify-metrics-has-line 'pod0:pod0c0.*cpu_class=\"classC\".*\"dyptype0\"'\nverify-metrics-has-line 'pod1:pod1c0.*cpu_class=\"classB\".*\"dyptype2\"'\nverify-metrics-has-line 'pod2:pod2c0.*cpu_class=\"classB\".*\"dyptype2\"'\n\ncleanup\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test07-numa/code.var.sh",
    "content": "terminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/dyp-numa.cfg launch cri-resmgr\n\n# pod0: besteffort, go to shared dynamic pool, make sure it still gets at least 1 CPU.\nCPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\"\nCONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 15'\n\n# pod1: guaranteed, go to fit-in-numa dynamic pool, make sure it gets the CPU it requested.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod1c0\"]) >= 1' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"])'\n\n# pod2: guaranteed, go to fit-in-numa dynamic pool, make sure it gets the CPU it requested.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod1c0\"]) >= 2' \\\n       'len(cpus[\"pod2c0\"]) >= 2' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod2c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod2c0\"])'\n\n# pod3: guaranteed, go to fit-in-numa dynamic pool, make sure it gets the CPU it requested.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod1c0\"]) >= 3' \\\n       'len(cpus[\"pod2c0\"]) >= 3' \\\n       'len(cpus[\"pod3c0\"]) >= 3' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod3c0\"])'\n\n# pod4: guaranteed, go to fit-in-numa dynamic pool, make sure it gets the CPU it requested.\nCPUREQ=\"1\" CPULIM=\"1\" MEMREQ=\"50M\" MEMLIM=\"50M\"\nCONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod1c0\"]) >= 4' \\\n       'len(cpus[\"pod2c0\"]) >= 4' \\\n       'len(cpus[\"pod3c0\"]) >= 4' \\\n       'len(cpus[\"pod4c0\"]) >= 4' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"] == cpus[\"pod4c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod4c0\"])'\n\n# pod5: besteffort, no CPU request, should fit into the shared dynamic pool.\nCPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\"\nCONTCOUNT=1 create dyp-busybox\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod1c0\"]) >= 4' \\\n       'len(cpus[\"pod2c0\"]) >= 4' \\\n       'len(cpus[\"pod3c0\"]) >= 4' \\\n       'len(cpus[\"pod4c0\"]) >= 4' \\\n       'len(cpus[\"pod5c0\"]) >= 1' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod2c0\"] == cpus[\"pod3c0\"] == cpus[\"pod4c0\"]' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod5c0\"]'\n\n# Leave only one guaranteed container to the fit-in-numa dynamic pool.\nkubectl delete pods pod1 pod2 pod3 --now\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod4c0\"]) >= 1' \\\n       'len(cpus[\"pod5c0\"]) >= 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod5c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod4c0\"])'\n\n# Leave only bestefforts to the dynamic pool.\nkubectl delete pods pod4 --now\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) >= 1' \\\n       'len(cpus[\"pod5c0\"]) >= 1' \\\n       'cpus[\"pod0c0\"] == cpus[\"pod5c0\"]'\n\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/test07-numa/dyp-numa.cfg",
    "content": "policy:\n  Active: dynamic-pools\n  AvailableResources:\n    CPU: cpuset:0-15\n  # Reserve one of our CPUs (cpu15) for kube-system tasks.\n  ReservedResources:\n    CPU: 1\n  dynamic-pools:\n    PinCPU: true\n    PinMemory: true\n    DynamicPoolTypes:\n      - Name: fit-in-numa\n        # All (non-system) containers are assigned to this dynamic pool\n        # type\n        Namespaces:\n          - \"*\"\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/policies.test-suite/dynamic-pools/verify.source.sh",
    "content": "# Utilities to verify data from metrics\n\nverify_metrics_url=\"http://localhost:8891/metrics\"\n\nverify-metrics-has-line() {\n    local expected_line=\"$1\"\n    vm-run-until --timeout 10 \"echo 'waiting for metrics line: $expected_line' >&2; curl --silent $verify_metrics_url | grep -E '$expected_line'\" || {\n        command-error \"expected line '$1' missing from the output\"\n    }\n}\n\nverify-metrics-has-no-line() {\n    local unexpected_line=\"$1\"\n    vm-run-until --timeout 10 \"echo 'checking absense of metrics line: $unexpected_line' >&2; ! curl --silent $verify_metrics_url | grep -Eq '$unexpected_line'\" || {\n        command-error \"unexpected line '$1' found from the output\"\n    }\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/cri-resmgr.cfg",
    "content": "policy:\n  Active: podpools\n  # Use 14 CPUs in total.\n  AvailableResources:\n    CPU: cpuset:2-15\n  # One CPU is dedicated for reserved tasks, 13 CPUs left.\n  ReservedResources:\n    CPU: cpuset:15\n  podpools:\n    PinCPU: true\n    PinMemory: true\n    Pools:\n      # Take 3 CPUs to \"singlecpu\" podpools, 10 CPUs left.\n      - Name: singlecpu\n        CPU: 1\n        MaxPods: 2\n        Instances: 3 CPUs\n        # Not defining pool fill order equals to the default:\n        # fillOrder: Balanced.\n\n      # Take at most ~6.5 CPUs (= 50% * 13) to \"dualcpu\" pools.\n      # Allocating 2 CPUs per pool allows instantiating 3 pools,\n      # that is, 6 CPUs is really taken.\n      # 4 CPUs left.\n      # Leftover CPUs will be shared among pods and containers not in\n      # pools.\n      - Name: dualcpu\n        CPU: 2\n        MaxPods: 3\n        Instances: 50 %\n        FillOrder: Packed\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/podpools-configmap.yaml.in",
    "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cri-resmgr-config.default\n  namespace: kube-system\ndata:\n  policy: |+\n    Active: podpools\n    ReservedResources:\n      CPU: 1\n    podpools:\n      Pools:\n        - Name: $NAME\n          Instances: $INSTANCES\n          CPU: $CPU\n          MaxPods: $MAXPODS\n  logger: |+\n    Debug: resource-manager,cache,policy,memory\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/py_consts.var.py",
    "content": "# This file captures expected CPU allocator behavior when the podpools\n# policy is started with the test default cri-resmgr configuration on\n# n4c16 topology.\n\n# cri-resmgr output on constructed pools.\nexpected_podpools_output = \"\"\"\npodpools policy pools:\n- pool 0: reserved[0]{cpus:15, mems:3, pods:0/0, containers:0}\n- pool 1: default[0]{cpus:5,12-14, mems:1,3, pods:0/0, containers:0}\n- pool 2: singlecpu[0]{cpus:2, mems:0, pods:0/2, containers:0}\n- pool 3: singlecpu[1]{cpus:3, mems:0, pods:0/2, containers:0}\n- pool 4: singlecpu[2]{cpus:4, mems:1, pods:0/2, containers:0}\n- pool 5: dualcpu[0]{cpus:6-7, mems:1, pods:0/3, containers:0}\n- pool 6: dualcpu[1]{cpus:8-9, mems:2, pods:0/3, containers:0}\n- pool 7: dualcpu[2]{cpus:10-11, mems:2, pods:0/3, containers:0}\n\"\"\"\n\n# 1. Parse expected_podpools_output into\n#    expected.cpus.POOLNAME[INSTANCE] = {\"cpuNN\", ...}\n# 2. Calculate memory nodes based on expected.cpus into\n#    expected.mems.POOLNAME[INSTANCE] = {\"nodeN\", ...}\n#    (do not read these from output in order to verify its correctness)\n#\n# As the result:\n# expected.cpus.singlecpu == [{\"cpu02\"}, {\"cpu03\"}, {\"cpu04\"}]\n# expected.mems.singlecpu == [{\"node0\"}, {\"node0\"}, {\"node1\"}]\n\nimport re\n\nclass expected:\n    class cpus:\n        pass\n    class mems:\n        pass\n\ndef _add_expected_pool(poolname, poolindex, cpuset):\n    cpus = []\n    for cpurange in cpuset.split(\",\"):\n        lower_upper = [int(n) for n in cpurange.split(\"-\")]\n        if len(lower_upper) == 1:\n            cpus.append(lower_upper[0])\n        else:\n            cpus.extend([i for i in range(lower_upper[0], lower_upper[1]+1)])\n    if not hasattr(expected.cpus, poolname):\n        setattr(expected.cpus, poolname, [])\n        setattr(expected.mems, poolname, [])\n    getattr(expected.cpus, poolname).append(set('cpu%s' % (str(cpu).zfill(2),) for cpu in cpus))\n    getattr(expected.mems, poolname).append(set(\"node%s\" % (cpu//4,) for cpu in cpus))\n\nfor poolname, poolindex, cpuset in re.findall(r': ([a-z]+)\\[([0-9]+)\\]\\{cpus:([0-9,-]+), ', expected_podpools_output):\n    _add_expected_pool(poolname, poolindex, cpuset)\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test01-basic-placement/code.var.sh",
    "content": "# Test placing containers with and without annotations to correct pools\n# reserved and shared CPUs.\n\n( kubectl delete pods pod3 -n kube-system --now --wait --ignore-not-found ) || true\n\n# pod0: singlecpu\nout \"\"\nout \"### Multicontainer pod, all containers run on single CPU\"\n# singlecpu pool has capacity for two pods => 500 mCPU/pod\n# test with 3 containers per pod => 167 mCPU/container\nCPUREQ=\"167m\" MEMREQ=\"\" CPULIM=\"\" MEMLIM=\"\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: singlecpu\" CONTCOUNT=3 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"] == cpus[\"pod0c2\"]' \\\n       'cpus[\"pod0c0\"] == expected.cpus.singlecpu[0]' \\\n       'mems[\"pod0c0\"] == expected.mems.singlecpu[0]'\n\n# pod1: dualcpu\nout \"\"\nout \"### Multicontainer pod, all containers run on two CPUs.\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CONTCOUNT=3 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod1c0\"] == cpus[\"pod1c1\"] == cpus[\"pod1c2\"]' \\\n       'cpus[\"pod1c0\"] == expected.cpus.dualcpu[0]' \\\n       'mems[\"pod1c1\"] == expected.mems.dualcpu[0]'\n\n# pod2: default\nout \"\"\nout \"### Multicontainer pod, no annotations. Runs on shared CPUs.\"\nCONTCOUNT=3 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod2c0\"] == cpus[\"pod2c1\"] == cpus[\"pod2c2\"]' \\\n       'cpus[\"pod2c0\"] == expected.cpus.default[0]' \\\n       'mems[\"pod2c2\"] == expected.mems.default[0]'\n\n# pod3: reserved\nout \"\"\nout \"### Multicontainer pod in kube-system namespace. Runs on reserved CPUs.\"\nnamespace=kube-system CONTCOUNT=3 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod3c0\"] == cpus[\"pod3c1\"] == cpus[\"pod3c2\"]' \\\n       'cpus[\"pod3c0\"] == expected.cpus.reserved[0]' \\\n       'mems[\"pod3c0\"] == expected.mems.reserved[0]'\n\nkubectl delete pods pod3 -n kube-system --now --wait --ignore-not-found\n\n# pod4: bad pool name\nout \"\"\nout \"### Single container pod, fallback to the default pool.\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: non-existing-pool\" create podpools-busybox\nreport allowed\nverify 'cpus[\"pod4c0\"] == expected.cpus.default[0]' \\\n       'mems[\"pod4c0\"] == expected.mems.default[0]'\n\nkubectl delete pods pod0 pod1 pod2 --now --wait --ignore-not-found\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test02-fill-order/code.var.sh",
    "content": "# Test filling pools with pods in correct order\n\n# Test only BestEffort containers\nCPUREQ=\"\" MEMREQ=\"\" CPULIM=\"\" MEMLIM=\"\"\n\n# pod0..2: balanced filling, every singlecpu pool should have one pod\nout \"### Filling singlecpu pool in Balanced fill order\"\nn=3 POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: singlecpu\" CONTCOUNT=2 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 1' \\\n       'len(cpus[\"pod2c0\"]) == 1' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"])'\n\n# pod3..5: balanced filling up to max, every singlecpu pool should have two pods\nn=3 POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: singlecpu\" CONTCOUNT=2 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod0c1\"]' \\\n       'cpus[\"pod1c0\"] == cpus[\"pod1c1\"]' \\\n       'cpus[\"pod2c0\"] == cpus[\"pod2c1\"]' \\\n       'len(cpus[\"pod0c0\"]) == 1' \\\n       'len(cpus[\"pod1c0\"]) == 1' \\\n       'len(cpus[\"pod2c0\"]) == 1' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"])' \\\n       'cpus[\"pod3c0\"] == cpus[\"pod3c1\"]' \\\n       'cpus[\"pod4c0\"] == cpus[\"pod4c1\"]' \\\n       'cpus[\"pod5c0\"] == cpus[\"pod5c1\"]' \\\n       'len(cpus[\"pod3c0\"]) == 1' \\\n       'len(cpus[\"pod4c0\"]) == 1' \\\n       'len(cpus[\"pod5c0\"]) == 1' \\\n       'disjoint_sets(cpus[\"pod3c0\"], cpus[\"pod4c0\"], cpus[\"pod5c0\"])' \\\n       'cpus[\"pod5c0\"] == cpus[\"pod2c0\"]' # the last pool should have been filled by pods 2 and 5\n\n# make a little room to the first pool and clear the last pool\nkubectl delete pods pod0 pod2 pod5 --now --wait --ignore-not-found\n\n# pod6: Balanced fill order should place this pod to the last pool (it has maximal free space)\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: singlecpu\" CONTCOUNT=1 create podpools-busybox\nreport allowed\nverify 'disjoint_sets(cpus[\"pod6c0\"],\n                      set.union(cpus[\"pod1c0\"], cpus[\"pod3c0\"], cpus[\"pod4c0\"]))'\n\nkubectl delete pods --all --now --wait\nreset counters\n\nout \"### Filling dualcpu pool in Packed fill order\"\n# pod0..2: should go to the first pool\nn=3 POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CONTCOUNT=1 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"]'\n\n# pod3..5: should go to the second pool\nn=3 POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CONTCOUNT=1 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod0c0\"] == cpus[\"pod1c0\"] == cpus[\"pod2c0\"]' \\\n       'cpus[\"pod3c0\"] == cpus[\"pod4c0\"] == cpus[\"pod5c0\"]' \\\n       'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod3c0\"])'\n\n# Deleting two pods from the first pool, one from the last.\nkubectl delete pods pod0 pod1 pod5 --now --wait --ignore-not-found\n\n# pod6: Packed fill order should place this to the last pool (it has minimal free space)\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CONTCOUNT=1 create podpools-busybox\nreport allowed\nverify 'cpus[\"pod3c0\"] == cpus[\"pod4c0\"] == cpus[\"pod6c0\"]' \\\n       'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod6c0\"])'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test03-qos/code.var.sh",
    "content": "# Test all QoS class pods in a pool, reserved and shared CPUs.\n# Verify that CFS CPU shares is set correctly in all cases.\n\nvm-put-file \"$HOST_PROJECT_DIR/scripts/testing/kube-cgroups\" \"/usr/local/bin/kube-cgroups\"\n\nverify-cpushare() {\n    podXcY=$1\n    expected_cgv1=$2\n    expected_cgv2=$3\n    vm-command \"kube-cgroups -n . -c $podXcY -f 'cpu.(shares|weight)\\$'\"\n    CPU_SHARES_WEIGHT=$(echo \"$COMMAND_OUTPUT\" | awk '/cpu.*:/{print $2}')\n    if [ \"$CPU_SHARES_WEIGHT\" = \"$expected_cgv1\" ]; then\n        echo \"verified cpu.shares of $podXcY == $expected_cgv1\"\n    elif [ \"$CPU_SHARES_WEIGHT\" = \"$expected_cgv2\" ]; then\n        echo \"verified cpu.weight of $podXcY == $expected_cgv2\"\n    else\n        echo \"assertion failed when verifying $podXcY: got '$COMMAND_OUTPUT' expected 'cpu.shares=$expected_cgv1' or 'cpu.weight=$expected_cgv2'\"\n        exit 1\n    fi\n}\n\nCPUREQ=\"\" MEMREQ=\"\" CPULIM=\"\" MEMLIM=\"\" POD_ANNOTATION=\"\"\n\nout \"### Assigning BestEffort, Burstable and Guaranteed pods to the same (dualcpu) pool\"\n# pod0c0: besteffort\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" create podpools-busybox\n# pod1c0: burstable\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=500m create podpools-busybox\n# pod2c0: guaranteed\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=1 CPULIM=1 MEMREQ=100M MEMLIM=100M create podpools-busybox\nreport allowed\n\nverify-cpushare pod0c0 2 1\nverify-cpushare pod1c0 512 20\nverify-cpushare pod2c0 1024 39\n\nkubectl delete pods --all --now --wait\nreset counters\n\nout \"### Assigning BestEffort, Burstable and Guaranteed pods shared CPUs\"\n# pod0c0: besteffort\ncreate podpools-busybox\n# pod1c0: burstable\nCPUREQ=500m create podpools-busybox\n# pod2c0: guaranteed\nCPUREQ=1 CPULIM=1 MEMREQ=100M MEMLIM=100M create podpools-busybox\nreport allowed\n\nverify-cpushare pod0c0 2 1\nverify-cpushare pod1c0 512 20\nverify-cpushare pod2c0 1024 39\n\nkubectl delete pods --all --now --wait\nreset counters\n\nout \"### Assigning BestEffort, Burstable and Guaranteed pods reserved CPUs\"\n# pod0c0: besteffort\nnamespace=kube-system create podpools-busybox\n# pod1c0: burstable\nnamespace=kube-system CPUREQ=500m create podpools-busybox\n# pod2c0: guaranteed\nnamespace=kube-system CPUREQ=1 CPULIM=1 MEMREQ=100M MEMLIM=100M create podpools-busybox\nreport allowed\n\nverify-cpushare pod0c0 2 1\nverify-cpushare pod1c0 512 20\nverify-cpushare pod2c0 1024 39\n\nkubectl delete pods pod0 pod1 pod2 -n kube-system --now --wait --ignore-not-found\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test04-overbook-cpus/code.var.sh",
    "content": "# Test CPU request warnings and errors:\n# - Overbooked CPU sets\n# - Bad CPU requests: mismatch between pool CPUs per pod and container CPU requests\n\nCRI_RESMGR_OUTPUT=\"cat cri-resmgr.output.txt\"\n\n# pod0: overbook with single burstable pod and container\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=2900m CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\" create podpools-busybox\nreport allowed\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*overbooked.*(2899|2900)m'\" || error \"missing overbook warning\"\nkubectl delete pods --all --now --wait\n\n# pod1: overbook with single burstable pod with two containers\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=1050m CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\" CONTCOUNT=2 create podpools-busybox\nreport allowed\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*overbooked.*2100m'\" || error \"missing overbook warning\"\nkubectl delete pods --all --now --wait\n\n# pod2, pod3: overbook with two guaranteed pods, one container in each pod\nn=2 POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=1001m MEMREQ=100M CPULIM=1001m MEMLIM=100M create podpools-busybox\nreport allowed\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*overbooked.*2002m'\" || error \"missing overbook warning\"\nkubectl delete pods --all --now --wait\n\n# pod4, pod5: no overbooking with exact CPUs guaranteed + besteffort pod\nterminate cri-resmgr # restart to clear log\nlaunch cri-resmgr\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=1000m CPULIM=1000m MEMREQ=100M MEMLIM=100M CONTCOUNT=2 create podpools-busybox\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=\"\" CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\" create podpools-busybox\nreport allowed\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*overbooked'\" && error \"overbook warning with maximum allowed load\"\nkubectl delete pods --all --now --wait\n# podpools logs misaligned CPU requests after pod deletion\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*bad CPU requests:.*pod4.* requested 2000 mCPUs.* 666 mCPUs'\" || error \"bad CPU request from pod4 expected but not found\"\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*bad CPU requests:.*pod5.* requested 0 mCPUs.* 666 mCPUs'\" || error \"bad CPU request from pod5 expected but not found\"\n\n# pod6: request 4 * 167 mCPU, that is almost required 666 mCPU. Should not be bad CPU request\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" CPUREQ=167m CPULIM=\"\" MEMREQ=\"\" MEMLIM=\"\" CONTCOUNT=4 create podpools-busybox\nvm-command \"$CRI_RESMGR_OUTPUT | grep -E '^E.*bad CPU requests:.*pod6'\" && error \"pod6 CPU request was ok, but 'bad CPU request' error found\"\n\nkubectl delete pods --all --now --wait\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test05-agent-updates-config/code.var.sh",
    "content": "# Relaunch cri-resmgr so that it will listen to cri-resmgr-agent\ncleanup() {\n    vm-command \"kubectl delete pod -n kube-system pod0 --now --wait --ignore-not-found; kubectl delete pods --all --now --wait; kubectl delete cm -n kube-system cri-resmgr-config.default\"\n    terminate cri-resmgr\n    terminate cri-resmgr-agent\n    vm-command \"cri-resmgr -reset-policy; cri-resmgr -reset-config\"\n}\n\ncleanup\ncri_resmgr_config=fallback launch cri-resmgr\nlaunch cri-resmgr-agent\n\n# Create a pod to every pod pool in the default config:\n# reserved, shared, singlecpu, dualcpu\n# pod0: reserved\nCPUREQ=\"\" namespace=kube-system create podpools-busybox\n# pod1: default\nCPUREQ=\"\" create podpools-busybox\n# pod2: singlecpu\nCPUREQ=\"1\" POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: singlecpu\" create podpools-busybox\n# pod3, pod4, pod5, pod6: dualcpu (dualcpu 3 pods/pool, packed)\nn=4 CPUREQ=\"1\" POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: dualcpu\" create podpools-busybox\nreport allowed\nverify \"cpus['pod0c0'] == expected.cpus.reserved[0]\" \\\n       \"cpus['pod1c0'] == expected.cpus.default[0]\" \\\n       \"cpus['pod2c0'] == expected.cpus.singlecpu[0]\" \\\n       \"cpus['pod3c0'] == expected.cpus.dualcpu[0]\" \\\n       \"cpus['pod4c0'] == expected.cpus.dualcpu[0]\" \\\n       \"cpus['pod5c0'] == expected.cpus.dualcpu[0]\" \\\n       \"cpus['pod6c0'] == expected.cpus.dualcpu[1]\"\n\necho \"Switch to new configuration without singlecpu pools\"\nvm-put-file $(NAME=dualcpu CPU=2 MAXPODS=2 INSTANCES=\"100 %\" instantiate podpools-configmap.yaml) podpools-dualcpu-configmap.yaml\nkubectl apply -f podpools-dualcpu-configmap.yaml\nsleep 5\nreport allowed\nverify \"cpus['pod0c0'] == expected.cpus.reserved[0]\" `# reserved remains the same` \\\n       \"len(cpus['pod1c0']) == 1\" `# the default pool has only one CPU` \\\n       \"cpus['pod2c0'] == cpus['pod1c0']\" `# no singlecpu pool -> assign to default` \\\n       `# there are many dualcpu pools (1 out of 2 pods/pool, balanced)` \\\n       \"len(cpus['pod3c0']) == 2\" \\\n       \"len(cpus['pod4c0']) == 2\" \\\n       \"len(cpus['pod5c0']) == 2\" \\\n       \"len(cpus['pod6c0']) == 2\" \\\n       \"disjoint_sets(cpus['pod3c0'], cpus['pod4c0'], cpus['pod5c0'], cpus['pod6c0'])\"\n\necho \"Negative test: try switching to an invalid configuration, check assignments have not changed\"\nvm-put-file $(NAME=borked CPU=130 MAXPODS=2 INSTANCES=1 instantiate podpools-configmap.yaml) podpools-borked-configmap.yaml\nkubectl apply -f podpools-borked-configmap.yaml\nsleep 5\nreport allowed\nverify \"cpus['pod0c0'] == {'cpu15'}\" \\\n       \"cpus['pod1c0'] == cpus['pod2c0']\" \\\n       \"disjoint_sets(cpus['pod3c0'], cpus['pod4c0'], cpus['pod5c0'], cpus['pod6c0'])\" \\\n\necho \"After broken reconfiguration trial, switch to valid configuration without dualcpu pools\"\n# This configuration leaves no left-over CPUs for the default pool\n# => the default pool will use the same CPUs as the reserved pool.\nvm-put-file $(NAME=singlecpu CPU=1 MAXPODS=1 INSTANCES=\"100 %\" instantiate podpools-configmap.yaml) podpools-dualcpu-configmap.yaml\nkubectl apply -f podpools-dualcpu-configmap.yaml\nsleep 5\nreport allowed\n\nverify \"cpus['pod0c0'] == expected.cpus.reserved[0]\" `# reserved remains the same` \\\n       \"cpus['pod1c0'] == expected.cpus.reserved[0]\" `# the default pool equals to reserved` \\\n       \"len(cpus['pod2c0']) == 1\" `# pod2 in singlecpu[0]` \\\n       \"disjoint_sets(cpus['pod2c0'], expected.cpus.reserved[0])\" \\\n       `# all dualcpu pods endup into the default pool` \\\n       \"cpus['pod3c0'] == cpus['pod4c0'] == cpus['pod5c0'] == cpus['pod6c0']\" \\\n       \"cpus['pod3c0'] == expected.cpus.reserved[0]\"\n\necho \"Not enough dualcpu pools for all running dualcpu pods, the rest fall back to the default pool\"\nvm-put-file $(NAME=dualcpu CPU=2 MAXPODS=1 INSTANCES=\"2\" instantiate podpools-configmap.yaml) podpools-dualcpu-configmap.yaml\nkubectl apply -f podpools-dualcpu-configmap.yaml\nsleep 5\nreport allowed\npp cpus\nverify \"cpus['pod0c0'] == expected.cpus.reserved[0]\" `# reserved remains the same` \\\n       \"len(cpus['pod1c0']) == 9\" `# the default pool` \\\n       \"cpus['pod2c0'] == cpus['pod1c0']\" `# no singlecpu pool -> assign to default` \\\n       `# two dualcpu pods go to dualcpu pools, two to the default pool` \\\n       \"len([c for c in ['pod3c0', 'pod4c0', 'pod5c0', 'pod6c0'] if len(cpus[c])==2]) == 2\" \\\n       \"len([c for c in ['pod3c0', 'pod4c0', 'pod5c0', 'pod6c0'] if len(cpus[c])==9]) == 2\"\n\n# Clean up agent-delivered configuration setup as it might break tests\n# that by default rely on forced configurations.\ncleanup\nlaunch cri-resmgr\nlaunch cri-resmgr-agent\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test06-prometheus-metrics/code.var.sh",
    "content": "# Test reporting Prometheus metrics from podpools\n\ncleanup() {\n    vm-command \"kubectl get pods -A | grep -E ' pod[0-9]' | while read namespace pod rest; do kubectl -n \\$namespace delete pod \\$pod --now --wait --ignore-not-found; done\"\n}\n\nparse-commandoutput-log_pool_cpuset() {\n    log_pool_cpuset=$(awk -F 'cpus:|, ' \"{print \\$2}\" <<< \"$COMMAND_OUTPUT\")\n    out \"parsed: log_pool_cpuset=$log_pool_cpuset\"\n}\n\nparse-commandoutput-log_pool_name() {\n    log_pool_name=$(awk -F\"[ {]*\" \"{print \\$10}\" <<< \"$COMMAND_OUTPUT\")\n    out \"parsed: log_pool_name=$log_pool_name\"\n}\n\nverify-log-vs-metrics() {\n    local podXcY=\"$1\"\n    local cpuUsageMin=\"$2\" # optional\n    local cpuUsageMax=\"$3\" # optional\n    vm-command \"grep 'assigning container $podXcY to pool' cri-resmgr.output.txt\"\n    parse-commandoutput-log_pool_cpuset\n    parse-commandoutput-log_pool_name\n    local usageCmd=\"curl --silent $metrics_url | grep $log_pool_cpuset | grep $podXcY\"\n    vm-run-until --timeout 10 \"$usageCmd\" || {\n        error \"cannot find pod:container $1 and cpuset $log_pool_cpuset from the report\"\n    }\n    if [ -n \"$cpuUsageMax\" ]; then\n        echo \"verifying CPU usage $cpuUsageMin < X < $cpuUsageMax\"\n        vm-run-until --timeout 20 \"X=\\\"\\$($usageCmd)\\\"; echo \\\"\\$X\\\"; X=\\${X##* }; X=\\${X%%.*}; echo $cpuUsageMin \\< \\$X \\< $cpuUsageMax; (( $cpuUsageMin < \\$X )) && (( \\$X < $cpuUsageMax ))\"\n    fi\n}\n\nverify-metrics-has-line() {\n    local expected_line=\"$1\"\n    out \"verifying metrics line syntax...\"\n    vm-run-until --timeout 10 \"echo '    waiting for metrics line: $expected_line' >&2; curl --silent $metrics_url | grep -E '$expected_line'\" || {\n        command-error \"expected line '$1' missing from the output\"\n    }\n}\n\n# Delete left-over test pods from the kube-system namespace\nfor podX in $(kubectl get pods -n kube-system | awk '/^pod[0-9]/{print $1}'); do\n    kubectl delete pods $podX -n kube-system --now --wait --ignore-not-found\ndone\n\nmetrics_url=\"http://localhost:8891/metrics\"\n\n# Launch cri-resmgr with wanted metrics update interval\n# and configuration that opens the instrumentation http server.\nterminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/podpools-metrics.cfg  cri_resmgr_extra_args=\"-metrics-interval 4s\" launch cri-resmgr\n\n# pod0: single container, reserve 400m CPU, but do not use it.\nout \"\"\nout \"### Idle single-container pod\"\nCPUREQ=\"400m\" MEMREQ=\"\" CPULIM=\"400m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: 400mCPU\" CONTCOUNT=1 create podpools-busybox\nreport allowed\nverify-log-vs-metrics pod0:pod0c0 0 20\n\n# pod0: single container, reserve 400m CPU and use it.\n# \"yes\" should show up in top with 40 % CPU consumption.\nout \"\"\nout \"### Busy single-container pod\"\nCPUREQ=\"400m\" MEMREQ=\"\" CPULIM=\"400m\" MEMLIM=\"\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: 400mCPU\" CONTCOUNT=1 WORK='yes>/dev/null & ' create podpools-busybox\nreport allowed\nverify-log-vs-metrics pod1:pod1c0 30 50\n\nout \"\"\nout \"### Idle four-container pod\"\nCPUREQ=\"100m\" CPULIM=\"100m\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: 400mCPU\" CONTCOUNT=4 create podpools-busybox\nreport allowed\nverify-metrics-has-line 'pool_cpu_usage{CPUs=\"[0-9]-[0-9]\",container_name=\"pod2:pod2c0,pod2:pod2c1,pod2:pod2c2,pod2:pod2c3\",def_name=\"400mCPU\",memory=\"1\",pod_name=\"pod2\",policy=\"podpools\",pool_size=\"2000\",pretty_name=\"400mCPU\\[[0-9]\\]\"}'\nverify-log-vs-metrics pod2:pod2c3 0 20\n\nout \"\"\nout \"### Busy four-container pod\"\nCPUREQ=\"100m\" CPULIM=\"100m\"\nPOD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: 400mCPU\" CONTCOUNT=4 WORK='yes>/dev/null & ' create podpools-busybox\nreport allowed\nverify-log-vs-metrics pod3:pod3c3 30 50\n\nout \"\"\nout \"### Multicontainer pod, no annotations. Runs on shared CPUs.\"\nCPUREQ=\"\" CPULIM=\"\"\nCONTCOUNT=2 create podpools-busybox\nreport allowed\nvm-command \"curl --silent $metrics_url | grep -v ^cgroup_\"\nverify-log-vs-metrics pod4:pod4c1 0 20\n\nout \"\"\nout \"### Multicontainer pod in kube-system namespace. Runs on reserved CPUs.\"\nCPUREQ=\"\" CPULIM=\"\"\nnamespace=kube-system CONTCOUNT=3 create podpools-busybox\nreport allowed\nvm-command \"curl --silent $metrics_url | grep -v ^cgroup_\"\n# There should be kube-apiserver, etcd etc. running on reserved CPUs as well,\n# therefore allow a lot of CPU usage yet pod5 is not doing anything.\nverify-log-vs-metrics pod5:pod5c1 0 100\n\ncleanup\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test06-prometheus-metrics/podpools-metrics.cfg",
    "content": "policy:\n  Active: podpools\n  ReservedResources:\n    CPU: 1\n  podpools:\n    Pools:\n      - Name: 400mCPU\n        Instances: 90 %\n        CPU: 2\n        MaxPods: 5\n        # (2000m CPUs/pool) / (5 pods/pool) = 400m CPUs/pod\ninstrumentation:\n  HTTPEndpoint: :8891\n  PrometheusExport: true\nlogger:\n  Debug: resource-manager,cache,policy,memory\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test07-custom-default-pool/code.var.sh",
    "content": "# Launch cri-resmgr with a custom default pool and many highperf\n# pools. The CPUs in the custom default pool are disjoint from CPUs in\n# the reserved pool. 100 % of remaining CPUs are allocated to highperf\n# pools.\nterminate cri-resmgr\ncri_resmgr_cfg=${TEST_DIR}/podpools-custom-default.cfg launch cri-resmgr\n\ncleanup() {\n    ( kubectl delete pods --all --now --wait )\n    ( kubectl delete pod -n kube-system pod0c-mysystem --now --wait --ignore-not-found )\n    ( kubectl delete namespace daemons --now --wait --ignore-not-found )\n}\n\ncleanup\n\nnamespace=kube-system NAME=pod0c-mysystem CONTCOUNT=2 create podpools-busybox\nkubectl create namespace daemons\nnamespace=daemons NAME=pod0c-mydaemon CONTCOUNT=2 create podpools-busybox\nreport allowed\nverify 'len(cpus[\"pod0c-mysystemc0\"]) == 1' \\\n       'len(cpus[\"pod0c-mydaemonc0\"]) == 3' \\\n       'disjoint_sets(cpus[\"pod0c-mysystemc0\"], cpus[\"pod0c-mydaemonc0\"])'\n\nNAME=pod1c-highperf POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: highperf\" CPUREQ=2 CPULIM=2 MEMREQ=\"\" MEMLIM=\"\" create podpools-busybox\nNAME=pod2c-highperf POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: highperf\" CPUREQ=2 CPULIM=2 MEMREQ=\"\" MEMLIM=\"\" create podpools-busybox\nNAME=pod3c-highperf POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: highperf\" CPUREQ=2 CPULIM=2 MEMREQ=\"\" MEMLIM=\"\" create podpools-busybox\nNAME=pod4c-highperf POD_ANNOTATION=\"pool.podpools.cri-resource-manager.intel.com: highperf\" CPUREQ=2 CPULIM=2 MEMREQ=\"\" MEMLIM=\"\" create podpools-busybox\nreport allowed\nverify 'len(cpus[\"pod1c-highperfc0\"]) == 2' \\\n       'len(cpus[\"pod2c-highperfc0\"]) == 2' \\\n       'len(cpus[\"pod3c-highperfc0\"]) == 2' \\\n       'len(cpus[\"pod4c-highperfc0\"]) == 2' \\\n       'disjoint_sets(cpus[\"pod1c-highperfc0\"], cpus[\"pod2c-highperfc0\"], cpus[\"pod3c-highperfc0\"], cpus[\"pod4c-highperfc0\"])'\n\ncleanup\nvm-command \"cat < cri-resmgr.output.txt > cri-resmgr-podpools-single-pool.output.txt\"\nterminate cri-resmgr\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/test07-custom-default-pool/podpools-custom-default.cfg",
    "content": "policy:\n  Active: podpools\n  ReservedResources:\n    CPU: cpuset:0\n  podpools:\n    Pools:\n      - Name: default\n        CPU: 3\n      - Name: highperf\n        Instances: 100%\n        CPU: 2\n        MaxPods: 1\nlogger:\n  Debug: resource-manager,cache,policy,memory\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/policies.test-suite/podpools/podpools-busybox.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"$POD_ANNOTATION\" ]; then echo \"\n  annotations:\n    $POD_ANNOTATION\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - ${WORK}echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    $(if [ -n \"${CPUREQ}\" ]; then echo \"\n    resources:\n      requests:\n        cpu: ${CPUREQ}\n        $(if [ -n \"${MEMREQ}\" ]; then echo \"\n        memory: '${MEMREQ}'\n        \"; fi)\n      $(if [ -n \"${CPULIM}\" ]; then echo \"\n      limits:\n        cpu: ${CPULIM}\n        $(if [ -n \"$MEMLIM\" ]; then echo \"\n        memory: '${MEMLIM}'\n        \"; fi)\n    \"; fi)\n    \"; fi)\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/README.txt",
    "content": "# E2E static-pools policy test\n\n## Requirements\n\nThis test requires containerd v1.4 or later on the VM. Earlier\ncontainerd versions fail to mount container images built on top of\nClear Linux base image. That includes mounting cri-resmgr-webhook.\n\n`cri-resmgr-webhook` image must be present on the host (`make\nimages`). The latest image in `docker images cri-resmgr-webhook` list\nwill be installed and tested on the VM.\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/cmk-exclusive.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\nspec:\n  terminationGracePeriodSeconds: 1\n  tolerations:\n    - key: 'cmk'\n      operator: 'Equal'\n      value: 'true'\n      effect: 'NoSchedule'\n  containers:\n    - name: ${NAME}c0\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      env:\n$([ -z $STP_POOL ] || echo \"\n        - name: STP_POOL\n          value: '${STP_POOL}'\")\n$([ -z $STP_SOCKET_ID ] || echo \"\n        - name: STP_SOCKET_ID\n          value: '${STP_SOCKET_ID}'\")\n      command: ['sh', '-c']\n      args:\n        - 'while :; do echo ${NAME}c0 CMK_CPUS_ASSIGNED=\\\"\\$CMK_CPUS_ASSIGNED\\\"; sleep 1; done'\n      resources:\n        requests:\n          cpu: ${CPU}\n$([ \"$EXCLCORES\" = \"omit\" ] || echo \"\n          cmk.intel.com/exclusive-cores: '${EXCLCORES}'\")\n        limits:\n          cpu: ${CPU}\n$([ \"$EXCLCORES\" = \"omit\" ] || echo \"\n          cmk.intel.com/exclusive-cores: '${EXCLCORES}'\")\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/cmk-isolate.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\nspec:\n  terminationGracePeriodSeconds: 1\n  tolerations:\n    - key: 'cmk'\n      operator: 'Equal'\n      value: 'true'\n      effect: 'NoSchedule'\n  containers:\n    - name: ${NAME}c0\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      env:\n$([ -z $STP_POOL ] || echo \"\n        - name: STP_POOL\n          value: '${STP_POOL}'\")\n$([ -z $STP_SOCKET_ID ] || echo \"\n        - name: STP_SOCKET_ID\n          value: '${STP_SOCKET_ID}'\")\n$([ \"$CMDSPLIT\" = \"command_all\" ] && echo \"\n      command: ['cmk', 'isolate' $CMK_ISOLATE, 'sh', '-c', 'while :; do echo ${NAME}c0 ${ECHO_VARS}; sleep 1; done']\"\n  [ \"$CMDSPLIT\" = \"command_cmk_sh\" ] && echo \"\n      command: ['cmk', 'isolate' $CMK_ISOLATE, 'sh', '-c']\n      args: ['while :; do echo ${NAME}c0 ${ECHO_VARS}; sleep 1; done']\"\n  [ \"$CMDSPLIT\" = \"command_cmk\" ] && echo \"\n      command: ['cmk', 'isolate' $CMK_ISOLATE]\n      args: ['sh', '-c', 'while :; do echo ${NAME}c0 ${ECHO_VARS}; sleep 1; done']\")\n      resources:\n        requests:\n          cpu: ${CPU}\n          $([ -z $EXCLCORES ] || echo \"cmk.intel.com/exclusive-cores: '${EXCLCORES}'\")\n        limits:\n          cpu: ${CPU}\n          $([ -z $EXCLCORES ] || echo \"cmk.intel.com/exclusive-cores: '${EXCLCORES}'\")\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/cmk-tolerating-guaranteed.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\nspec:\n  tolerations:\n    - {'key': 'cmk', 'operator': 'Equal', 'value': 'true', 'effect': 'NoSchedule'}\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/cri-resmgr.cfg",
    "content": "policy:\n  Active: static-pools\n  ReservedResources:\n    CPU: 750m\n  static-pools:\n    pools:\n      shared:\n        cpuLists:\n          - Cpuset: 0-7\n            Socket: 0\n          - Cpuset: 8-15\n            Socket: 1\n        exclusive: false\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy,stp\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/cri-resmgr-static-pools.cfg",
    "content": "policy:\n  Active: static-pools\n  ReservedResources:\n    CPU: 750m\n  static-pools:\n    ConfFilePath: \"/etc/cmk/pools.conf\"\n    LabelNode: true\n    TaintNode: true\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy,stp\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/py_consts.var.py",
    "content": "exclusive_cores={'node0/core0', 'node0/core1', 'node2/core0'}\nshared_cores={'node1/core2', 'node1/core3'}\ninfra_cores={'node2/core1', 'node3/core2', 'node3/core3'}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test00-node-status/code.var.sh",
    "content": "# Test that the static-pools policy\n# 1. labels the node with cmk.intel.com/cmk-node\n# 2. advertises correct number of exclusive-cores resources\n# 3. taints the node\n\n# shellcheck disable=SC2148\ncri_resmgr_cfg=\"$TEST_DIR/../cri-resmgr-static-pools.cfg\" static-pools-relaunch-cri-resmgr\n\nout \"\"\nout \"### Verifying that node has cmk-node label\"\nvm-run-until 'kubectl get nodes -o jsonpath=\"{.items[*].metadata.labels}\" | grep \\\"cmk.intel.com/cmk-node\\\"\\:\\\"true\\\"' ||\n    error \"cmk.intel.com/cmk-node label missing\"\n\nout \"\"\nout \"### Verifying that amount exclusive cores on node matches /etc/cmk/pools.conf\"\nvm-run-until 'kubectl get nodes -o jsonpath=\"{.items[*].status.allocatable}\" | grep -q \\\"cmk.intel.com/exclusive-cores\\\"\\:\\\"3\\\"' ||\n    error \"expected 3 allocatable cmk.intel.com/exclusive-cores\"\n\nout \"\"\nout \"### Creating a pod that should not be scheduled due to node taint\"\n( wait_t=2s create besteffort ) || {\n    echo \"failed as expected due to node taint\"\n}\n\nout \"\"\nout \"### Verifying that scheduling normal pod failed\"\nvm-command 'kubectl describe pods/pod0 | grep -E \"FailedScheduling .*cmk: true\"' || {\n    error \"FailedScheduling expected but not found\"\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test01-exclusive-pods/code.var.sh",
    "content": "# Test that exclusive-cores containers\n# 1. run on exclusive cores\n# 2. are pinned according to STP_POOL and STP_SOCKET_ID\n#    when \"cmk isolate\" is not used.\n# 3. all exclusive cores can be consumed with and without\n#    specifying STP_SOCKET_ID.\n\n# shellcheck disable=SC2148\ncri_resmgr_cfg=\"$TEST_DIR/../cri-resmgr-static-pools.cfg\" static-pools-relaunch-cri-resmgr\nexport STP_POOL=exclusive\n\nout \"\"\nout \"### Creating exclusive CMK pod with 1 exclusive core\"\nCPU=1000m STP_SOCKET_ID=1 EXCLCORES=1 create cmk-exclusive\nreport allowed\nverify 'len(cores[\"pod0c0\"]) == 1' \\\n       'packages[\"pod0c0\"] == {\"package1\"}'\n\nout \"\"\nout \"### Deleting exclusive CMK pod\"\nkubectl delete pods --all --now --wait\n\nout \"\"\nout \"### Creating exclusive CMK pod with 2 exclusive cores\"\nCPU=1000m STP_SOCKET_ID=0 EXCLCORES=2 create cmk-exclusive\nreport allowed\nverify 'len(cores[\"pod1c0\"]) == 2' \\\n       'packages[\"pod1c0\"] == {\"package0\"}'\n\nout \"\"\nout \"### Deleting exclusive CMK pod\"\nkubectl delete pods --all --now --wait\n\nout \"\"\nout \"### Creating two exclusive CMK pods with 1 exclusive core each\"\nn=2 CPU=1000m STP_SOCKET_ID=0 EXCLCORES=1 create cmk-exclusive\nreport allowed\nverify 'len(cores[\"pod2c0\"]) == 1' \\\n       'len(cores[\"pod3c0\"]) == 1' \\\n       'disjoint_sets(cores[\"pod2c0\"], cores[\"pod3c0\"])' \\\n       'packages[\"pod2c0\"] == packages[\"pod3c0\"] == {\"package0\"}'\n\nout \"\"\nout \"### Creating one more exclusive CMK pods, consuming all exclusive cores\"\nCPU=1000m STP_SOCKET_ID=1 EXCLCORES=1 create cmk-exclusive\nreport allowed\nverify 'len(cores[\"pod2c0\"]) == 1' \\\n       'len(cores[\"pod3c0\"]) == 1' \\\n       'len(cores[\"pod4c0\"]) == 1' \\\n       'disjoint_sets(cores[\"pod2c0\"], cores[\"pod3c0\"], cores[\"pod4c0\"])' \\\n       'set.union(cores[\"pod2c0\"], cores[\"pod3c0\"], cores[\"pod4c0\"]) == exclusive_cores'\nkubectl delete pods --all --now --wait\n\nout \"\"\nout \"### Test consuming all exclusive cores without specifying STP_SOCKET_ID\"\nn=3 CPU=1000m STP_SOCKET_ID=\"\" EXCLCORES=1 create cmk-exclusive\nverify 'len(cores[\"pod5c0\"]) == 1' \\\n       'len(cores[\"pod6c0\"]) == 1' \\\n       'len(cores[\"pod7c0\"]) == 1' \\\n       'disjoint_sets(cores[\"pod5c0\"], cores[\"pod6c0\"], cores[\"pod7c0\"])' \\\n       'set.union(cores[\"pod5c0\"], cores[\"pod6c0\"], cores[\"pod7c0\"]) == exclusive_cores'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test02-pods-without-cmk/code.var.sh",
    "content": "# Test that normal pods/containers scheduled on a CMK node\n# are running in the shared pool, yet there are not as many\n# CPUs as required.\n\ncri_resmgr_cfg=\"$TEST_DIR/../cri-resmgr-static-pools.cfg\" static-pools-relaunch-cri-resmgr\n\nout \"\"\nout \"### Creating a guaranteed pod, 1 CPU, goes to the shared bool\"\nCPU=1 create cmk-tolerating-guaranteed\nreport allowed\nverify 'cores[\"pod0c0\"].issubset(shared_cores)'\n\nout \"\"\nout \"### Creating next guaranteed pod, 2 CPUs, goes to the shared pool\"\nCPU=2 create cmk-tolerating-guaranteed\nreport allowed\nverify 'cores[\"pod0c0\"].issubset(shared_cores)' \\\n       'cores[\"pod1c0\"].issubset(shared_cores)'\n\nout \"\"\nout \"### Creating next guaranteed pod, 4 CPUs, goes to the shared pool\"\nCPU=4 create cmk-tolerating-guaranteed\nreport allowed\nverify 'cores[\"pod0c0\"].issubset(shared_cores)' \\\n       'cores[\"pod1c0\"].issubset(shared_cores)' \\\n       'cores[\"pod2c0\"].issubset(shared_cores)'\n\nout \"\"\nout \"### Creating next guaranteed pod, 8 CPUs, goes to the shared pool\"\nCPU=6 create cmk-tolerating-guaranteed\nreport allowed\nverify 'cores[\"pod0c0\"].issubset(shared_cores)' \\\n       'cores[\"pod1c0\"].issubset(shared_cores)' \\\n       'cores[\"pod2c0\"].issubset(shared_cores)' \\\n       'cores[\"pod3c0\"].issubset(shared_cores)'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test03-cmk-isolate/code.var.sh",
    "content": "# Test that legacy exclusive-cores containers\n# 1. run on exclusive cores\n# 2. are pinned according to \"cmk isolate\" command\n#    parameters\n# 3. run without \"cmk\" existing on the image\n# 3. all exclusive cores can be consumed\n\n# shellcheck disable=SC2148\ncri_resmgr_cfg=\"$TEST_DIR/../cri-resmgr-static-pools.cfg\" static-pools-relaunch-cri-resmgr\nexport STP_POOL=\"\" STP_SOCKET_ID=\"\"\n\nexport CMK_ISOLATE=\", '--conf-dir=/etc/cmk.conf', '--pool=exclusive', '--socket-id=1'\"\nout \"\"\nout \"### Creating pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=1 CMDSPLIT=\"command_all\" create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod0c0\"]) == 1' \\\n       'cores[\"pod0c0\"].issubset(exclusive_cores)' \\\n       'packages[\"pod0c0\"] == {\"package1\"}'\n\nexport CMK_ISOLATE=\", '--socket-id=0', '--pool=exclusive'\"\nout \"\"\nout \"### Creating pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=2000m EXCLCORES=2 CMDSPLIT=\"command_cmk_sh\" create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod1c0\"]) == 2' \\\n       'cores[\"pod1c0\"].issubset(exclusive_cores)' \\\n       'packages[\"pod1c0\"] == {\"package0\"}'\n\nexport CMK_ISOLATE=\", '--pool=shared'\"\nout \"\"\nout \"### Creating pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=\"\" CMDSPLIT=\"command_cmk\" create cmk-isolate\nreport allowed\nverify 'cores[\"pod2c0\"] == shared_cores'\n\nexport CMDSPLIT=\"command_cmk\"\n\nexport CMK_ISOLATE=\", '--conf-dir=/etc/cmk.conf', '--pool=infra'\"\nout \"\"\nout \"### Creating pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=\"\" create cmk-isolate\nreport allowed\nverify 'cores[\"pod3c0\"] == infra_cores'\n\nout \"\"\nout \"### Deleting only exclusive CMK pods, leave shared/infra running\"\nkubectl delete pods/pod0 pods/pod1 --now --wait --ignore-not-found\n\nexport CMK_ISOLATE=\", '--pool=exclusive'\"\nout \"\"\nout \"### Creating 3 exclusive pods 'cmk', 'isolate'$CMK_ISOLATE...\"\nn=3 CPU=1000m EXCLCORES=1 create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod4c0\"]) == 1' \\\n       'len(cores[\"pod5c0\"]) == 1' \\\n       'len(cores[\"pod6c0\"]) == 1' \\\n       'disjoint_sets(cores[\"pod4c0\"], cores[\"pod5c0\"], cores[\"pod6c0\"])' \\\n       'cores[\"pod4c0\"].issubset(exclusive_cores)' \\\n       'cores[\"pod5c0\"].issubset(exclusive_cores)' \\\n       'cores[\"pod6c0\"].issubset(exclusive_cores)'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test04-cmk-isolate-noaffinity/code.var.sh",
    "content": "# Test that cmk isolate --no-affinity is effective on every pool\n# with and without STP_POOL / STP_SOCKET_ID env vars.\n# Test that all exclusive cores can be consumed with --no-affinity.\n\n# shellcheck disable=SC2148\ncri_resmgr_cfg=\"$TEST_DIR/../cri-resmgr-static-pools.cfg\" static-pools-relaunch-cri-resmgr\nexport STP_POOL=\"\" STP_SOCKET_ID=\"\" CMDSPLIT=\"command_all\"\nexport ECHO_VARS='CMK_CPUS_ASSIGNED=\"$CMK_CPUS_ASSIGNED\" CMK_CPUS_SHARED=\"$CMK_CPUS_SHARED\" CMK_CPUS_INFRA=\"$CMK_CPUS_INFRA\"'\n\nexport CMK_ISOLATE=\", '--conf-dir=/etc/cmk.conf', '--pool=exclusive', '--socket-id=0', '--no-affinity'\"\nout \"\"\nout \"### Creating no-affinity pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=1 create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod0c0\"]) == 8'\ncpus_assigned=\"$(kubectl logs pod0 | tail -n 1 | awk '{print $2}')\"\ncpus_shared=\"$(kubectl logs pod0 | tail -n 1 | awk '{print $3}')\"\ncpus_infra=\"$(kubectl logs pod0 | tail -n 1 | awk '{print $4}')\"\n[ \"$cpus_assigned\" == \"CMK_CPUS_ASSIGNED=0,1\" ] ||\n    error \"expected CMK_CPUS_ASSIGNED=0,1, got $cpus_assigned\"\n[ \"$cpus_shared\" == \"CMK_CPUS_SHARED=4-6,7\" ] ||\n    error \"expected CMK_CPUS_SHARED=4-6,7, got $cpus_shared\"\n[ \"$cpus_infra\" == \"CMK_CPUS_INFRA=10-15\" ] ||\n    error \"expected CMK_CPUS_INFRA=10-15, got $cpus_infra\"\n\nexport CMK_ISOLATE=\", '--conf-dir=/etc/cmk.conf', '--pool=exclusive', '--socket-id=1', '--no-affinity'\"\nout \"\"\nout \"### Creating no-affinity pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=1 STP_POOL=\"exclusive\" STP_SOCKET_ID=\"1\" create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod1c0\"]) == 8'\ncpus_assigned=\"$(kubectl logs pod1 | tail -n 1 | awk '{print $2}')\"\n[ \"$cpus_assigned\" == \"CMK_CPUS_ASSIGNED=8,9\" ] ||\n    error \"expected CMK_CPUS_ASSIGNED=8,9, got $cpus_assigned\"\n\nexport CMK_ISOLATE=\", '--conf-dir=/etc/cmk.conf', '--pool=exclusive', '--no-affinity'\"\nout \"\"\nout \"### Creating no-affinity pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=1 STP_POOL=\"exclusive\" create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod2c0\"]) == 8'\ncpus_assigned=\"$(kubectl logs pod2 | tail -n 1 | awk '{print $2}')\"\n[ \"$cpus_assigned\" == \"CMK_CPUS_ASSIGNED=2,3\" ] ||\n    error \"expected CMK_CPUS_ASSIGNED=2,3, got $cpus_assigned\"\n\nexport CMK_ISOLATE=\", '--no-affinity', '--pool=shared'\"\nout \"\"\nout \"### Creating no-affinity pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=\"\" create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod3c0\"]) == 8'\ncpus_assigned=\"$(kubectl logs pod3 | tail -n 1 | awk '{print $2}')\"\n[ \"$cpus_assigned\" == \"CMK_CPUS_ASSIGNED=4-6,7\" ] ||\n    error \"expected CMK_CPUS_ASSIGNED=5-6,7, got $cpus_assigned\"\n\nexport CMK_ISOLATE=\", '--pool=infra', '--no-affinity'\"\nout \"\"\nout \"### Creating no-affinity pod 'cmk', 'isolate'$CMK_ISOLATE...\"\nCPU=1000m EXCLCORES=\"\" create cmk-isolate\nreport allowed\nverify 'len(cores[\"pod4c0\"]) == 8'\ncpus_assigned=\"$(kubectl logs pod4 | tail -n 1 | awk '{print $2}')\"\n[ \"$cpus_assigned\" == \"CMK_CPUS_ASSIGNED=10-15\" ] ||\n    error \"expected CMK_CPUS_ASSIGNED=10-15 got $cpus_assigned\"\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test05-negative-tests/code.var.sh",
    "content": "\n# shellcheck disable=SC2148\ncri_resmgr_cfg=\"$TEST_DIR/../cri-resmgr-static-pools.cfg\" static-pools-relaunch-cri-resmgr\nexport STP_POOL=exclusive\n\nerrmsg_zero_cores=\"static-pools: exclusive pool specified but the number of exclusive CPUs requested is 0\"\nerrmsg_non_existing_pool=\"static-pools: non-existent pool\"\nerrmsg_not_enough_exclcores=\"static-pools: not enough free cpu lists\"\n\nout \"\"\nout \"### Request cores from non-existing pool\"\n( CPU=1000m STP_SOCKET_ID=0 EXCLCORES=1 STP_POOL=elusive wait_t=5s create cmk-exclusive )  &&\n    error \"expected timeout, but pod launched with cores from non-existing pool\"\nvm-run-until \"kubectl describe pods/pod0 | grep '$errmsg_non_existing_pool'\" ||\n    error \"cannot find expected error message from pod description\"\nout \"Failed as expected\"\n\nkubectl delete pods --all --now --wait || error \"failed to delete pods\"\n\nout \"\"\nout \"### Request cores from non-existing socket\"\n( CPU=1000m STP_SOCKET_ID=2 EXCLCORES=1 wait_t=5s create cmk-exclusive )  &&\n    error \"expected timeout, but pod launched with cores from non-existing socket\"\nvm-run-until \"kubectl describe pods/pod0 | grep '$errmsg_not_enough_exclcores'\" ||\n    error \"cannot find expected error message from pod description\"\nout \"Failed as expected\"\n\nkubectl delete pods --all --now --wait || error \"failed to delete pods\"\n\nout \"\"\nout \"### Request exclusive pool but do not mention exclusive-cores\"\n( CPU=1000m STP_SOCKET_ID=0 EXCLCORES='omit' wait_t=5s create cmk-exclusive )  &&\n    error \"expected timeout, but pod launched without mentioning exclusive cores from the exclusive pool\"\nvm-run-until \"kubectl describe pods/pod0 | grep '$errmsg_zero_cores'\" ||\n    error \"cannot find expected error message from pod description\"\nout \"Failed as expected\"\n\nkubectl delete pods --all --now --wait || error \"failed to delete pods\"\n\nout \"\"\nout \"### Request 0 cores from exclusive pool\"\n( CPU=1000m STP_SOCKET_ID=0 EXCLCORES=0 wait_t=5s create cmk-exclusive )  &&\n    error \"expected timeout, but pod launched with 0 cores from the exclusive pool\"\nvm-run-until \"kubectl describe pods/pod0 | grep '$errmsg_zero_cores'\" ||\n    error \"cannot find expected error message from pod description\"\nout \"Failed as expected\"\n\nkubectl delete pods --all --now --wait || error \"failed to delete pods\"\n\nout \"\"\nout \"### Request more cores from socket 0 than available\"\n( CPU=3000m STP_SOCKET_ID=0 EXCLCORES=3 wait_t=5s create cmk-exclusive ) &&\n    error \"expected timeout, but pod got too many cores successfully\"\nvm-run-until \"kubectl describe pods/pod0 | grep '$errmsg_not_enough_exclcores'\" ||\n    error \"cannot find expected error message from pod description\"\nout \"Failed as expected\"\n\nkubectl delete pods --all --now --wait || error \"failed to delete pods\"\n\nout \"\"\nout \"### Request more cores from socket 1 than available\"\n( CPU=1000m STP_SOCKET_ID=1 EXCLCORES=2 wait_t=5s create cmk-exclusive ) &&\n    error \"expected timeout, but pod got too many cores successfully\"\nvm-run-until \"kubectl describe pods/pod0 | grep '$errmsg_not_enough_exclcores'\" ||\n    error \"cannot find expected error message from pod description\"\nout \"Failed as expected\"\n\nkubectl delete pods --all --now --wait || error \"failed to delete pods\"\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/test99-cleanup/code.var.sh",
    "content": "# This test cleans up static-pools test suite configurations from vm.\n# Other policy tests can be run after this test on the same vm without\n# recreating the vm from scratch.\n\nstatic-pools-cleanup\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/n4c16/vm-files/etc/cmk/pools.conf",
    "content": "pools:\n  exclusive:\n    cpuLists:\n    - Cpuset: 8,9\n      Socket: 1\n    - Cpuset: 0,1\n      Socket: 0\n    - Cpuset: 2,3\n      Socket: 0\n    exclusive: true\n  shared:\n    cpuLists:\n    - Cpuset: 4-6,7\n      Socket: 0\n    exclusive: false\n  infra:\n    cpuLists:\n    - Cpuset: 10-15\n      Socket: 1\n    exclusive: false\n"
  },
  {
    "path": "test/e2e/policies.test-suite/static-pools/static-pools-lib.source.sh",
    "content": "# shellcheck disable=SC2148\nstatic-pools-relaunch-cri-resmgr() {\n    local webhook_running=0\n    out \"# Relaunching cri-resmgr and agent, launch webhook if not already running\"\n\n    vm-command-q \"kubectl get mutatingwebhookconfiguration/cri-resmgr\" >& /dev/null && {\n        webhook_running=1\n    }\n    # cleanup\n    terminate cri-resmgr\n    terminate cri-resmgr-agent\n    vm-command \"rm -rf /var/lib/cri-resmgr\"\n    extended-resources remove cmk.intel.com/exclusive-cpus >/dev/null\n\n    # launch again\n    launch cri-resmgr-agent\n    launch cri-resmgr\n    vm-run-until \"! kubectl get node | grep NotReady\" ||\n        error \"kubectl node is NotReady after launching cri-resmgr-agent and cri-resmgr\"\n    if [ \"$webhook_running\" == 0 ]; then\n        vm-command-q \"[ -f webhook/webhook-deployment.yaml ]\" ||\n            install cri-resmgr-webhook\n        launch cri-resmgr-webhook\n    fi\n}\n\nstatic-pools-cleanup() {\n    ( terminate cri-resmgr-agent )\n    ( uninstall cri-resmgr-webhook )\n    ( extended-resources remove cmk.intel.com/exclusive-cpus >/dev/null )\n    ( terminate cri-resmgr )\n    vm-command 'kubectl taint node $(hostname) cmk=true:NoSchedule-' || true\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test01-pmem-node-assigning/code.var.sh",
    "content": "# Test that CPU-less PMEM nodes are assigned to closest nodes with CPU.\n\n# Restart cri-resmgr in order to clear logs and make sure assignment\n# is successful with installed cri-resmgr.\nterminate cri-resmgr\nlaunch cri-resmgr\n\nCRI_RESMGR_OUTPUT_COMMAND=\"cat cri-resmgr.output.txt\"\n\necho \"Verify PMEM node assignment to CPU-ful nodes\"\nfor expected_output in \\\n    \"PMEM node #4 assigned to .*#2\" \\\n    \"PMEM node #5 assigned to .*#3\" \\\n    \"PMEM node #6 assigned to .*#0\" \\\n    \"PMEM node #7 assigned to .*#1\"; do\n    vm-command \"$CRI_RESMGR_OUTPUT_COMMAND | grep -E '$expected_output'\" ||\n        command-error \"expected PMEM assignment not found\"\ndone\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test02-annotation-memory-type/code.var.sh",
    "content": "# Test that container memory is pinned according to memory-type annotation\n\n# pod0c0 runs on node 1, uses only dram\n# pod0c1 runs on node 2, uses only pmem\n# pod0c2 runs on node 3, uses dram+pmem\n# pod0c9 runs on root node (all non-reserved CPUs),\n#     no memory-type restrictions (=> use all memory nodes)\nMEM=250M MEMTYPEC0=dram MEMTYPEC1=pmem MEMTYPEC2=pmem,dram create memtype-guaranteed\nreport allowed\nverify 'cpus[\"pod0c0\"] == {\"cpu1\"}' \\\n       'mems[\"pod0c0\"] == {\"node1\"}' \\\n       'cpus[\"pod0c1\"] == {\"cpu2\"}' \\\n       'mems[\"pod0c1\"] == {\"node4\"}' \\\n       'cpus[\"pod0c2\"] == {\"cpu3\"}' \\\n       'mems[\"pod0c2\"] == {\"node3\", \"node5\"}' \\\n       'cpus[\"pod0c9\"] == {\"cpu1\", \"cpu2\", \"cpu3\"}' \\\n       'mems[\"pod0c9\"] == {\"node0\", \"node1\", \"node2\", \"node3\", \"node4\", \"node5\", \"node6\", \"node7\"}'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test02-annotation-memory-type/memtype-guaranteed.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  annotations:\n    memory-type.cri-resource-manager.intel.com/container.${NAME}c0: ${MEMTYPEC0}\n    memory-type.cri-resource-manager.intel.com/container.${NAME}c1: ${MEMTYPEC1}\n    memory-type.cri-resource-manager.intel.com/container.${NAME}c2: ${MEMTYPEC2}\nspec:\n  containers:\n    $(for CONT in 0 1 2; do echo \"\n    - name: ${NAME}c${CONT}\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command: ['sh', '-c', 'echo ${NAME}c${CONT} \\$(sleep inf)']\n      resources:\n        requests:\n          cpu: 500m\n          memory: ${MEM}\n        limits:\n          cpu: 500m\n          memory: ${MEM}\n    \"; done)\n    - name: ${NAME}c9\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command: ['sh', '-c', 'echo ${NAME}c9 \\$(sleep inf)']\n      resources:\n        requests:\n          cpu: 500m\n          memory: ${MEM}\n        limits:\n          cpu: 500m\n          memory: ${MEM}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test02-annotation-memory-type-deprecated-syntax/code.var.sh",
    "content": "# Test that container memory is pinned according to memory-type annotation\n\n# pod0c0 runs on node 1, uses only dram\n# pod0c1 runs on node 2, uses only pmem\n# pod0c2 runs on node 3, uses dram+pmem\n# pod0c9 runs on root node (all non-reserved CPUs),\n#     no memory-type restrictions (=> use all memory nodes)\nMEM=250M MEMTYPEC0=dram MEMTYPEC1=pmem MEMTYPEC2=pmem,dram create memtype-guaranteed\nreport allowed\nverify 'cpus[\"pod0c0\"] == {\"cpu1\"}' \\\n       'mems[\"pod0c0\"] == {\"node1\"}' \\\n       'cpus[\"pod0c1\"] == {\"cpu2\"}' \\\n       'mems[\"pod0c1\"] == {\"node4\"}' \\\n       'cpus[\"pod0c2\"] == {\"cpu3\"}' \\\n       'mems[\"pod0c2\"] == {\"node3\", \"node5\"}' \\\n       'cpus[\"pod0c9\"] == {\"cpu1\", \"cpu2\", \"cpu3\"}' \\\n       'mems[\"pod0c9\"] == {\"node0\", \"node1\", \"node2\", \"node3\", \"node4\", \"node5\", \"node6\", \"node7\"}'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test02-annotation-memory-type-deprecated-syntax/memtype-guaranteed.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  annotations:\n    cri-resource-manager.intel.com/memory-type: |\n      ${NAME}c0: ${MEMTYPEC0}\n      ${NAME}c1: ${MEMTYPEC1}\n      ${NAME}c2: ${MEMTYPEC2}\nspec:\n  containers:\n    $(for CONT in 0 1 2; do echo \"\n    - name: ${NAME}c${CONT}\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command: ['sh', '-c', 'echo ${NAME}c${CONT} \\$(sleep inf)']\n      resources:\n        requests:\n          cpu: 500m\n          memory: ${MEM}\n        limits:\n          cpu: 500m\n          memory: ${MEM}\n    \"; done)\n    - name: ${NAME}c9\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command: ['sh', '-c', 'echo ${NAME}c9 \\$(sleep inf)']\n      resources:\n        requests:\n          cpu: 500m\n          memory: ${MEM}\n        limits:\n          cpu: 500m\n          memory: ${MEM}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart/bb-coldstart.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  annotations:\n    memory-type.cri-resource-manager.intel.com/container.${NAME}c0: dram,pmem\n    cold-start.cri-resource-manager.intel.com/container.${NAME}c0: |\n      duration: ${DURATION}\nspec:\n  containers:\n    - name: ${NAME}c0\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - 'cold_alloc=\\$(dd if=/dev/zero bs=${COLD_ALLOC_KB}kB count=1 | tr \\\"\\\\\\0\\\" \\\"x\\\");\n           sh -c \\\"paused after cold_alloc \\\\\\$(sleep inf)\\\";\n           warm_alloc=\\$(dd if=/dev/zero bs=${WARM_ALLOC_KB}kB count=1 | tr \\\"\\\\\\0\\\" \\\"x\\\");\n           sh -c \\\"paused after warm_alloc \\\\\\$(sleep inf)\\\";\n           echo ${NAME}c0 \\$(sleep inf); # needed for pod resource discovery'\n      resources:\n        requests:\n          cpu: 500m\n          memory: ${MEM}\n        limits:\n          cpu: 500m\n          memory: ${MEM}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart/code.var.sh",
    "content": "# Test that a cold-started pod...\n# 1. is allowed to allocate memory only from PMEM nodes\n#    during cold period (of length $DURATION).\n# 2. is restricted from the very beginning of pod execution:\n#    immediately allocated memory blob consumes PMEM from expected node.\n# 3. is allowed to allocate memory from both PMEM and DRAM after\n#    the cold period.\n# 4. is no more restricted after $DURATION + 1s has passed in pod:\n#    warm-allocated memory is not taken from PMEM nodes.\n\nPMEM_NODES='{\"node4\", \"node5\", \"node6\", \"node7\"}'\n\n# pmem-used returns total MemUsed (allocated) memory on PMEM nodes\npmem-used() {\n    local pmem_nodes_shell=${PMEM_NODES//[\\\" ]/}\n    vm-command \"cat /sys/devices/system/node/$pmem_nodes_shell/meminfo | awk '/MemUsed:/{mem+=\\$4}END{print mem}'\" >/dev/null ||\n        command-error \"cannot read PMEM usage from node $node\"\n    echo \"$COMMAND_OUTPUT\"\n}\n\nCRI_RESMGR_OUTPUT=\"cat cri-resmgr.output.txt\"\n\nPMEM_USED_BEFORE_POD0=\"$(pmem-used)\"\n\nDURATION=10s\nCOLD_ALLOC_KB=$((50 * 1024))\nWARM_ALLOC_KB=$((100 * 1024))\nMEM=1G\ncreate bb-coldstart\n\necho \"Wait that coldstart period is started for the pod\"\nvm-run-until \"$CRI_RESMGR_OUTPUT | grep 'coldstart: triggering coldstart for pod0:pod0c0'\" ||\n    error \"cri-resmgr did not report triggering coldstart period\"\n\nverify 'cores[\"pod0c0\"] == {\"node1/core0\"}' \\\n       \"mems['pod0c0'] == {'node7'}\"\n\necho \"Wait that the pod has finished memory allocation during cold period.\"\nvm-run-until \"pgrep -f '^sh -c paused after cold_alloc'\" >/dev/null ||\n    error \"cold memory allocation timed out\"\n\necho \"Verify PMEM consumption during cold period.\"\n# meminfo MemUsed vs dd bytes error margin, use 10%\nPMEM_ERROR_MARGIN=$((COLD_ALLOC_KB / 10))\nsleep 1\nPMEM_USED_COLD_POD0=\"$(pmem-used)\"\nPMEM_COLD_CONSUMED=$(( $PMEM_USED_COLD_POD0 - $PMEM_USED_BEFORE_POD0 ))\nif (( $PMEM_COLD_CONSUMED + $PMEM_ERROR_MARGIN < $COLD_ALLOC_KB )); then\n    error \"pod0 did not allocate ${COLD_ALLOC_KB}kB from PMEM. MemUsed PMEM delta: $PMEM_COLD_CONSUMED\"\nelse\n    echo \"### Verified: PMEM memory consumed during cold period: $PMEM_COLD_CONSUMED kB, pod script allocated: $COLD_ALLOC_KB kB\"\nfi\n\ncoldstarts=$(vm-command-q \"$CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l\")\necho \"Wait that cri-resmgr finishes coldstart period within 5s + $DURATION.\"\nsleep 5s\nvm-run-until --timeout ${DURATION%s} \"[ \\$($CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l) -gt $coldstarts ]\" ||\n    error \"cri-resmgr did not report finishing coldstart period within $DURATION\"\n\nvm-command \"$CRI_RESMGR_OUTPUT | grep 'pinning to memory 1,7'\" ||\n    error \"cri-resmgr did not report pinning to expected memory nodes\"\n\nverify 'cores[\"pod0c0\"] == {\"node1/core0\"}' \\\n       'mems[\"pod0c0\"] == {\"node1\", \"node7\"}'\n\necho \"Let the pod continue from cold_alloc to warm_alloc.\"\nvm-command 'kill -9 $(pgrep -f \"^sh -c paused after cold_alloc\")'\n\necho \"Make sure that bb-coldstart finishes allocating memory in warm mode.\"\nvm-run-until \"pgrep -f '^sh -c paused after warm_alloc'\"  ||\n    error \"warm memory allocation timed out\"\n\necho \"Verify (soft): PMEM consumption after cold period.\"\nsleep 1\nPMEM_USED_WARM_POD0=\"$(pmem-used)\"\nPMEM_WARM_CONSUMED=$(( $PMEM_USED_WARM_POD0 - $PMEM_USED_COLD_POD0 ))\nif (( $PMEM_WARM_CONSUMED > 0 )); then\n    echo \"### Verify (soft) failed: pod0 allocated $WARM_ALLOC_KB kB from PMEM. Should have been taken from DRAM.\"\nelse\n    echo \"### Verified (soft): PMEM memory consumption delta during warm period: $PMEM_WARM_CONSUMED kB, pod script allocated: $WARM_ALLOC_KB kB\"\nfi\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart-deprecated-syntax/bb-coldstart.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  annotations:\n    cri-resource-manager.intel.com/memory-type: |\n      ${NAME}c0: dram,pmem\n    cri-resource-manager.intel.com/cold-start: |\n      ${NAME}c0:\n        duration: ${DURATION_S}s\nspec:\n  containers:\n    - name: ${NAME}c0\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - 'cold_alloc=\\$(dd if=/dev/zero bs=${COLD_ALLOC_KB}kB count=1 | tr \\\"\\\\\\0\\\" \\\"x\\\");\n           sh -c \\\"paused after cold_alloc \\\\\\$(sleep inf)\\\";\n           warm_alloc=\\$(dd if=/dev/zero bs=${WARM_ALLOC_KB}kB count=1 | tr \\\"\\\\\\0\\\" \\\"x\\\");\n           sh -c \\\"paused after warm_alloc \\\\\\$(sleep inf)\\\";\n           echo ${NAME}c0 \\$(sleep inf); # needed for pod resource discovery'\n      resources:\n        requests:\n          cpu: 500m\n          memory: ${MEM}\n        limits:\n          cpu: 500m\n          memory: ${MEM}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart-deprecated-syntax/code.var.sh",
    "content": "# Test that a cold-started pod...\n# 1. is allowed to allocate memory only from PMEM nodes\n#    during cold period (of length $DURATION_S).\n# 2. is restricted from the very beginning of pod execution:\n#    immediately allocated memory blob consumes PMEM from expected node.\n# 3. is allowed to allocate memory from both PMEM and DRAM after\n#    the cold period.\n# 4. is no more restricted after $DURATION_S + 1s has passed in pod:\n#    warm-allocated memory is not taken from PMEM nodes.\n\nPMEM_NODES='{\"node4\", \"node5\", \"node6\", \"node7\"}'\n\n# pmem-used returns total MemUsed (allocated) memory on PMEM nodes\npmem-used() {\n    local pmem_nodes_shell=${PMEM_NODES//[\\\" ]/}\n    vm-command \"cat /sys/devices/system/node/$pmem_nodes_shell/meminfo | awk '/MemUsed:/{mem+=\\$4}END{print mem}'\" >/dev/null ||\n        command-error \"cannot read PMEM usage from node $node\"\n    echo \"$COMMAND_OUTPUT\"\n}\n\nCRI_RESMGR_OUTPUT=\"cat cri-resmgr.output.txt\"\n\nPMEM_USED_BEFORE_POD0=\"$(pmem-used)\"\n\nDURATION_S=10\nCOLD_ALLOC_KB=$((50 * 1024))\nWARM_ALLOC_KB=$((100 * 1024))\nMEM=1G\ncreate bb-coldstart\n\necho \"Wait that coldstart period is started for the pod\"\nvm-run-until \"$CRI_RESMGR_OUTPUT | grep 'coldstart: triggering coldstart for pod0:pod0c0'\" ||\n    error \"cri-resmgr did not report triggering coldstart period\"\n\nverify 'cores[\"pod0c0\"] == {\"node1/core0\"}' \\\n       \"mems['pod0c0'] == {'node7'}\"\n\necho \"Wait that the pod has finished memory allocation during cold period.\"\nvm-run-until \"pgrep -f '^sh -c paused after cold_alloc'\" >/dev/null ||\n    error \"cold memory allocation timed out\"\n\necho \"Verify PMEM consumption during cold period.\"\n# meminfo MemUsed vs dd bytes error margin, use 10%\nPMEM_ERROR_MARGIN=$((COLD_ALLOC_KB / 10))\nsleep 1\nPMEM_USED_COLD_POD0=\"$(pmem-used)\"\nPMEM_COLD_CONSUMED=$(( $PMEM_USED_COLD_POD0 - $PMEM_USED_BEFORE_POD0 ))\nif (( $PMEM_COLD_CONSUMED + $PMEM_ERROR_MARGIN < $COLD_ALLOC_KB )); then\n    error \"pod0 did not allocate ${COLD_ALLOC_KB}kB from PMEM. MemUsed PMEM delta: $PMEM_COLD_CONSUMED\"\nelse\n    echo \"### Verified: PMEM memory consumed during cold period: $PMEM_COLD_CONSUMED kB, pod script allocated: $COLD_ALLOC_KB kB\"\nfi\n\ncoldstarts=$(vm-command-q \"$CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l\")\necho \"Wait that cri-resmgr finishes coldstart period within $(($DURATION_S + 10)) seconds.\"\nvm-run-until --timeout $((DURATION_S + 10)) \"[ \\$($CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l) -gt $coldstarts ]\" ||\n    error \"cri-resmgr did not report finishing coldstart period within $DURATION_S seconds\"\n\nvm-command \"$CRI_RESMGR_OUTPUT | grep 'pinning to memory 1,7'\" ||\n    error \"cri-resmgr did not report pinning to expected memory nodes\"\n\nverify 'cores[\"pod0c0\"] == {\"node1/core0\"}' \\\n       'mems[\"pod0c0\"] == {\"node1\", \"node7\"}'\n\necho \"Let the pod continue from cold_alloc to warm_alloc.\"\nvm-command 'kill -9 $(pgrep -f \"^sh -c paused after cold_alloc\")'\n\necho \"Make sure that bb-coldstart finishes allocating memory in warm mode.\"\nvm-run-until \"pgrep -f '^sh -c paused after warm_alloc'\"  ||\n    error \"warm memory allocation timed out\"\n\necho \"Verify (soft): PMEM consumption after cold period.\"\nsleep 1\nPMEM_USED_WARM_POD0=\"$(pmem-used)\"\nPMEM_WARM_CONSUMED=$(( $PMEM_USED_WARM_POD0 - $PMEM_USED_COLD_POD0 ))\nif (( $PMEM_WARM_CONSUMED > 0 )); then\n    echo \"### Verify (soft) failed: pod0 allocated $WARM_ALLOC_KB kB from PMEM. Should have been taken from DRAM.\"\nelse\n    echo \"### Verified (soft): PMEM memory consumption delta during warm period: $PMEM_WARM_CONSUMED kB, pod script allocated: $WARM_ALLOC_KB kB\"\nfi\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test04-dynamic-page-demotion/bb-memload.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  annotations:\n    memory-type.cri-resource-manager.intel.com/pod: dram,pmem\nspec:\n  containers:\n    - name: ${NAME}c0\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c0; done | awk '{r+=1;if(r<${WORN%M}*1024*1024/$BSIZE){worn[r]=\\$1;wr+=1;}if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WORN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WORN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n    - name: ${NAME}c1\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c1; done | awk '{r+=1;wmrn[r%(${WMRN%M}*1024*1024/$BSIZE)]=\\$1;wr+=1;if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WMRN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WMRN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n    - name: ${NAME}c2\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c2; done | awk '{r+=1;if (worm[r%(${WORM%M}*1024*1024/$BSIZE)]!=\\$1){worm[r%(${WORM%M}*1024*1024/$BSIZE)]=\\$1;wr+=1;}if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WORM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WORM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n    - name: ${NAME}c3\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c3; done | awk '{r+=1;if (wmrm[r%(${WMRM%M}*1024*1024/$BSIZE)]!=\\$1 || length(\\$1) > 0){wmrm[r%(${WMRM%M}*1024*1024/$BSIZE)]=\\$1;wr+=1;}if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WMRM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WMRM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test04-dynamic-page-demotion/code.var.sh",
    "content": "# Test migrating memory pages from DRAM to PMEM.\n# - Memory pages that are written once and never read\n#   must be migrated to PMEM and must stay there.\n# - Memory pages that are actively written and read\n#   must not be migrated to PMEM.\n# - Migration speed is as configured.\n\nvm-command \"echo 0 > /proc/sys/kernel/numa_balancing || true\"\n\n# Relaunch cri-resmgr with dynamic page demotion configuration.\ncri_resmgr_cfg=$TEST_DIR/cri-resmgr-dynamic-page-demotion.cfg\nterminate cri-resmgr\nlaunch cri-resmgr\n\n# Different memory usage profiles are implemented with awk\n# in order to manage with the same busybox image as other tests.\n# Memory size parameters for the busybox memory load pod:\n# - BSIZE: Block size in bytes (length of each stored string)\n#   The larger the block the faster the awk goes through its memory.\n#   If too large, memory for strings is no more allocated from heap\n#   which makes page tracking harder and breaks this test.\n# - WORN: Write Once Read Never\n# - WORM: Write Once Read Many\n# - WMRN: Write Many Read Never\n# - WMRM: Write Many Read Many\nPRINT_WRBYTES_IF=\"wr%1000==0 && wr<10000\"\nCPU=500m\nBSIZE=4096\nawkmem=2M\nWORN=$awkmem WORM=$awkmem WMRN=$awkmem WMRM=$awkmem create bb-memload\n\n# Calculate page migration speed from cri-resmgr configuration.\npages_per_second_per_process=\"$(awk '\n    /MaxPageMoveCount:/{mpmc=$2}\n    /PageMoveInterval:/{gsub(/[^0-9]/, \"\", $2); pmi=$2}\n    END{print mpmc/pmi}\n    ' < \"$cri_resmgr_cfg\")\"\n\n# After how many rounds (seconds) first migrations should be visible.\nfirst_migrations_visible=\"$(awk '\n    /PageScanInterval:/{gsub(/[^0-9]/, \"\", $2); print $2+8}\n    ' < \"$cri_resmgr_cfg\")\"\n\n# Expected migrated number of pages when fully migrated.\npages_error_margin=100\nfully_migrated_threshold=$(( ${awkmem%M} * 1024 * 1024 / 4096 - pages_error_margin ))\n\n# Maximum number of pages in PMEM when not migrated.\nnot_migrated_threshold=$pages_error_margin\n\n# Watch memory page locations and validate results.\nmemload_stats=\"$OUTPUT_DIR/memload-stats.txt\"\necho -n \"\" > \"$memload_stats\"\nmax_rounds=30\nround=0\ndeclare -A pmem_pages_prev # number of pages in PMEM in previous round\nfor wxrx in wmrm wmrn worm worn; do\n    pmem_pages_prev[$wxrx]=0\ndone\nwhile (( round < max_rounds )); do\n    vm-command-q '\n       cat /sys/devices/system/node/node[0-7]/meminfo | awk \"/Active:/{a[\\$2]=(\\$4/1024)}END{s=\\\"active mem\\\";for(n=0;n<8;n++){s=sprintf(\\\"%s N%d=%.0fM\\\",s,n,a[n])}print s}\"\n       for p in $(pidof awk); do\n           awkinfo=$(grep -a -o -E w[om]r[nm] /proc/$p/cmdline | head -n 1)\n           rss=$(awk \"/VmRSS:/{print \\$2}\" < /proc/$p/status);\n           pages=$(echo $(grep -v file= /proc/$p/numa_maps | tr \" \" \"\\n\" | awk -F= \"/N([0-9])/{s[\\$1]+=\\$2}END{for(n=0;n<8;n++)if (s[\\\"N\\\"n]>0)print \\\"N\\\"n\\\"=\\\"s[\\\"N\\\"n]}\"))\n           echo \"$awkinfo\" pid \"$p\" VmRSS \"$rss\" kB, \"pages:\" \"$pages\"\n       done' | while read line; do echo \"round $round $line\"; done | tee -a \"$memload_stats\"\n\n    echo \"validating...\"\n\n    # Check that at least something has migrated after scan period.\n    if (( round > first_migrations_visible )); then\n        grep -q -E 'pages:.*N[4-7]' \"$memload_stats\" ||\n            error \"any of the awk processes was not migrated to PMEM in time\"\n    fi\n\n    # Validate PMEM page migration speed.\n    # Allow double the configured speed because stats polling interval > 1s.\n    for wxrx in wmrm wmrn worm worn; do\n        pmem_pages_now=\"$(grep \"round $round $wxrx .*pages:\" < \"$memload_stats\" | awk 'BEGIN{RS=\" \";FS=\"=\";pmem=0}/N[4-9]/{pmem+=$2}END{print pmem}')\"\n        if (( pmem_pages_now - pmem_pages_prev[$wxrx] > 2 * pages_per_second_per_process )); then\n            error \"number of PMEM pages of $wxrx grew too quickly on this round\"\n        fi\n        pmem_pages_prev[$wxrx]=$pmem_pages_now\n    done\n\n    # Check that write-once-read-never (worn) has migrated and stays in PMEM.\n    if (( round > 20 )); then\n        worn_pmem_pages=\"$(grep \"round $round worn .*pages:\" < \"$memload_stats\" | awk 'BEGIN{RS=\" \";FS=\"=\";pmem=0}/N[4-9]/{pmem+=$2}END{print pmem}')\"\n        if (( worn_pmem_pages < fully_migrated_threshold )); then\n            error \"write-once-read-never was expected to end up and stay in PMEM, but only $worn_pmem_pages pages in PMEM.\"\n        fi\n    fi\n\n    # Check that write-many-read-many and -read-never (wmrm and wmrn) stay in DRAM.\n    for wmrx in wmrm wmrn; do\n        wmrx_pmem_pages=\"$(grep \"round $round $wmrx .*pages:\" < \"$memload_stats\" | awk 'BEGIN{RS=\" \";FS=\"=\";pmem=0}/N[4-9]/{pmem+=$2}END{print pmem}')\"\n        if (( wmrx_pmem_pages > not_migrated_threshold )); then\n            error \"$wmrx was expected to stay in DRAM, but $wmrx_pmem_pages pages migrated to PMEM.\"\n        fi\n    done\n\n    sleep 1 >/dev/null\n    round=$(( round + 1 ))\ndone\necho \"All rounds were good.\"\nkubectl delete pods --all --now --wait\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test04-dynamic-page-demotion/cri-resmgr-dynamic-page-demotion.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 250m\nresource-manager:\n  control:\n    page-migration:\n      PageScanInterval: 10s\n      PageMoveInterval: 1s\n      MaxPageMoveCount: 100\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test04-dynamic-page-demotion-deprecated-syntax/bb-memload.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  annotations:\n    cri-resource-manager.intel.com/memory-type: |\n      pod0c0: dram,pmem\n      pod0c1: dram,pmem\n      pod0c2: dram,pmem\n      pod0c3: dram,pmem\nspec:\n  containers:\n    - name: ${NAME}c0\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c0; done | awk '{r+=1;if(r<${WORN%M}*1024*1024/$BSIZE){worn[r]=\\$1;wr+=1;}if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WORN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WORN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n    - name: ${NAME}c1\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c1; done | awk '{r+=1;wmrn[r%(${WMRN%M}*1024*1024/$BSIZE)]=\\$1;wr+=1;if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WMRN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WMRN%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n    - name: ${NAME}c2\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c2; done | awk '{r+=1;if (worm[r%(${WORM%M}*1024*1024/$BSIZE)]!=\\$1){worm[r%(${WORM%M}*1024*1024/$BSIZE)]=\\$1;wr+=1;}if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WORM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WORM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n    - name: ${NAME}c3\n      image: busybox\n      imagePullPolicy: IfNotPresent\n      command:\n        - sh\n        - -c\n        - while :; do dd status=none if=/dev/zero bs=$(( $BSIZE - 7 )) count=1 | tr '\\\\\\0' 'A'; echo ${NAME}c3; done | awk '{r+=1;if (wmrm[r%(${WMRM%M}*1024*1024/$BSIZE)]!=\\$1 || length(\\$1) > 0){wmrm[r%(${WMRM%M}*1024*1024/$BSIZE)]=\\$1;wr+=1;}if($PRINT_WRBYTES_IF)print wr*$BSIZE;}'\n      resources:\n        requests:\n          cpu: ${CPU}\n          memory: $(( ${WMRM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n        limits:\n          cpu: ${CPU}\n          memory: $(( ${WMRM%M} * 1024 * 1024 / $BSIZE + 100000 ))k\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test04-dynamic-page-demotion-deprecated-syntax/code.var.sh",
    "content": "# Test migrating memory pages from DRAM to PMEM.\n# - Memory pages that are written once and never read\n#   must be migrated to PMEM and must stay there.\n# - Memory pages that are actively written and read\n#   must not be migrated to PMEM.\n# - Migration speed is as configured.\n\n# Relaunch cri-resmgr with dynamic page demotion configuration.\ncri_resmgr_cfg=$TEST_DIR/cri-resmgr-dynamic-page-demotion.cfg\nterminate cri-resmgr\nlaunch cri-resmgr\n\n# Different memory usage profiles are implemented with awk\n# in order to manage with the same busybox image as other tests.\n# Memory size parameters for the busybox memory load pod:\n# - BSIZE: Block size in bytes (length of each stored string)\n#   The larger the block the faster the awk goes through its memory.\n#   If too large, memory for strings is no more allocated from heap\n#   which makes page tracking harder and breaks this test.\n# - WORN: Write Once Read Never\n# - WORM: Write Once Read Many\n# - WMRN: Write Many Read Never\n# - WMRM: Write Many Read Many\nPRINT_WRBYTES_IF=\"wr%1000==0 && wr<10000\"\nCPU=500m\nBSIZE=4096\nawkmem=2M\nWORN=$awkmem WORM=$awkmem WMRN=$awkmem WMRM=$awkmem create bb-memload\n\n# Calculate page migration speed from cri-resmgr configuration.\npages_per_second_per_process=\"$(awk '\n    /MaxPageMoveCount:/{mpmc=$2}\n    /PageMoveInterval:/{gsub(/[^0-9]/, \"\", $2); pmi=$2}\n    END{print mpmc/pmi}\n    ' < \"$cri_resmgr_cfg\")\"\n\n# After how many rounds (seconds) first migrations should be visible.\nfirst_migrations_visible=\"$(awk '\n    /PageScanInterval:/{gsub(/[^0-9]/, \"\", $2); print $2+8}\n    ' < \"$cri_resmgr_cfg\")\"\n\n# Expected migrated number of pages when fully migrated.\npages_error_margin=100\nfully_migrated_threshold=$(( ${awkmem%M} * 1024 * 1024 / 4096 - pages_error_margin ))\n\n# Maximum number of pages in PMEM when not migrated.\nnot_migrated_threshold=$pages_error_margin\n\n# Watch memory page locations and validate results.\nmemload_stats=\"$OUTPUT_DIR/memload-stats.txt\"\necho -n \"\" > \"$memload_stats\"\nmax_rounds=30\nround=0\ndeclare -A pmem_pages_prev # number of pages in PMEM in previous round\nfor wxrx in wmrm wmrn worm worn; do\n    pmem_pages_prev[$wxrx]=0\ndone\nwhile (( round < max_rounds )); do\n    vm-command-q '\n       cat /sys/devices/system/node/node[0-7]/meminfo | awk \"/Active:/{a[\\$2]=(\\$4/1024)}END{s=\\\"active mem\\\";for(n=0;n<8;n++){s=sprintf(\\\"%s N%d=%.0fM\\\",s,n,a[n])}print s}\"\n       for p in $(pidof awk); do\n           awkinfo=$(grep -a -o -E w[om]r[nm] /proc/$p/cmdline | head -n 1)\n           rss=$(awk \"/VmRSS:/{print \\$2}\" < /proc/$p/status);\n           pages=$(echo $(grep -v file= /proc/$p/numa_maps | tr \" \" \"\\n\" | awk -F= \"/N([0-9])/{s[\\$1]+=\\$2}END{for(n=0;n<8;n++)if (s[\\\"N\\\"n]>0)print \\\"N\\\"n\\\"=\\\"s[\\\"N\\\"n]}\"))\n           echo \"$awkinfo\" pid \"$p\" VmRSS \"$rss\" kB, \"pages:\" \"$pages\"\n       done' | while read line; do echo \"round $round $line\"; done | tee -a \"$memload_stats\"\n\n    echo \"validating...\"\n\n    # Check that at least something has migrated after scan period.\n    if (( round > first_migrations_visible )); then\n        grep -q -E 'pages:.*N[4-7]' \"$memload_stats\" ||\n            error \"any of the awk processes was not migrated to PMEM in time\"\n    fi\n\n    # Validate PMEM page migration speed.\n    # Allow double the configured speed because stats polling interval > 1s.\n    for wxrx in wmrm wmrn worm worn; do\n        pmem_pages_now=\"$(grep \"round $round $wxrx .*pages:\" < \"$memload_stats\" | awk 'BEGIN{RS=\" \";FS=\"=\";pmem=0}/N[4-9]/{pmem+=$2}END{print pmem}')\"\n        if (( pmem_pages_now - pmem_pages_prev[$wxrx] > 2 * pages_per_second_per_process )); then\n            error \"number of PMEM pages of $wxrx grew too quickly on this round\"\n        fi\n        pmem_pages_prev[$wxrx]=$pmem_pages_now\n    done\n\n    # Check that write-once-read-never (worn) has migrated and stays in PMEM.\n    if (( round > 20 )); then\n        worn_pmem_pages=\"$(grep \"round $round worn .*pages:\" < \"$memload_stats\" | awk 'BEGIN{RS=\" \";FS=\"=\";pmem=0}/N[4-9]/{pmem+=$2}END{print pmem}')\"\n        if (( worn_pmem_pages < fully_migrated_threshold )); then\n            error \"write-once-read-never was expected to end up and stay in PMEM, but only $worn_pmem_pages pages in PMEM.\"\n        fi\n    fi\n\n    # Check that write-many-read-many and -read-never (wmrm and wmrn) stay in DRAM.\n    for wmrx in wmrm wmrn; do\n        wmrx_pmem_pages=\"$(grep \"round $round $wmrx .*pages:\" < \"$memload_stats\" | awk 'BEGIN{RS=\" \";FS=\"=\";pmem=0}/N[4-9]/{pmem+=$2}END{print pmem}')\"\n        if (( wmrx_pmem_pages > not_migrated_threshold )); then\n            error \"$wmrx was expected to stay in DRAM, but $wmrx_pmem_pages pages migrated to PMEM.\"\n        fi\n    done\n\n    sleep 1 >/dev/null\n    round=$(( round + 1 ))\ndone\necho \"All rounds were good.\"\nkubectl delete pods --all --now --wait\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test04-dynamic-page-demotion-deprecated-syntax/cri-resmgr-dynamic-page-demotion.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 250m\nresource-manager:\n  control:\n    page-migration:\n      PageScanInterval: 10s\n      PageMoveInterval: 1s\n      MaxPageMoveCount: 100\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/test05-guarantee-memory/code.var.sh",
    "content": "CRI_RESMGR_OUTPUT=\"cat cri-resmgr.output.txt | tr -d '\\0'\"\nCRI_RESMGR_ROTATE=\"echo > cri-resmgr.output.txt\"\n\npodno=0\nkubectl delete pod --all --now --wait\n\n# account for being done with test for the current pod\nnextpod () {\n    podno=$((podno+1))\n}\n\n# print current pod name\npod () {\n    echo pod$podno\n}\n\n# print current container name, by default for current pod\ncontainer () {\n    local _p _c\n    case $# in\n        0) _p=${podno}; _c=0;;\n        1) _p=${podno}; _c=$1;;\n        2) _p=$1; _c=$2;;\n        *)\n            _c=pod${1}c${2}; shift 2\n            echo ${_c}_INVALID_WITH_EXTRA_${#}_ARGS_$(echo $* | tr -s ' ' '_')\n            return 1\n            ;;\n    esac\n    case $_p in\n        +*|-*) _p=$((${podno}$_p));;\n    esac\n    echo pod${_p}c${_c}\n}\n\n# rotate cri-resmgr logs\nrotate_log () {\n    vm-command \"$CRI_RESMGR_ROTATE\"\n}\n\n###########################################################################\n# test #1: squeeze multiple containers in every NUMA node\n#\n# We squeeze an increasing number of containers in all NUMA node pools\n# in a loop. For every iteration we calculate the usable amount of CPU\n# and memory based on the available number of NUMA nodes and the amount\n# of CPU and memory per NUMA node. We use a conservative estimate for\n# the amount of memory available per NUMA node because some of them will\n# have a sizeable allocation by the kernel.\n#\n\nrotate_log\n\n# use conservative estimate for available memory per node\nPER_NODE_MEM=$((1500+4000))\nPER_NODE_CPU=1000\nPER_NODE_PMEM=1\nNODE_COUNT_TOTAL=4\n# All nodes have only a single CPU. Thus, with any (< 1000m) CPU reservation\n# we'll have one node (#0) fully reserved for kube-system containers. Hence,\n# our (usable) node count is one less than the total one.\nNODE_COUNT=$((NODE_COUNT_TOTAL - 1))\n\nfor pernode in 2 3 4; do\n    cpu=$(echo \"scale=3;0.75*$PER_NODE_CPU/$pernode\" | bc | cut -d '.' -f1)\n    mem=$(echo \"scale=3;0.75*$PER_NODE_MEM/$pernode\" | bc | cut -d '.' -f1)\n    CPU=${cpu}m MEM=${mem}Mi CONTCOUNT=$((pernode*NODE_COUNT)) create guaranteed\n\n    echo \"Verify that any pod's containers were not raised to guarantee memory\"\n    echo \"\"\n    vm-command \"$CRI_RESMGR_OUTPUT | grep upward\" && {\n        pp mems\n        error \"Unexpected memset upward expansion detected!\"\n    }\n\n    echo \"Verify that all containers are pinned to a single NUMA node\"\n    echo \"\"\n    c=0; while [ \"$c\" -lt \"$((pernode*NODE_COUNT))\" ]; do\n        verify \"len(mems['$(container $c)']) == $((1+PER_NODE_PMEM))\"\n        c=$((c+1))\n    done\n\n    kubectl delete pod --all --now --wait\n\n    nextpod\ndone\n\n###########################################################################\n# test #2: negative test for lifting containers upwards.\n#\n# This test first creates a pod that fits into a singe NUMA node. Then\n# it creates a pod that allocates a negligible amount of memory from the\n# root node (by asking for more CPU than a single NUMA node can provide).\n# The allocation of this pod must not cause lifting pod0 containers'\n# memory assignment upwards in the pool tree.\n#\n\nrotate_log\n\nCPU=200m MEM=100M create guaranteed\nreport allowed\nverify \"len(mems['$(container 0)']) == 2\"\n\nnextpod\n\nCPU=1200m MEM=100M create guaranteed\nreport allowed\nverify \"len(mems['$(container -1 0)']) == 2\" \\\n       \"len(mems['$(container 0)']) == 8\"\n\necho \"Verify that $(pod)'s containers were not raised to guarantee memory\"\necho \"\"\nvm-command \"$CRI_RESMGR_OUTPUT | grep upward\" && {\n    pp mems\n    error \"Unexpected memset upward expansion detected!\"\n}\n\nkubectl delete pod $(pod) --now --wait --ignore-not-found\n\nnextpod\n\n###########################################################################\n# test #3: positive test for lifting containers upwards.\n#\n# This test creates two containers which both get their own socket and\n# take > 50 % of their socket's mem. Then it reserves a lot of memory\n# from the root node to force lifting one of the containers. Every socket\n# has 6G PMEM+DRAM, one pods containers take 5G and the other take 2G.\n# => pessimistic max 7G will not fit to any socket\n# => no memory grants can be given to any socket alone.\n#\n\nCPU=200m MEM=5G CONTCOUNT=2 create guaranteed\nreport allowed\nverify \"len(mems['$(container 0)']) == 2\" \\\n       \"len(mems['$(container 1)']) == 2\" \\\n       \"mems['$(container 0)'] != mems['$(container 1)']\"\n\nnextpod\n\nCPU=1200m MEM=2G create guaranteed\n\necho \"Verify that $(pod)'s containers were raised to guarantee memory\"\necho \"\"\nvm-command \"$CRI_RESMGR_OUTPUT | grep upward\" || {\n    error \"Expected memset upward expansion not found!\"\n}\nreport allowed\npp mems\nverify \"len(mems['$(container 0)']) == 8\" \\\n       \"len(mems['$(container -1 0)']) == 8\" \\\n       \"len(mems['$(container -1 1)']) == 8\"\n\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/c4pmem4/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"threads\": 1, \"cores\": 1, \"packages\": 4},\n    {\"mem\": \"4G\", \"node-dist\": {\"2\": 17}},\n    {\"mem\": \"4G\", \"node-dist\": {\"3\": 17}},\n    {\"mem\": \"4G\", \"node-dist\": {\"0\": 17}},\n    {\"mem\": \"4G\", \"node-dist\": {\"1\": 17}}\n]\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/cri-resmgr.cfg",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/code.var.sh",
    "content": "\n# pod0: Test that 4 guaranteed containers eligible for isolated CPU allocation\n# gets evenly spread over NUMA nodes.\nCONTCOUNT=4 CPU=1 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) == 1' \\\n    'len(cpus[\"pod0c1\"]) == 1' \\\n    'len(cpus[\"pod0c2\"]) == 1' \\\n    'len(cpus[\"pod0c3\"]) == 1' \\\n    'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod0c1\"], cpus[\"pod0c2\"], cpus[\"pod0c3\"])' \\\n    'disjoint_sets(nodes[\"pod0c0\"], nodes[\"pod0c1\"], nodes[\"pod0c2\"], nodes[\"pod0c3\"])'\n\nkubectl delete pods --all --now --wait\n\n# pod1: Test that 4 guaranteed containers not eligible for isolated CPU allocation\n# gets evenly spread over NUMA nodes.\nCONTCOUNT=4 CPU=3 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod1c0\"]) == 3' \\\n    'len(cpus[\"pod1c1\"]) == 3' \\\n    'len(cpus[\"pod1c2\"]) == 3' \\\n    'len(cpus[\"pod1c3\"]) == 3' \\\n    'disjoint_sets(cpus[\"pod1c0\"], cpus[\"pod1c1\"], cpus[\"pod1c2\"], cpus[\"pod1c3\"])' \\\n    'disjoint_sets(nodes[\"pod1c0\"], nodes[\"pod1c1\"], nodes[\"pod1c2\"], nodes[\"pod1c3\"])'\n\nkubectl delete pods --all --now --wait\n\n# pod2: Test that 4 burstable containers not eligible for isolated/exclusive CPU allocation\n# gets evenly spread over NUMA nodes.\nCONTCOUNT=4 CPUREQ=2 CPULIM=4 create burstable\nreport allowed\nverify \\\n    'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod2c1\"], cpus[\"pod2c2\"], cpus[\"pod2c3\"])' \\\n    'disjoint_sets(nodes[\"pod2c0\"], nodes[\"pod2c1\"], nodes[\"pod2c2\"], nodes[\"pod2c3\"])'\n\nkubectl delete pods --all --now --wait\n\n# pod3: Test that initContainer resources are freed before launching\n# containers: instantiate 5 init containers, each requiring 5 CPUs. If\n# the resources of an init container weren't freed before next init\n# container is launched, not all of them could be launched, and not\n# real containers could fit on the node.\nICONTCOUNT=5 ICONTSLEEP=1 CONTCOUNT=2 CPU=5 MEM=100M create guaranteed\nreport allowed\nverify \\\n    'disjoint_sets(cpus[\"pod3c0\"], cpus[\"pod3c1\"])' \\\n    'disjoint_sets(nodes[\"pod3c0\"], nodes[\"pod3c1\"])' \\\n    'disjoint_sets(packages[\"pod3c0\"], packages[\"pod3c1\"])'\n\nkubectl delete pods --all --now --wait\n\n# pod4: Test that with pod colocation enabled containers within a pod get\n# colocated (assigned topologically close to each other) as opposed to being\n# evenly spread out.\nterminate cri-resmgr\ncri_resmgr_cfg=$(COLOCATE_PODS=true instantiate cri-resmgr.cfg)\nlaunch cri-resmgr\n\nCONTCOUNT=4 CPU=100m create guaranteed\nreport allowed\nverify \\\n    'cpus[\"pod4c1\"] == cpus[\"pod4c0\"]' \\\n    'cpus[\"pod4c2\"] == cpus[\"pod4c0\"]' \\\n    'cpus[\"pod4c3\"] == cpus[\"pod4c0\"]'\n\nkubectl delete pods --all --now --wait\n\n# pod{5,6,7}: Test that with namespace colocation enabled containers of pods\n# in the same namespace get colocated (assigned topologically close to each\n# other) as opposed to being evenly spread out.\nterminate cri-resmgr\ncri_resmgr_cfg=$(COLOCATE_NAMESPACES=true instantiate cri-resmgr.cfg)\nlaunch cri-resmgr\n\nkubectl create namespace test-ns\n\nCONTCOUNT=1 CPU=100m namespace=test-ns create guaranteed\nCONTCOUNT=1 CPU=100m namespace=test-ns create guaranteed\nCONTCOUNT=2 CPU=100m namespace=test-ns create guaranteed\nreport allowed\nverify \\\n    'cpus[\"pod6c0\"] == cpus[\"pod5c0\"]' \\\n    'cpus[\"pod7c0\"] == cpus[\"pod5c0\"]' \\\n    'cpus[\"pod7c1\"] == cpus[\"pod5c0\"]'\n\nkubectl delete namespace test-ns --now --wait --ignore-not-found\n\n# Restore default test configuration, restart cri-resmgr.\nterminate cri-resmgr\ncri_resmgr_cfg=$(instantiate cri-resmgr.cfg)\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test00-basic-placement/cri-resmgr.cfg.in",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\n  topology-aware:\n    ColocatePods: $(echo ${COLOCATE_PODS:-false})\n    ColocateNamespaces: $(echo ${COLOCATE_NAMESPACES:-false})\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test01-always-fits/code.var.sh",
    "content": "# Test that guaranteed and burstable pods get the CPUs they require\n# when there are enough CPUs available.\n\n# pod0, fits in a core\nCPU=1 create guaranteed\nreport allowed\nverify \\\n    'node_ids(nodes[\"pod0c0\"]) == {1}' \\\n    'cpu_ids(cpus[\"pod0c0\"]) == {4}'\n\n# pod1, takes full core - from a different node than pod0\nCPU=2 create guaranteed\nreport allowed\nverify \\\n    'cpu_ids(cpus[\"pod0c0\"]) == {4}' \\\n    'node_ids(nodes[\"pod1c0\"]) == {2}' \\\n    'cpu_ids(cpus[\"pod1c0\"]) == {8, 9}'\n\n# pod2, does not fit in a core but fits in a node\nCPU=3 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) == 1' \\\n    'len(cpus[\"pod1c0\"]) == 2' \\\n    'len(cores[\"pod1c0\"]) == 1' \\\n    'len(cpus[\"pod2c0\"]) == 3' \\\n    'len(cores[\"pod2c0\"]) == 2' \\\n    'len(nodes[\"pod2c0\"]) == 1' \\\n    'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"])'\n\n# pod3, tries to fully exhaust the shared subset of a (NUMA node) pool\n# Currently topology-aware refuses to exhaust even idle shared CPU subsets of\n# a pool. Therefore such attempts will try to squeeze the container to\n# another pool at the same level or, if none found, push the container\n# one level up to the parent pool.\n#\n# There is a pending commit to change this behavior to allow exhausting\n# fully idle subsets (no active shared grants). Once that lands, update\n# this test accordingly as well.\nCPU=4 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) == 1' \\\n    'len(cpus[\"pod1c0\"]) == 2' \\\n    'len(cores[\"pod1c0\"]) == 1' \\\n    'len(cpus[\"pod2c0\"]) == 3' \\\n    'len(cores[\"pod2c0\"]) == 2' \\\n    'len(nodes[\"pod2c0\"]) == 1' \\\n    'len(cpus[\"pod3c0\"]) == 4' \\\n    'len(cores[\"pod3c0\"]) == 2' \\\n    'len(nodes[\"pod3c0\"]) == 2' \\\n    'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod1c0\"], cpus[\"pod2c0\"], cpus[\"pod3c0\"])'\n\nkubectl delete pods --all --now --wait\n\n# pod4, fits in a die/package\nCPU=5 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod4c0\"]) == 5' \\\n    'len(cores[\"pod4c0\"]) == 3' \\\n    'len(nodes[\"pod4c0\"]) == 2' \\\n    'len(dies[\"pod4c0\"]) == 1'\n\n# pod5, takes a full die/package\n# cpu0 is reserved, so allocating 7 CPUs is expected to fill package0/die0\nCPU=7 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod4c0\"]) == 5' \\\n    'len(cores[\"pod4c0\"]) == 3' \\\n    'len(nodes[\"pod4c0\"]) == 2' \\\n    'len(dies[\"pod4c0\"]) == 1' \\\n    'len(cpus[\"pod5c0\"]) == 7' \\\n    'len(cores[\"pod5c0\"]) == 4' \\\n    'len(dies[\"pod5c0\"]) == 1' \\\n    'disjoint_sets(cpus[\"pod4c0\"], cpus[\"pod5c0\"])'\n\nkubectl delete pods --all --now --wait\n\n# pod6, doesn't fit in a die/package, needs virtual root\nCPU=9 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod6c0\"]) == 9' \\\n    'len(packages[\"pod6c0\"]) == 2'\n\nkubectl delete pods --all --now --wait\n\nreset counters\n\n# pod0, burstable containers must get at least the cores they require\nCPUREQ=3 CPULIM=$(( CPUREQ + 1 )) create burstable\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) >= 2'\n\n# pod1\nCPUREQ=4 CPULIM=$(( CPUREQ + 1 )) create burstable\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) >= 2' \\\n    'len(cpus[\"pod1c0\"]) >= 4'\n\n# pod2\nCPUREQ=5 CPULIM=$(( CPUREQ + 1 )) create burstable\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) >= 2' \\\n    'len(cpus[\"pod1c0\"]) >= 4' \\\n    'len(cpus[\"pod2c0\"]) >= 5'\n\nkubectl delete pods pod0 pod1 --now --wait --ignore-not-found\n\n# pod3\nCPUREQ=8 CPULIM=$(( CPUREQ + 1 )) create burstable\nreport allowed\nverify \\\n    'len(cpus[\"pod2c0\"]) >= 5' \\\n    'len(cpus[\"pod3c0\"]) >= 8'\n\nkubectl delete pods pod3 --now --wait --ignore-not-found\n\n# pod4, pod5 (and existing pod2) take 5 and 4 CPUs. As there are 8\n# CPUs/node, pod2 and pod4 have consumed free node\n# pairs/dies/packages. pod5 will be spread across nodes.\nCPUREQ=5 CPULIM=$(( CPUREQ + 1 )) create burstable\nreport allowed\nCPUREQ=4 CPULIM=$(( CPUREQ + 1 )) create burstable\nreport allowed\nverify \\\n    'len(cpus[\"pod2c0\"]) >= 5' \\\n    'len(cpus[\"pod4c0\"]) >= 5' \\\n    'len(cpus[\"pod5c0\"]) >= 4'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test02-shrink-and-grow-shared/code.var.sh",
    "content": "# pod0: require 10 out of 16 CPUs with two containers.\n# Both containers should fit in their own die. (8 CPUs per die.)\nCPU=5 CONTCOUNT=2 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) == 5' \\\n    'len(cpus[\"pod0c1\"]) == 5' \\\n    'len(nodes[\"pod0c0\"]) == len(nodes[\"pod0c1\"]) == 2' \\\n    'len(dies[\"pod0c0\"]) == len(dies[\"pod0c1\"]) == 1' \\\n    'disjoint_sets(cpus[\"pod0c0\"], cpus[\"pod0c1\"])'\n\n# pod1: two containers in a besteffort pod.\nCONTCOUNT=2 create besteffort\nreport allowed\nverify \\\n    'len(cpus[\"pod0c0\"]) == 5' \\\n    'len(cpus[\"pod0c1\"]) == 5' \\\n    'disjoint_sets(set.union(cpus[\"pod0c0\"], cpus[\"pod0c1\"]))' \\\n    'len(cpus[\"pod1c0\"]) > 0' \\\n    'len(cpus[\"pod1c1\"]) > 0' \\\n    'disjoint_sets(\n         set.union(cpus[\"pod0c0\"], cpus[\"pod0c1\"]),\n         set.union(cpus[\"pod1c0\"], cpus[\"pod1c1\"]))'\n\n# Delete pod0\ndelete pods/pod0 --now\nreport allowed\n\n# Next squeeze the besteffort containers to the minimum.\n\n# pod2: 4 guaranteed containers, each requiring 3 CPUs.\nCPU=3 CONTCOUNT=4 create guaranteed\nreport allowed\nverify \\\n    'len(cpus[\"pod2c0\"]) == len(cpus[\"pod2c1\"]) == len(cpus[\"pod2c2\"]) == len(cpus[\"pod2c3\"]) == 3' \\\n    'disjoint_sets(cpus[\"pod2c0\"], cpus[\"pod2c1\"], cpus[\"pod2c2\"], cpus[\"pod2c3\"])'\n\n# pod3: 1 guaranteed container taking the last non-reserved CPU\n# that can be taken from shared pools.\nCPU=1 create guaranteed\nreport allowed\nverify \\\n    'disjoint_sets(\n         set.union(cpus[\"pod1c0\"], cpus[\"pod1c1\"]),\n         set.union(cpus[\"pod3c0\"],\n                   cpus[\"pod2c0\"], cpus[\"pod2c1\"], cpus[\"pod2c2\"], cpus[\"pod2c3\"]))'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test03-simple-affinity/code.var.sh",
    "content": "# Test that guaranteed and burstable pods get the CPUs they require\n# when there are enough CPUs available.\n\ninject-affinities() {\n    local var=$1 srcdst src dst hdr line\n    shift\n    if [ -z \"$var\" ] || [ -z \"${!var}\" ]; then\n        return 0\n    fi\n    case \"$var\" in\n        ANTI_*|*_ANTI_*) hdr=\"cri-resource-manager.intel.com/anti-affinity\";;\n        *)      hdr=\"cri-resource-manager.intel.com/affinity\";;\n    esac\n    for srcdst in ${!var}; do\n        src=${srcdst%:*}\n        dst=${srcdst#*:}\n        [ -n \"$hdr\" ] && { echo \"    $hdr: |\"; hdr=\"\"; }\n        line=\"$src: [ ${dst//,/, } ]\"\n        echo 1>&2 \"* [affinity]: injecting affinity '$line'\"\n        echo \"      $line\"\n    done\n}\n\nderef_keys() {\n    eval \"echo \\${!$1[@]}\"\n}\n\nderef_value() {\n    eval \"echo \\${$1[$2]}\"\n}\n\ninject-annotations() {\n    local var=$1 values key value\n    shift\n    if [ -z \"$var\" ] || [ -z \"${!var}\" ]; then\n        return 0\n    fi\n    for key in $(deref_keys ${!var}); do\n        value=$(deref_value ${!var} $key)\n        line=\"$key: $value\"\n        echo 1>&2 \"* [annotation]: injecting annotation '$line'\"\n        echo \"    $line\"\n    done\n}\n\n# pod0\n# 4 containers, no affinities => spread out evenly over NUMA nodes\nCONTCOUNT=4 CPU=1 create guaranteed+affinity\nreport allowed\n\nverify \\\n    'nodes[\"pod0c0\"] == {\"node1\"}' \\\n    'nodes[\"pod0c1\"] == {\"node2\"}' \\\n    'nodes[\"pod0c2\"] == {\"node3\"}' \\\n    'nodes[\"pod0c3\"] == {\"node0\"}'\n\nkubectl delete pods --all --now --wait\n\n# pod1\n# 4 containers, affinites [0,1], [2,3] => colocate c0,c1 in node1, c2,c3 in node2\nCONTCOUNT=4 AFFINITIES=\"pod1c0:pod1c1 pod1c2:pod1c3\" CPU=1 create guaranteed+affinity\nreport allowed\n\nverify \\\n    'nodes[\"pod1c0\"] == nodes[\"pod1c1\"] == {\"node1\"}' \\\n    'nodes[\"pod1c2\"] == nodes[\"pod1c3\"] == {\"node2\"}'\n\nkubectl delete pods --all --now --wait\n\n# pod2\n# 6 containers, anti-affinites 4:[0,1,2], 5:[0,2,3]\n#   => don't co-locate 4 with {0,1,2}, or 5 with {0,2,3}\nCONTCOUNT=6 ANTI_AFFINITIES=\"pod2c4:pod2c0,pod2c1,pod2c2 pod2c5:pod2c0,pod2c2,pod2c3\" CPU=1 \\\n    create guaranteed+affinity\nreport allowed\n\nverify \\\n    'disjoint_sets(nodes[\"pod2c4\"], nodes[\"pod2c0\"], nodes[\"pod2c1\"], nodes[\"pod2c2\"])' \\\n    'disjoint_sets(nodes[\"pod2c5\"], nodes[\"pod2c0\"], nodes[\"pod2c2\"], nodes[\"pod2c3\"])'\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test03-simple-affinity/guaranteed+affinity.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  labels:\n    app: ${NAME}\n  annotations:\n$([ -z \"$(type -t inject-affinities)\" ] || inject-affinities AFFINITIES)\n$([ -z \"$(type -t inject-affinities)\" ] || inject-affinities ANTI_AFFINITIES)\n$([ -z \"$(type -t inject-annotations)\" ] || inject-annotations ANNOTATIONS)\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test04-available-resources/code.var.sh",
    "content": "# Test that AvailableResources are honored.\n\n# Test explicit cpuset in AvailableResources.CPU\nterminate cri-resmgr\nAVAILABLE_CPU=\"cpuset:4-7,8-11\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-available-resources.cfg)\nlaunch cri-resmgr\n\n# pod0: exclusive CPUs\nCPU=3 create guaranteed\nverify \"cpus['pod0c0'] == {'cpu04', 'cpu05', 'cpu06'}\" \\\n       \"mems['pod0c0'] == {'node1'}\"\n\n# pod1: shared CPUs\nCONTCOUNT=2 CPU=980m create guaranteed\nverify \"cpus['pod1c0'] == {'cpu08', 'cpu09', 'cpu10'}\" \\\n       \"cpus['pod1c1'] == {'cpu08', 'cpu09', 'cpu10'}\" \\\n       \"mems['pod1c0'] == {'node2'}\" \\\n       \"mems['pod1c1'] == {'node2'}\"\nkubectl delete pods --all --now --wait\nreset counters\n\n# Test cgroup cpuset directory in AvailableResources.CPU\n\ntest-and-verify-allowed() {\n    # pod0: shared CPUs\n    CONTCOUNT=2 CPU=980m create guaranteed\n    report allowed\n    verify \"cpus['pod0c0'] == {'cpu0$1', 'cpu0$2', 'cpu0$3'}\" \\\n           \"cpus['pod0c1'] == {'cpu0$4'}\"\n\n    # pod1: exclusive CPU\n    CPU=1 create guaranteed\n    report allowed\n    verify \"disjoint_sets(cpus['pod1c0'], cpus['pod0c0'])\" \\\n           \"disjoint_sets(cpus['pod1c0'], cpus['pod0c1'])\"\n\n    kubectl delete pods --all --now --wait\n    reset counters\n}\n\nif vm-command \"[ -d /sys/fs/cgroup/cpuset ]\"; then\n    # cgroup v1\n    CGROUP_CPUSET=/sys/fs/cgroup/cpuset\nelse\n    # cgroup v2\n    CGROUP_CPUSET=/sys/fs/cgroup\nfi\nCRIRM_CGROUP=$CGROUP_CPUSET/cri-resmgr-test-05-1\nvm-command \"rmdir $CRIRM_CGROUP; mkdir $CRIRM_CGROUP; echo 1-4,11 > $CRIRM_CGROUP/cpuset.cpus\"\n\nterminate cri-resmgr\nAVAILABLE_CPU=\"\\\"$CRIRM_CGROUP\\\"\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-available-resources.cfg)\nlaunch cri-resmgr\ntest-and-verify-allowed 1 2 3 4\nvm-command \"rmdir $CRIRM_CGROUP || true\"\n\nCRIRM_CGROUP=$CGROUP_CPUSET/cri-resmgr-test-05-2\nvm-command \"rmdir $CRIRM_CGROUP; mkdir $CRIRM_CGROUP; echo 5-8,11 > $CRIRM_CGROUP/cpuset.cpus\"\n\nterminate cri-resmgr\nAVAILABLE_CPU=\"\\\"${CRIRM_CGROUP#/sys/fs/cgroup/cpuset}\\\"\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-available-resources.cfg)\nlaunch cri-resmgr\ntest-and-verify-allowed 5 6 7 8\nvm-command \"rmdir $CRIRM_CGROUP || true\"\n\n# cleanup, do not leave weirdly configured cri-resmgr running\nterminate cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test04-available-resources/cri-resmgr-available-resources.cfg.in",
    "content": "policy:\n  Active: topology-aware\n  AvailableResources:\n    cpu: ${AVAILABLE_CPU}\n  ReservedResources:\n    cpu: cpuset:11\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/code.var.sh",
    "content": "# Test that\n# - kube-system containers are pinned on Reserved CPUs.\n# - Reserved CPU allocation and releasing works.\n# - A pod cannot be launched if reserved CPU capacity in insufficient.\n\nAVAILABLE_CPU=\"cpuset:4-7,8-13\"\n\ncri_resmgr_cfg_orig=$cri_resmgr_cfg\n\n# This script will create pods to the kube-system namespace\n# that is not automatically cleaned up by the framework.\n# Make sure the namespace is clear when starting the test and clean it up\n# if exiting with success. Otherwise leave the pod running for\n# debugging in case of a failure.\ncleanup-kube-system() {\n    ( kubectl delete pods pod0 pod1 pod2 pod3 pod4 pod5 -n kube-system --now --wait --ignore-not-found ) || true\n}\ncleanup-kube-system\n\n# Test launch failure, Reserved CPUs is not subset of Available CPUs\nterminate cri-resmgr\nRESERVED_CPU=\"cpuset:3,7,11,15\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved.cfg)\n( launch cri-resmgr ) && error \"unexpected success\" || {\n    echo \"Launch failed as expected\"\n}\n\n# Test launch failure, there are more reserved CPUs than available CPUs\nterminate cri-resmgr\nRESERVED_CPU=\"11\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved.cfg)\n( launch cri-resmgr ) && error \"unexpected success\" || {\n    echo \"Launch failed as expected\"\n}\n\n# Test that BestEffort containers are allowed to run on both Reserved\n# CPUs when the CPUs are on the same NUMA node.\nterminate cri-resmgr\nRESERVED_CPU=\"cpuset:10-11\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved.cfg)\nlaunch cri-resmgr\n\nnamespace=kube-system CONTCOUNT=3 create besteffort\nreport allowed\nverify \"cpus['pod0c0'] == cpus['pod0c1'] == cpus['pod0c2'] == {'cpu10', 'cpu11'}\"\nkubectl delete -n kube-system pods pod0 --now --wait --ignore-not-found\n\n# Test that BestEffort containers are pinned to reserved CPUs.\nterminate cri-resmgr\nRESERVED_CPU=\"cpuset:7,11\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved.cfg)\nlaunch cri-resmgr\n\nnamespace=kube-system CONTCOUNT=4 create besteffort\nreport allowed\nverify \"cpus['pod1c0'] == cpus['pod1c1'] == cpus['pod1c2'] == cpus['pod1c3']\" \\\n       \"cpus['pod1c0'] == {'cpu07', 'cpu11'}\"\n\n# Test that guaranteed kube-system pods are pinned to Reserved CPUs.\nnamespace=kube-system CPU=200m CONTCOUNT=4 create guaranteed\nreport allowed\nverify \"cpus['pod2c0'] == cpus['pod2c1'] == cpus['pod2c2'] == cpus['pod2c3']\" \\\n       \"cpus['pod2c0'] == {'cpu07', 'cpu11'}\"\n\n# Test requesting more reserved CPUs than available on single node\n# but what fits in the node tree.\n# pod2 already consumed 4 * 200m of reserved CPUs that have been balanced\n# so that at least 200m from both nodes have been consumed. There are\n# at most 800m reserved CPUs free on both nodes. Root node still has\n# 1200m free. That is, 1000m requesting, isolated-looking guaranteed\n# pod should fit in because reserved CPUs are not isolated.\n#\n# Run this twice to make sure allocated reserved CPUs are released correctly.\nfor pod in pod3 pod4; do\n    namespace=kube-system CPU=1 CONTCOUNT=1 create guaranteed\n    verify \"cpus['${pod}c0'] == {'cpu07', 'cpu11'}\"\n    kubectl delete -n kube-system pods/$pod --now --wait --ignore-not-found\ndone\n\n# Test requesting more reserved CPUs than available in the system.\n# pod5 is expected to run on shared CPUs.\nnamespace=kube-system CPU=2 CONTCOUNT=1 create guaranteed\nreport allowed\nverify \"cpus['pod5c0'] == {'cpu04', 'cpu05', 'cpu06', 'cpu08', 'cpu09', 'cpu10', 'cpu12', 'cpu13'}\"\n\ncleanup-kube-system\n\n# Test that the first available CPUs are reserved when reserving milli CPUs.\n# The number of reserved CPUs is the ceiling of the milli CPUs.\nreset counters\nterminate cri-resmgr\nRESERVED_CPU=\"2250m\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved.cfg)\nlaunch cri-resmgr\nnamespace=kube-system CPU=2 CONTCOUNT=1 create besteffort\nverify \"cpus['pod0c0'] == {'cpu04', 'cpu05', 'cpu06'}\"\n\nkubectl delete -n kube-system pods/pod0 --now --wait --ignore-not-found\n\nterminate cri-resmgr\ncri_resmgr_cfg=$cri_resmgr_cfg_orig\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test05-reserved-resources/cri-resmgr-reserved.cfg.in",
    "content": "policy:\n  Active: topology-aware\n  AvailableResources:\n    cpu: ${AVAILABLE_CPU}\n  ReservedResources:\n    cpu: ${RESERVED_CPU}\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/code.var.sh",
    "content": "source $TEST_DIR/codelib.sh || {\n    echo \"error importing codelib.sh\"\n    exit 1\n}\n\n# Clean test pods from the kube-system namespace\n( kubectl delete pods --now --wait --ignore-not-found -n kube-system $(kubectl get pods -n kube-system | awk '/t[0-9]r[gb][ue]/{print $1}') ) || true\n\n# Run generated*.sh test scripts in this directory.\ngenscriptcount=0\nfor genscript in \"$TEST_DIR\"/generated*.sh; do\n    if [ ! -f \"$genscript\" ]; then\n        continue\n    fi\n    (\n        paralleloutdir=\"$outdir/parallel$genscriptcount\"\n        [ -d \"$paralleloutdir\" ] && rm -rf \"$paralleloutdir\"\n        mkdir \"$paralleloutdir\"\n        OUTPUT_DIR=\"$paralleloutdir\"\n        COMMAND_OUTPUT_DIR=\"$paralleloutdir/commands\"\n        mkdir \"$COMMAND_OUTPUT_DIR\"\n        source \"$genscript\" 2>&1 | sed -u -e \"s/^/$(basename \"$genscript\"): /g\"\n    ) &\n    genscriptcount=$(( genscriptcount + 1))\ndone\n\nif [[ \"$genscriptcount\" == \"0\" ]]; then\n    echo \"WARNING:\"\n    echo \"WARNING: Skipping fuzz tests:\"\n    echo \"WARNING: - Generated tests not found.\"\n    echo \"WARNING: - Generate a test by running:\"\n    echo \"WARNING:   $TEST_DIR/generate.sh\"\n    echo \"WARNING: - See test generation options:\"\n    echo \"WARNING:   $TEST_DIR/generate.sh --help\"\n    echo \"WARNING:\"\n    sleep 5\n    exit 0\nfi\n\necho \"waiting for $genscriptcount generated tests to finish...\"\nwait\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/codelib.sh",
    "content": "container-exit0() {\n    # Terminate a container by killing the \"sleep inf\" child process in\n    # echo CONTNAME $(sleep inf)\n    local contname=\"$1\"\n    vm-command \"contpid=\\$(ps axf | grep -A1 'echo $contname' | grep -v grep | awk '/_ sleep inf/{print \\$1}'); kill -KILL \\$contpid\"\n}\n\ncontainer-signal() {\n    local contname=\"$1\"\n    local signal=\"$2\"\n    vm-command \"pkill -$signal -f 'echo $contname'\"\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/fuzz.aal",
    "content": "language python {\n    max_mem=7500  # maximum memory on VM in MB\n    max_cpu=15000 # maximum CPUs on node in mCPU\n    max_reserved_cpu=1000 # maximum reserved CPUs on node in mCPU\n    class Vars:\n        # namespace for variables in input names\n        def __repr__(self):\n            return \"{\" + \",\".join(\"%s:%s\" % (a, getattr(self, a)) for a in sorted(self.__dict__.keys()) if not a.startswith(\"_\")) + \"}\\n\"\n    def inputvars(input_name):\n        # parse VAR=VALUE's from input_name\n        v = Vars()\n        for word in input_name.split():\n            keyvalue = word.split(\"=\")\n            if len(keyvalue) == 2:\n                if (keyvalue[1].endswith(\"m\") or keyvalue[1].endswith(\"M\")) and len(keyvalue[1]) > 1 and keyvalue[1][-2] in '0123456789':\n                    keyvalue[1] = keyvalue[1][:-1]\n                try:\n                    setattr(v, keyvalue[0], int(keyvalue[1]))\n                except:\n                    setattr(v, keyvalue[0], keyvalue[1])\n        return v\n}\n\nvariables {\n    mem, cpu, reserved_cpu, pods\n}\n\ninitial_state {\n    mem=0\n    cpu=0\n    reserved_cpu=0\n    pods={}\n}\n\n# Create non-reserved CPU pods\n# On this topology, there is\n# - 2G mem/numanode, 4G mem/package, 8G mem in total\n# -  4 CPU/numanode,  8 CPU/package, 16 CPU in total\ninput\n    \"NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed\",\n    \"NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed\",\n    \"NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed\",\n    \"NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed\",\n    \"NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed\",\n    \"NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable\",\n    \"NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable\",\n    \"NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort\",\n    \"NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort\"\n{\n    guard {\n        v = inputvars(input_name)\n        return (v.NAME not in pods\n                and (mem + v.MEM * v.CONTCOUNT < max_mem)\n                and (cpu + v.CPU * v.CONTCOUNT < max_cpu))\n    }\n    body {\n        v = inputvars(input_name)\n        v.namespace = getattr(v, \"namespace\", \"default\")\n        mem += v.MEM * v.CONTCOUNT\n        cpu += v.CPU * v.CONTCOUNT\n        pods[v.NAME] = v\n    }\n}\n\n# Create pods to the kube-system namespace\ninput\n    \"NAME=rgu0 CONTCOUNT=2 CPU=100m MEM=1000M namespace=kube-system create guaranteed\",\n    \"NAME=rbu0 CONTCOUNT=1 CPU=100m MEM=100M CPUREQ=99m MEMREQ=99M CPULIM=100m MEMLIM=100M namespace=kube-system create burstable\",\n    \"NAME=rbe0 CONTCOUNT=2 CPU=0 MEM=0 namespace=kube-system create besteffort\"\n{\n    guard {\n        v = inputvars(input_name)\n        return (v.NAME not in pods\n                and (mem + v.MEM * v.CONTCOUNT < max_mem)\n                and (reserved_cpu + v.CPU * v.CONTCOUNT < max_reserved_cpu))\n\n    }\n    body {\n        v = inputvars(input_name)\n        mem += v.MEM * v.CONTCOUNT\n        reserved_cpu += v.CPU * v.CONTCOUNT\n        pods[v.NAME] = v\n    }\n}\n\n# Kill a process in a container\n# - \"echo gu0c1\" matches and kills process only in container gu0c1 in pod gu0\n# - \"echo gu0\" matches and kills processes in all containers of pod gu0\ninput\n    \"NAME=gu0 container-exit0 gu0c0\",\n    \"NAME=gu1 container-exit0 gu1c0\",\n    \"NAME=gu2 container-exit0 gu2c0\",\n    \"NAME=gu3 container-exit0 gu3\",\n    \"NAME=gu4 container-exit0 gu4c\",\n    \"NAME=bu0 container-exit0 bu0c0\",\n    \"NAME=bu1 container-exit0 bu1c0\",\n    \"NAME=be0 container-exit0 be0c0\",\n    \"NAME=be1 container-exit0 be0c0\",\n    \"NAME=rgu0 container-exit0 rgu0c0\",\n    \"NAME=rbu0 container-exit0 rbu0c0\",\n    \"NAME=rbe0 container-exit0 rbe0c0\"\n{\n    guard {\n        v = inputvars(input_name)\n        return v.NAME in pods\n    }\n}\n\n# Delete single pod\ninput\n    \"NAME=gu0 kubectl delete pod gu0 --now --wait --ignore-not-found\",\n    \"NAME=gu1 kubectl delete pod gu1 --now --wait --ignore-not-found\",\n    \"NAME=gu2 kubectl delete pod gu2 --now --wait --ignore-not-found\",\n    \"NAME=gu3 kubectl delete pod gu3 --now --wait --ignore-not-found\",\n    \"NAME=gu4 kubectl delete pod gu4 --now --wait --ignore-not-found\",\n    \"NAME=bu0 kubectl delete pod bu0 --now --wait --ignore-not-found\",\n    \"NAME=bu1 kubectl delete pod bu1 --now --wait --ignore-not-found\",\n    \"NAME=be0 kubectl delete pod be0 --now --wait --ignore-not-found\",\n    \"NAME=be1 kubectl delete pod be1 --now --wait --ignore-not-found\",\n    \"NAME=rgu0 kubectl delete pod rgu0 -n kube-system --now --wait --ignore-not-found\",\n    \"NAME=rbu0 kubectl delete pod rbu0 -n kube-system --now --wait --ignore-not-found\",\n    \"NAME=rbe0 kubectl delete pod rbe0 -n kube-system --now --wait --ignore-not-found\"\n{\n    guard {\n        v = inputvars(input_name)\n        return v.NAME in pods\n    }\n    body {\n        v = inputvars(input_name)\n        p = pods[v.NAME]\n        mem -= p.MEM * p.CONTCOUNT\n        if getattr(p, \"namespace\", \"\") == \"kube-system\":\n            reserved_cpu -= p.CPU * p.CONTCOUNT\n        else:\n            cpu -= p.CPU * p.CONTCOUNT\n        del pods[v.NAME]\n    }\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/fuzz.fmbt.conf",
    "content": "model = aal_remote(remote_pyaal --verbose-fmbt-log fuzz.aal)\nheuristic = mrandom(80,lookahead(1:2),20,random)\ncoverage = perm(2)\n\npass = coverage(10)\npass = steps(100)\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh",
    "content": "#!/bin/bash\n\nusage() {\n    cat <<EOF\ngenerate.sh - generate fuzz tests.\n\nConfiguring test generation with environment variables:\n  TESTCOUNT=<NUM>       Number of generated test scripts than run in parallel.\n  MEM=<NUM>             Memory [MB] available for test pods in the system.\n  CPU=<NUM>             Non-reserved CPU [mCPU] available for test pods in the system.\n  RESERVED_CPU=<NUM>    Reserved CPU [mCPU] available for test pods in the system.\n  STEPS=<NUM>           Total number of test steps in all parallel tests.\n\n  FMBT_IMAGE=<IMG:TAG>  Generate the test using fmbt from docker image IMG:TAG.\n                        The default is fmbt-cli:latest.\nEOF\n    exit 0\n}\n\nif [ -n \"$1\" ]; then\n    usage\nfi\n\nTESTCOUNT=${TESTCOUNT:-1}\nMEM=${MEM:-7500}\n# 950 mCPU taken by the control plane, split the remaining 15050 mCPU\n# available for test pods to CPU and RESERVED_CPU pods.\nCPU=${CPU:-14050}\nRESERVED_CPU=${RESERVED_CPU:-1000}\nSTEPS=${STEPS:-100}\nFMBT_IMAGE=${FMBT_IMAGE:-\"fmbt-cli:latest\"}\n\nmem_per_test=$(( MEM / TESTCOUNT ))\ncpu_per_test=$(( CPU / TESTCOUNT ))\nreserved_cpu_per_test=$(( RESERVED_CPU / TESTCOUNT ))\nsteps_per_test=$(( STEPS / TESTCOUNT ))\n\n# Check fmbt Docker image\ndocker run \"$FMBT_IMAGE\" fmbt --version 2>&1 | grep ^Version: || {\n    echo \"error: cannot run fmbt from Docker image '$FMBT_IMAGE'\"\n    echo \"You can build the image locally by running:\"\n    echo \"( cd /tmp && git clone --branch devel https://github.com/intel/fmbt && cd fmbt && docker build . -t $FMBT_IMAGE -f Dockerfile.fmbt-cli )\"\n    exit 1\n}\n\ncd \"$(dirname \"$0\")\" || {\n    echo \"cannot cd to the directory of $0\"\n    exit 1\n}\n\nfor testnum in $(seq 1 \"$TESTCOUNT\"); do\n    testid=$(( testnum - 1))\n    sed -e \"s/max_mem=.*/max_mem=${mem_per_test}/\" \\\n        -e \"s/max_cpu=.*/max_cpu=${cpu_per_test}/\" \\\n        -e \"s/max_reserved_cpu=.*/max_reserved_cpu=${reserved_cpu_per_test}/\" \\\n        < fuzz.aal > tmp.fuzz.aal\n    sed -e \"s/fuzz\\.aal/tmp.fuzz.aal/\" \\\n        -e \"s/pass = steps(.*/pass = steps(${steps_per_test})/\" \\\n        < fuzz.fmbt.conf > tmp.fuzz.fmbt.conf\n    OUTFILE=generated${testid}.sh\n    echo \"generating $OUTFILE...\"\n    docker run -v \"$(pwd):/mnt/models\" \"$FMBT_IMAGE\" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\\$sn\\$as\\$al' | grep -v AAL | sed -e 's/^, /  /g' -e '/^STEP/! s/\\(^.*\\)/echo \"TESTGEN: \\1\"/g' -e 's/^STEP\\([0-9]*\\)i:\\(.*\\)/echo \"TESTGEN: STEP \\1\"; vm-command \"date +%T.%N\"; \\2; vm-command \"date +%T.%N\"; kubectl get pods -A/g' | sed \"s/\\([^a-z0-9]\\)\\(r\\?\\)\\(gu\\|bu\\|be\\)\\([0-9]\\)/\\1t${testid}\\2\\3\\4/g\" > \"$OUTFILE\"\ndone\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/code.var.sh",
    "content": "# Place pod0c0 and pod0c1 to shared pools on separate nodes.\nCONTCOUNT=2 CPU=500m create guaranteed\nreport allowed\nverify \"len(mems['pod0c0']) == 1\" \\\n       \"len(mems['pod0c1']) == 1\" \\\n       \"disjoint_sets(mems['pod0c0'], mems['pod0c1'])\" \\\n       \"len(cpus['pod0c0']) == 4\" \\\n       \"len(cpus['pod0c1']) == 4\" \\\n       \"disjoint_sets(cpus['pod0c0'], cpus['pod0c1'])\"\n\n# Place pod1c0 to its own node, as there is still one 4-CPU node free.\n# The placement of pod1c1 is more interesting:\n# - node0 has only 3 CPUs (CPU #0 is reserved)\n# - node1, node2 and node3 have containers in their shared pools\n# - shared pools with pod0c* containers have more free space than node0\n#   => pod1c0 should be place to either of those\n# - because pod1c1 should get one exclusive CPU, either of pod0c0 and\n#   pod0c1 should run in a shared pool of only 3 CPUs from now on.\nCONTCOUNT=2 CPU=1500m create guaranteed\nreport allowed\nverify `# every container is placed on a single node (no socket, no root)` \\\n       \"[len(mems[c]) for c in mems] == [1] * len(mems)\" \\\n       `# pod1c0 and pod1c1 are on different nodes` \\\n       \"disjoint_sets(mems['pod1c0'], mems['pod1c1'])\" \\\n       `# either of pod0c0 and pod0c1 has only 3 CPUs, the other has 4.` \\\n       \"len(cpus['pod0c0']) == 3 or len(cpus['pod0c1']) == 3\" \\\n       \"len(cpus['pod0c0']) == 4 or len(cpus['pod0c1']) == 4\" \\\n       `# pod1c0 and pod1c1 are allowed to use all CPUs on their nodes` \\\n       \"len(cpus['pod1c0']) == 4\" \\\n       \"len(cpus['pod1c1']) == 4\" \\\n       `# pod1c1 should have one exclusive CPU on its node` \\\n       \"len(cpus['pod1c1'] - cpus['pod0c0'] - cpus['pod0c1']) == 1\"\n\n# Place pod2c0 to node0, as it has largest free shared pool (3 CPUs).\n# Place pod2c1 to the node that has only either pod0c0 or pod0c1,\n# while the other one of them already shares a node with pod1c1.\nCONTCOUNT=2 CPU=2400m create guaranteed\nreport allowed\nverify `# every container is placed on a single node (no socket, no root)` \\\n       \"[len(mems[c]) for c in mems] == [1] * len(mems)\" \\\n       `# pod1c1 should have kept its own exclusive CPU` \\\n       \"len(cpus['pod1c1'] - set.union(*[cpus[c] for c in cpus if c != 'pod1c1'])) == 1\" \\\n       `# pod2c0 is the only container in node0, so it happens to have 3 unshared CPUs for now` \\\n       \"len(cpus['pod2c0']) == 3\" \\\n       \"len(cpus['pod2c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod2c0'])) == 3\" \\\n       `# pod2c1 shares its node and should not have exclusive CPUs` \\\n       \"len(cpus['pod2c1']) == 4\" \\\n       \"len(cpus['pod2c1'] - set.union(*[cpus[c] for c in cpus if c != 'pod2c1'])) == 0\" \\\n       `# pod2c1 should run in the same node as either pod0c0 or pod0c1` \\\n       \"mems['pod2c1'] == mems['pod0c0'] or mems['pod2c1'] == mems['pod0c1']\"\n\n# pod3c0 should get 2 exclusive CPUs and 400m share from a shared pool.\n# To get that, annotate the pod to:\n# - opt-out from shared CPUs (=> opt-in to exclusive CPUs)\n# - opt-in to isolated CPUs (this should not matter, test opt-out with pod4).\n# There is only one node where the container fits: the same node as pod1c0.\nANNOTATIONS=('prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"false\"'\n             'prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"')\nCONTCOUNT=1 CPU=2400m create guaranteed-annotated\nreport allowed\nverify `# every container is placed on a single node (no socket, no root)` \\\n       \"[len(mems[c]) for c in mems] == [1] * len(mems)\" \\\n       `# pod3c0 and pod1c0 are placed in the same node` \\\n       \"mems['pod3c0'] == mems['pod1c0']\" \\\n       `# pod1c0 has 1 an exclusive CPU` \\\n       \"len(cpus['pod1c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod1c0'])) == 1\" \\\n       `# pod3c0 has 2 exclusive CPUs` \\\n       \"len(cpus['pod3c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod3c0'])) == 2\"\n\n# Replace pod3 with pod4.\n# Test release/(re)allocate mixed pod with exclusive CPUs and\n# no-effect from isolated preference.\n# - opt-out from shared CPUs (=> opt-in to exclusive CPUs)\n# - opt-out from isolated CPUs (this does not affect getting exclusive CPUs)\nkubectl delete pods pod3 --now --wait --ignore-not-found\nANNOTATIONS=('prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"false\"'\n             'prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"false\"')\nCONTCOUNT=1 CPU=2400m create guaranteed-annotated\nreport allowed\nverify `# every container is placed on a single node (no socket, no root)` \\\n       \"[len(mems[c]) for c in mems] == [1] * len(mems)\" \\\n       `# pod4c0 and pod1c0 are placed in the same node` \\\n       \"mems['pod4c0'] == mems['pod1c0']\" \\\n       `# pod1c0 has 1 exclusive CPU` \\\n       \"len(cpus['pod1c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod1c0'])) == 1\" \\\n       `# pod4c0 has 2 exclusive CPUs` \\\n       \"len(cpus['pod4c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod4c0'])) == 2\"\n\n# Replace pod1 with pod5.\n# pod1 implicitly opted-in to exlusive CPUs due to 1500 mCPU request.\n# Now explicitly opt-out of it by opting-in to shared-cpus.\nkubectl delete pods pod1 --now --wait --ignore-not-found\n# Make sure that shared pool size increased correctly after mixed pod deletion.\nverify `# pod0c0 or pod0c1 shared a node with pod1c1 and had only 3 CPUs` \\\n       \"len(cpus['pod0c0']) == 4\" \\\n       \"len(cpus['pod0c1']) == 4\"\n\nANNOTATIONS=('prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"true\"')\nCONTCOUNT=2 CPU=1500m create guaranteed-annotated\nreport allowed\nverify `# every container is placed on a single node (no socket, no root)` \\\n       \"[len(mems[c]) for c in mems] == [1] * len(mems)\" \\\n       `# pod5c0 should share a node with pod0c0 or pod0c1 and have access to all CPUs` \\\n       \"mems['pod5c0'] == mems['pod0c0'] or mems['pod5c0'] == mems['pod0c1']\" \\\n       \"len(cpus['pod5c0']) == 4\" \\\n       \"len(cpus['pod0c0']) == 4\" \\\n       \"len(cpus['pod0c1']) == 4\" \\\n       `# pod5c1 should run in a node with pod4c0 (this is where pod1c0 used to be)` \\\n       \"mems['pod5c1'] == mems['pod4c0']\" \\\n       \"len(cpus['pod5c1']) == 2\" \\\n       `# pod5c0 and pod5c1 share a node with another container => all their CPUs should be shared` \\\n       \"len(cpus['pod5c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod5c0'])) == 0\" \\\n       \"len(cpus['pod5c1'] - set.union(*[cpus[c] for c in cpus if c != 'pod5c1'])) == 0\"\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/guaranteed-annotated.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"${ANNOTATIONS[0]}\" ]; then echo \"\n  annotations:\n    $(for annotation in \"${ANNOTATIONS[@]}\"; do echo \"\n    $annotation\n    \"; done)\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test08-isolcpus/code.var.sh",
    "content": "vm-command \"grep isolcpus=8,9 /proc/cmdline\" || {\n    vm-set-kernel-cmdline \"isolcpus=8,9\"\n    vm-reboot\n    vm-command \"grep isolcpus=8,9 /proc/cmdline\" || {\n        error \"failed to set isolcpus kernel commandline parameter\"\n    }\n    launch cri-resmgr\n    vm-command \"systemctl restart kubelet\"\n    sleep 1\n    vm-wait-process --timeout 120 kube-apiserver\n    vm-run-until --timeout 120 \"kubectl get node\"\n}\n\nCONTCOUNT=1\n\n# pod0: opt-in isolated CPUs\nANNOTATIONS='prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"'\nCPU=1 create guaranteed-annotated\nreport allowed\nverify \"cpus['pod0c0'] == {'cpu08'} or cpus['pod0c0'] == {'cpu09'}\" \\\n       \"mems['pod0c0'] == {'node2'}\"\n\n# pod1: opt-out isolated CPUs\nANNOTATIONS='prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"false\"'\nCPU=1 create guaranteed-annotated\nreport allowed\nverify \"disjoint_sets(cpus['pod1c0'], {'cpu08', 'cpu09'})\"\n\n# pod2: without annotation CPU=1 guaranteed pod is eligible to run on isolated CPUs\nANNOTATIONS=''\nCPU=1 create guaranteed-annotated\nreport allowed\nverify \"cpus['pod0c0'] == {'cpu08'} or cpus['pod0c0'] == {'cpu09'}\" \\\n       \"cpus['pod2c0'] == {'cpu08'} or cpus['pod2c0'] == {'cpu09'}\" \\\n       \"disjoint_sets(cpus['pod0c0'], cpus['pod2c0'])\" \\\n       \"mems['pod0c0'] == {'node2'}\" \\\n       \"mems['pod2c0'] == {'node2'}\"\n\n# free isolated (and all other) cpus\nkubectl delete pods --all --now --wait\n\n# pod3: opt-in isolated CPUs, take all of them\nANNOTATIONS='prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"'\nCPU=2000m create guaranteed-annotated\nreport allowed\nverify \"cpus['pod3c0'] == {'cpu08', 'cpu09'}\" \\\n       \"len(cpus['pod3c0']) == 2\"\n\n# free isolated cpus\nkubectl delete pods --all --now --wait\n\n# pod4: opt-in isolated CPUs but require a fraction more CPUs than there are isolated CPUs\nANNOTATIONS=('prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"'\n             'prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"false\"')\nCPU=2500m create guaranteed-annotated\nreport allowed\nverify \"'cpu08' in cpus['pod4c0'] and 'cpu09' in cpus['pod4c0']\" \\\n       \"len(cpus['pod4c0']) == 4\"\n\n# free isolated cpus\nkubectl delete pods --all --now --wait\n\n# pod5: opt-in isolated CPUs but require a fraction less CPUs than there are isolated CPUs\nANNOTATIONS=('prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"'\n             'prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"false\"')\nCPU=1500m create guaranteed-annotated\nreport allowed\nverify \"'cpu08' in cpus['pod5c0'] or 'cpu09' in cpus['pod5c0']\" \\\n       \"'cpu10' in cpus['pod5c0'] and 'cpu11' in cpus['pod5c0']\" \\\n       \"len(cpus['pod5c0']) == 3\"\n\n# free isolated cpus\nkubectl delete pods --all --now --wait\n\n# pod6: opt-in isolated CPUs but require a full CPU more than there\n# are isolated CPUs\nANNOTATIONS=('prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"'\n             'prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"false\"')\nCPU=3000m create guaranteed-annotated\nreport allowed\nverify \"len(cpus['pod6c0']) == 3\" \\\n       \"disjoint_sets(cpus['pod6c0'], {'cpu08', 'cpu09'})\" \\\n       \"len(mems['pod6c0']) == 1\"\n\n# pod7: sub-core is never eligble for isolated CPUs, even if annotated\n# to opt-in.\nANNOTATIONS=('prefer-isolated-cpus.cri-resource-manager.intel.com/pod: \"true\"'\n             'prefer-shared-cpus.cri-resource-manager.intel.com/pod: \"false\"')\nCONTCOUNT=4 CPU=200m create guaranteed-annotated\nreport allowed\nverify \"disjoint_sets(set.union(cpus['pod7c0'], cpus['pod7c1'], cpus['pod7c2'], cpus['pod7c3']), {'cpu08', 'cpu09'})\" \\\n       \"len(cpus['pod7c0']) >= 2\" \\\n       \"len(cpus['pod7c1']) >= 2\" \\\n       \"len(cpus['pod7c2']) >= 2\" \\\n       \"len(cpus['pod7c3']) >= 2\"\n\n# Cleanup kernel commandline, otherwise isolcpus will affect CPU\n# pinning and cause false negatives from other tests on this VM.\nvm-set-kernel-cmdline \"\"\nvm-reboot\nvm-command \"grep isolcpus /proc/cmdline\" && {\n    error \"failed to clean up isolcpus kernel commandline parameter\"\n}\necho \"isolcpus removed from kernel commandline\"\nlaunch cri-resmgr\nvm-command \"systemctl restart kubelet\"\nvm-wait-process --timeout 120 kube-apiserver\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test08-isolcpus/guaranteed-annotated.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"${ANNOTATIONS[0]}\" ]; then echo \"\n  annotations:\n    $(for annotation in \"${ANNOTATIONS[@]}\"; do echo \"\n    $annotation\n    \"; done)\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test09-container-exit/code.var.sh",
    "content": "# Test resource allocation / free on different container exit and\n# restart scenarios.\n\nCONTCOUNT=1 CPU=1000m MEM=64M create guaranteed\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       '\"pod0c0\" in allocations'\n\nout '### Crash and restart pod0c0'\nvm-command \"kubectl get pods pod0\"\n\nvm-command \"set -x; [[ -n \\\"\\$(pgrep -f pod0c0)\\\" ]] && [[ \\\"\\$(pgrep -f pod0c0 --oldest)\\\" != \\\"\\$(pgrep -f pod0c0 --newest)\\\" ]]\" || {\n    command-error \"There must be separate parent and child 'pod0c0' processes in order to run this test\"\n}\n\nout '### Kill the root process in pod0c0. The container should get Restarted.'\nvm-command \"kill -KILL \\$(pgrep -f pod0c0 --oldest)\"\nsleep 2\nvm-command 'kubectl wait --for=condition=Ready pods/pod0'\nvm-run-until --timeout 30 \"pgrep -f pod0c0 > /dev/null 2>&1\"\nvm-command \"kubectl get pods pod0\"\nreport allowed\nverify 'len(cpus[\"pod0c0\"]) == 1' \\\n       '\"pod0c0\" in allocations'\n\nout '### Kill the child process in pod0c0. The root process exits with status 0, the container should get Completed.'\nvm-command \"kubectl get pods pod0\"\nvm-command \"ps axf | grep pod0c0; echo newest: \\$(pgrep -f pod0c0 --newest)\"\nvm-command \"kill -KILL \\$(pgrep -f pod0c0 --newest)\"\nsleep 2\nvm-command \"kubectl get pods pod0\"\n# pod0c0 process is not on vm anymore\nverify '\"pod0c0\" not in cpus'\n# pod0c0 is not allocated any resources on CRI-RM\n( verify '\"pod0c0\" not in allocations' ) || {\n    # pretty-print allocations contents\n    pp allocations\n    error \"pod0c0 expected to disappear from allocations\"\n}\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test10-additional-reserved-namespaces/code.var.sh",
    "content": "# Test that\n# - containers marked in ReservedPoolNamespaces option pinned on Reserved CPUs.\n\n(kubectl create namespace reserved-test) || true\n\ncri_resmgr_cfg_orig=$cri_resmgr_cfg\n\n# This script will create pods to the reserved and default namespace.\n# Make sure the namespace is clear when starting the test and clean it up\n# if exiting with success. Otherwise leave the pod running for\n# debugging in case of a failure.\ncleanup-test-pods() {\n    ( kubectl delete pods pod0 -n kube-system --now --wait --ignore-not-found ) || true\n    ( kubectl delete pods pod1 --now --wait --ignore-not-found ) || true\n}\ncleanup-test-pods\n\nterminate cri-resmgr\nAVAILABLE_CPU=\"cpuset:8-11\"\nRESERVED_CPU=\"cpuset:10-11\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved-namespaces.cfg)\nlaunch cri-resmgr\n\nCONTCOUNT=1 namespace=kube-system create besteffort\nCONTCOUNT=1 create besteffort\nreport allowed\nverify 'cpus[\"pod0c0\"] == {\"cpu10\", \"cpu11\"}'\nverify 'cpus[\"pod1c0\"] == {\"cpu08\", \"cpu09\"}'\n\ncleanup-test-pods\n\n# Test that\n# - containers that are namespace-assigned to reserved pools are pinned there\n# - containers that are annotated to opt-put that are pinned elsewhere, and\n# - containers that are namespace-assigned and annotated to reserved pools are pinned there\n\n(kubectl create namespace foobar) || true\n\ncleanup-foobar-namespace() {\n    (kubectl delete pods -n foobar --all --now --wait) || true\n}\ncleanup-foobar-namespace\n\nCONTCOUNT=1 namespace=foobar create besteffort\nANN0='prefer-reserved-cpus.cri-resource-manager.intel.com/pod: \"false\"'\nCONTCOUNT=1 namespace=foobar create besteffort\nANN0='prefer-reserved-cpus.cri-resource-manager.intel.com/pod: \"true\"'\nCONTCOUNT=1 namespace=foobar create besteffort\n\nreport allowed\nverify 'cpus[\"pod2c0\"] == {\"cpu10\", \"cpu11\"}'\nverify 'cpus[\"pod3c0\"] == {\"cpu08\", \"cpu09\"}'\nverify 'cpus[\"pod4c0\"] == {\"cpu10\", \"cpu11\"}'\n\ncleanup-foobar-namespace\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test10-additional-reserved-namespaces/cri-resmgr-reserved-namespaces.cfg.in",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    cpu: ${RESERVED_CPU}\n  AvailableResources:\n    cpu: ${AVAILABLE_CPU}\n  topology-aware:\n    ReservedPoolNamespaces: [\\\"reserved-pool\\\",\\\"reserved-*\\\",\\\"foobar\\\"]\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/code.var.sh",
    "content": "# Test that\n# - containers marked in Annotations pinned on Reserved CPUs.\n\ncri_resmgr_cfg_orig=$cri_resmgr_cfg\n\ncleanup-test-pods() {\n    ( kubectl delete pods pod0 --now --wait --ignore-not-found ) || true\n    ( kubectl delete pods pod1 --now --wait --ignore-not-found ) || true\n}\ncleanup-test-pods\n\ncri_resmgr_cfg_orig=$cri_resmgr_cfg\nterminate cri-resmgr\n\nAVAILABLE_CPU=\"cpuset:8-11\"\nRESERVED_CPU=\"cpuset:10-11\"\ncri_resmgr_cfg=$(instantiate cri-resmgr-reserved-annotations.cfg)\nlaunch cri-resmgr\n\nANNOTATIONS='prefer-reserved-cpus.cri-resource-manager.intel.com/pod: \"true\"'\nCONTCOUNT=1 create reserved-annotated\nreport allowed\n\nANNOTATIONS='prefer-reserved-cpus.cri-resource-manager.intel.com/container.special: \"false\"'\nCONTCOUNT=1 create reserved-annotated\nreport allowed\n\nverify 'cpus[\"pod0c0\"] == {\"cpu10\", \"cpu11\"}'\nverify 'cpus[\"pod1c0\"] == {\"cpu08\"}'\n\ncleanup-test-pods\n\nterminate cri-resmgr\ncri_resmgr_cfg=$cri_resmgr_cfg_orig\nlaunch cri-resmgr\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/cri-resmgr-reserved-annotations.cfg.in",
    "content": "policy:\n  Active: topology-aware\n  ReservedResources:\n    cpu: ${RESERVED_CPU}\n  AvailableResources:\n    cpu: ${AVAILABLE_CPU}\nlogger:\n  Debug: cri-resmgr,resource-manager,cache,policy\n  Klog:\n    skip_headers: true\ndump:\n  Config: off:.*,full:((Create)|(Start)|(Run)|(Update)|(Stop)|(Remove)).*,off:.*Image.*\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/test11-reserved-cpu-annotations/reserved-annotated.yaml.in",
    "content": "apiVersion: v1\nkind: Pod\nmetadata:\n  name: ${NAME}\n  $(if [ -n \"${ANNOTATIONS[0]}\" ]; then echo \"\n  annotations:\n    $(for annotation in \"${ANNOTATIONS[@]}\"; do echo \"\n    $annotation\n    \"; done)\n  \"; fi)\n  labels:\n    app: ${NAME}\nspec:\n  containers:\n  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo \"\n  - name: ${NAME}c$(( contnum - 1 ))\n    image: busybox\n    imagePullPolicy: IfNotPresent\n    command:\n      - sh\n      - -c\n      - echo ${NAME}c$(( contnum - 1 )) \\$(sleep inf)\n    resources:\n      requests:\n        cpu: ${CPU}\n        memory: '${MEM}'\n      limits:\n        cpu: ${CPU}\n        memory: '${MEM}'\n  \"; done )\n  terminationGracePeriodSeconds: 1\n"
  },
  {
    "path": "test/e2e/policies.test-suite/topology-aware/n4c16/topology.var.json",
    "content": "[\n    {\"mem\": \"2G\", \"cores\": 2, \"nodes\": 2, \"packages\": 2}\n]\n"
  },
  {
    "path": "test/e2e/run.sh",
    "content": "#!/bin/bash\n\nDEMO_TITLE=\"Container Runtime End-to-End Testing\"\nDEFAULT_DISTRO=\"ubuntu-22.04\"\n\nPV='pv -qL'\n\nbinsrc=${binsrc-local}\n\nSCRIPT_DIR=\"$(dirname \"${BASH_SOURCE[0]}\")\"\nDEMO_LIB_DIR=$(realpath \"$SCRIPT_DIR/../../demo/lib\")\nOUTPUT_DIR=${outdir-\"$SCRIPT_DIR\"/output}\nCOMMAND_OUTPUT_DIR=\"$OUTPUT_DIR\"/commands\n\n# shellcheck disable=SC1091\n# shellcheck source=../../demo/lib/command.bash\nsource \"$DEMO_LIB_DIR\"/command.bash\n# shellcheck disable=SC1091\n# shellcheck source=../../demo/lib/host.bash\nsource \"$DEMO_LIB_DIR\"/host.bash\n# shellcheck disable=SC1091\n# shellcheck source=../../demo/lib/vm.bash\nsource \"$DEMO_LIB_DIR\"/vm.bash\n\nscript_source=\"$(< \"$0\") $(< \"$DEMO_LIB_DIR/host.bash\") $(< \"$DEMO_LIB_DIR/command.bash\") $(< \"$DEMO_LIB_DIR/vm.bash\")\"\n\nusage() {\n    echo \"$DEMO_TITLE\"\n    echo \"Usage: [VAR=VALUE] ./run.sh MODE [SCRIPT]\"\n    echo \"  MODE:     \\\"play\\\" plays the test as a demo.\"\n    echo \"            \\\"record\\\" plays and records the demo.\"\n    echo \"            \\\"test\\\" runs fast, reports pass or fail.\"\n    echo \"            \\\"debug\\\" enables k8scri pipe debugging and\"\n    echo \"                    copies sources of all *_src VARs (see below) to vm.\"\n    echo \"            \\\"interactive\\\" launches interactive shell\"\n    echo \"            for running test script commands\"\n    echo \"            (see ./run.sh help script [FUNCTION]).\"\n    echo \"  SCRIPT:   test script file to run instead of the default test.\"\n    echo \"\"\n    echo \"  VARs:\"\n    echo \"    vm:      govm virtual machine name.\"\n    echo \"             For non-govm-managed hosts: set VM_IP and VM_SSH_USER, too.\"\n    echo \"             'ssh \\$VM_SSH_USER@\\$VM_IP sudo id' must not require password.\"\n    echo \"    containerd_src:\"\n    echo \"             \\\"/host/path/to/go/project\\\": replace vm /usr/bin binaries\"\n    echo \"             from /host/path/to/go/project/bin directory.\"\n    echo \"             The default is to use vm OS package manager containerd.\"\n    echo \"    crio_src:\"\n    echo \"             \\\"/host/path/to/go/project\\\": replace vm /usr/bin binaries\"\n    echo \"             from /host/path/to/go/project/bin directory.\"\n    echo \"             Must be set if crio is a part of \\$k8scri and the vm distro\"\n    echo \"             does not have (or implement installing) cri-o packages.\"\n    echo \"    crirm_src:\"\n    echo \"             \\\"/host/path/to/go/project\\\": replace vm /usr/local/bin binaries\"\n    echo \"             from /host/path/to/go/project/bin directory.\"\n    echo \"             The default is to use the project of these e2e tests.\"\n    echo \"    runc_src:\"\n    echo \"             \\\"/host/path/to/go/project\\\": replace vm /usr/bin binaries\"\n    echo \"             from /host/path/to/go/project/bin directory.\"\n    echo \"    distro_binaries:\"\n    echo \"             0: use the normal binaries built for this host (the default).\"\n    echo \"             1: use binaries cross-built for distros.\"\n    echo \"    binsrc:  Where to get cri-resmgr to the vm.\"\n    echo \"             \\\"github\\\": go get from master and build inside vm.\"\n    echo \"             \\\"local\\\": (the default) copy from \\${crirm_src}/bin, or\"\n    echo \"                      from \\${crirm_src}/binaries/\\$distro if \\$distro_binaries=1.\"\n    echo \"             \\\"packages/<distro>\\\": use distro packages from this dir\"\n    echo \"    reinstall_<containerd|crio|cri_resmgr|cri_resmgr_agent|runc>:\"\n    echo \"             If 1, stop the daemon (if not runc),\"\n    echo \"             then reinstall and restart it before starting test run.\"\n    echo \"             The default is 0.\"\n    echo \"             Set containerd_src/crio_src/runc_src to install a local build.\"\n    echo \"    reinstall_k8s: if 1, destroy existing k8s cluster and create a new one.\"\n    echo \"    reinstall_bootstrap: if 1, run the bootstrap and proxy setup commands.\"\n    echo \"                         Only available if VM_IP is set when calling the script.\"\n    echo \"    reinstall_all: if 1, set all above reinstall_* options to 1.\"\n    echo \"    omit_cri_resmgr: if 1, omit checking/installing/starting cri-resmgr.\"\n    echo \"    omit_agent: if 1, omit checking/installing/starting cri-resmgr-agent.\"\n    echo \"    outdir:  Save output under given directory.\"\n    echo \"             The default is \\\"${SCRIPT_DIR}/output\\\".\"\n    echo \"    speed:   Demo play speed.\"\n    echo \"             The default is 10 (keypresses per second).\"\n    echo \"    cleanup: Level of cleanup after a test run:\"\n    echo \"             0: leave vm running (the default)\"\n    echo \"             1: delete vm\"\n    echo \"             2: stop vm, but do not delete it.\"\n    echo \"  Hook VARs:\"\n    echo \"    on_vm_online: code to be executed when SSH connection to vm works.\"\n    echo \"    on_k8s_online: code to be executed when Kubernetes is ready for use.\"\n    echo \"    on_verify_fail, on_create_fail: code to be executed in case\"\n    echo \"             verify() or create() fails. Example: go to interactive\"\n    echo \"             mode if a verification fails: on_verify_fail=interactive\"\n    echo \"    on_verify, on_create, on_launch: code to be executed every time\"\n    echo \"             after verify/create/launch function\"\n    echo \"    on_{cri,runc,k8s}_install: code to be executed right after installing\"\n    echo \"             these components.\"\n    echo \"\"\n    echo \"  VM configuration VARs: (effective when vm is not already configured)\"\n    echo \"    topology: JSON to override NUMA node list used in tests.\"\n    echo \"             See: python3 ${DEMO_LIB_DIR}/topology2qemuopts.py --help\"\n    echo \"    distro:  Linux distribution to be / already installed on vm.\"\n    echo \"             Supported values: debian-11, debian-12, debian-sid\"\n    echo \"                 fedora, opensuse-tumbleweed,\"\n    echo \"                 opensuse-15.6 (same as opensuse), sles,\"\n    echo \"                 ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, ubuntu-24.04\"\n    echo \"             If sles: set VM_SLES_REGCODE=<CODE> to use official packages.\"\n    echo \"    cgroups: cgroups version in the VM, v1 or v2. The default is v1.\"\n    echo \"             cgroups=v2 is supported only on distro=fedora\"\n    echo \"    k8s:     Kubernetes version to be installed on VM creation\"\n    echo \"             The default is the latest available on selected distro.\"\n    echo \"             Example: k8s=1.31\"\n    echo \"    k8scri:  The container runtime pipe where kubelet connects to.\"\n    echo \"             Options are:\"\n    echo \"             \\\"cri-resmgr|containerd\\\" cri-resmgr is a proxy to containerd.\"\n    echo \"             \\\"cri-resmgr|crio\\\"       cri-resmgr is a proxy to cri-o.\"\n    echo \"             \\\"containerd\\\"            containerd, no cri-resmgr.\"\n    echo \"             \\\"containerd&cri-resmgr\\\" containerd, cri-resmgr is an NRI plugin.\"\n    echo \"             \\\"crio\\\"                  cri-o, no cri-resmgr.\"\n    echo \"             \\\"crio&cri-resmgr\\\"       cri-o, cri-resmgr is an NRI plugin.\"\n    echo \"             The default is \\\"cri-resmgr|containerd\\\".\"\n    echo \"    k8scni:  The container network interface plugin to install. Options are:\"\n    echo \"             \\\"cilium\\\" (the default), \\\"flannel\\\", \\\"weavenet\\\".\"\n    echo \"    k8smaster: Name of the existing vm whose cluster this vm will join.\"\n    echo \"             If empty (default), this vm forms its own single-node cluster.\"\n    echo \"    crio_version: Version of cri-o to try to pull in, if cri-o is\"\n    echo \"                  not being installed from sources.\"\n    echo \"    setup_proxies: Setup proxies even if not using govm based VM.\"\n    echo \"                   This is only needed if you have set VM_IP and want\"\n    echo \"                   the proxy information set in the target host. By default\"\n    echo \"                   the proxies are not set if VM_IP is set.\"\n    echo \"\"\n    echo \"  Test input VARs:\"\n    echo \"    cri_resmgr_cfg: configuration file forced to cri-resmgr.\"\n    echo \"    cri_resmgr_extra_args: arguments to be added on cri-resmgr\"\n    echo \"             command line when launched\"\n    echo \"    cri_resmgr_agent_extra_args: arguments to be added on\"\n    echo \"              cri-resmgr-agent command line when launched\"\n    echo \"    use_host_images: if \\\"1\\\", export images from the host docker\"\n    echo \"              to vm whenever they are available.\"\n    echo \"              The default is 0: always pull images from repositories to vm.\"\n    echo \"    vm_files: \\\"serialized\\\" associative array of files to be created on vm\"\n    echo \"             associative array syntax:\"\n    echo \"             vm_files['/path/file']=file:/path/on/host\"\n    echo \"                                   ='data:,plain text content'\"\n    echo \"                                   =data:;base64,ZGF0YQ==\"\n    echo \"                                   =dir: (creates only /path/file directory)\"\n    echo \"             vm_files['/etc/motd']='data:,hello world'\"\n    echo \"             How to execute run.sh with serialized array:\"\n    echo \"             vm_files=\\$(declare -p vm_files) ./run.sh\"\n    echo \"    code:    Variable that contains test script code to be run\"\n    echo \"             if SCRIPT is not given.\"\n    echo \"    py_consts: Python code that runs always before pyexec in SCRIPT.\"\n    echo \"\"\n    echo \"Default test input VARs: ./run.sh help defaults\"\n    echo \"\"\n    echo \"Create VM 'foo' that runs k8s 1.28 on Debian Sid:\"\n    echo \"vm=foo distro=debian-sid k8s=1.28 ./run.sh interactive\"\n}\n\nerror() {\n    (echo \"\"; echo \"error: $1\" ) >&2\n    command-exit-if-not-interactive\n}\n\nout() {\n    if [ -n \"$PV\" ]; then\n        speed=${speed-10}\n        echo \"$1\" | $PV \"$speed\"\n    else\n        echo \"$1\"\n    fi\n    echo \"\"\n}\n\nrecord() {\n    clear\n    out \"Recording this screencast...\"\n    host-command \"asciinema rec -t \\\"$DEMO_TITLE\\\" crirm-demo-blockio.cast -c \\\"./run.sh play\\\"\"\n}\n\nscreen-create-vm() {\n    speed=60 out \"### Running the test in vm=\\\"$VM_NAME\\\".\"\n    host-create-vm \"$vm\" \"$topology\"\n    vm-networking\n    if [ -z \"$VM_IP\" ]; then\n        error \"creating VM failed\"\n    fi\n}\n\nscreen-install-cri-resmgr() {\n    speed=60 out \"### Installing CRI Resource Manager to VM.\"\n    vm-install-cri-resmgr\n}\n\nscreen-launch-cri-resmgr() {\n    speed=60 out \"### Launching cri-resmgr with config $cri_resmgr_cfg.\"\n    if [ \"${binsrc#packages}\" != \"$binsrc\" ]; then\n        launch cri-resmgr-systemd\n    else\n        launch cri-resmgr\n    fi\n}\n\nscreen-create-singlenode-cluster() {\n    speed=60 out \"### Setting up single-node Kubernetes cluster.\"\n    speed=60 out \"### Container runtime parts: $k8scri\"\n    vm-create-singlenode-cluster\n}\n\nscreen-launch-cri-resmgr-agent() {\n    speed=60 out \"### Launching cri-resmgr-agent.\"\n    speed=60 out \"### The agent will make cri-resmgr configurable with ConfigMaps.\"\n    launch cri-resmgr-agent\n}\n\nget-py-allowed() {\n    topology_dump_file=\"$OUTPUT_DIR/topology_dump.$VM_NAME\"\n    res_allowed_file=\"$OUTPUT_DIR/res_allowed.$VM_NAME\"\n    if ! [ -f \"$topology_dump_file\" ]; then\n        vm-command \"$(\"$DEMO_LIB_DIR/topology.py\" bash_topology_dump)\" >/dev/null || {\n            command-error \"error fetching topology_dump from $VM_NAME\"\n        }\n        echo -e \"$COMMAND_OUTPUT\" > \"$topology_dump_file\"\n    fi\n    get-res-allowed \"$res_allowed_file\"\n    py_allowed=\"\nimport re\nallowed=$(\"$DEMO_LIB_DIR/topology.py\" -t \"$topology_dump_file\" -r \"$res_allowed_file\" res_allowed -o json)\n_branch_pod=[(p, d, n, c, t, cpu, pod.rsplit('/', 1)[0])\n             for p in allowed\n             for d in allowed[p]\n             for n in allowed[p][d]\n             for c in allowed[p][d][n]\n             for t in allowed[p][d][n][c]\n             for cpu in allowed[p][d][n][c][t]\n             for pod in allowed[p][d][n][c][t][cpu]]\n# cpu resources allowed for a pod:\npackages, dies, nodes, cores, threads, cpus = {}, {}, {}, {}, {}, {}\n# mem resources allowed for a pod:\nmems = {}\nfor p, d, n, c, t, cpu, pod in _branch_pod:\n    if c == 'mem': # this _branch_pod entry is about memory\n        if not pod in mems:\n            mems[pod] = set()\n        # topology.py can print memory nodes as children of cpu-ful nodes\n        # if distance looks like they are behind the same memory controller.\n        # The thread field, however, is the true node who contains the memory.\n        mems[pod].add(t)\n        continue\n    # this _branch_pod entry is about cpu\n    if not pod in packages:\n        packages[pod] = set()\n        dies[pod] = set()\n        nodes[pod] = set()\n        cores[pod] = set()\n        threads[pod] = set()\n        cpus[pod] = set()\n    packages[pod].add(p)\n    dies[pod].add('%s/%s' % (p, d))\n    nodes[pod].add(n)\n    cores[pod].add('%s/%s' % (n, c))\n    threads[pod].add('%s/%s/%s' % (n, c, t))\n    cpus[pod].add(cpu)\n\ndef disjoint_sets(*sets):\n    'set.isdisjoint() for n > 1 sets'\n    s = sets[0]\n    for next in sets[1:]:\n        if not s.isdisjoint(next):\n            return False\n        s = s.union(next)\n    return True\n\ndef set_ids(str_ids, chars='[a-z]'):\n    num_ids = set()\n    for str_id in str_ids:\n        if '/' in str_id:\n            num_ids.add(tuple(int(re.sub(chars, '', s)) for s in str_id.split('/')))\n        else:\n            num_ids.add(int(re.sub(chars, '', str_id)))\n    return num_ids\npackage_ids = lambda i: set_ids(i, '[package]')\ndie_ids = lambda i: set_ids(i, '[packagedie]')\nnode_ids = lambda i: set_ids(i, '[node]')\ncore_ids = lambda i: set_ids(i, '[nodecore]')\nthread_ids = lambda i: set_ids(i, '[nodecorethread]')\ncpu_ids = lambda i: set_ids(i, '[cpu]')\n\"\n}\n\nget-res-allowed() {\n    local res_allowed_file=\"$1\"\n    local retries=5\n    while (( retries > 0 )); do\n        # Fetch data and update allowed* variables from the virtual machine\n        vm-command \"$(\"$DEMO_LIB_DIR/topology.py\" bash_res_allowed 'pod[0-9]*c[0-9]*')\" >/dev/null || {\n            command-error \"error fetching res_allowed from $VM_NAME\"\n        }\n        echo -e \"$COMMAND_OUTPUT\" > \"$res_allowed_file\"\n        # Validate res_allowed_file. Retry if there is same container\n        # name with two different sets of allowed CPUs or\n        # memories. This is possible if cpuset.cpus of the cgroup has\n        # been just changed and different processes in the same\n        # container are just going through the change. Or if there are\n        # several pods/containers running with the same name.\n        awk -F \"[ /]\" '{if (pod[$1]!=0 && pod[$1]!=\"\"$3\"\"$4){print \"error: ambiguous allowed resources for name \"$1; exit(1)};pod[$1]=\"\"$3\"\"$4}' \"$res_allowed_file\" && return 0\n        mv \"$res_allowed_file\" \"$res_allowed_file.retries${retries}\"\n        echo \"    see $res_allowed_file.retries${retries} for more details\"\n        retries=$(( retries - 1 ))\n    done\n    error \"error: container/process name collision: test environment may need cleanup.\"\n}\n\nget-py-cache() {\n    # Fetch current cri-resmgr cache from a virtual machine.\n    speed=1000 vm-command \"cat \\\"/var/lib/cri-resmgr/cache\\\"\" >/dev/null 2>&1 || {\n        command-error \"fetching cache file failed\"\n    }\n    cat > \"${OUTPUT_DIR}/cache\" <<<\"$COMMAND_OUTPUT\"\n    py_cache=\"\nimport json\ncache=json.load(open(\\\"${OUTPUT_DIR}/cache\\\"))\ntry:\n    allocations=json.loads(cache['PolicyJSON']['allocations'])\nexcept KeyError:\n    allocations=None\ncontainers=cache['Containers']\npods=cache['Pods']\nfor _contid in list(containers.keys()):\n    try:\n        _cmd = ' '.join(containers[_contid]['Command'])\n    except:\n        continue # Command may be None\n    # Recognize echo podXcY ; sleep inf -type test pods and make them\n    # easily accessible: containers['pod0c0'], pods['pod0']\n    if 'echo pod' in _cmd and 'sleep inf' in _cmd:\n        _contname = _cmd.split()[3] # _contname is podXcY\n        _podid = containers[_contid]['PodID']\n        _podname = pods[_podid]['Name'] # _podname is podX\n        if not allocations is None and _contid in allocations:\n            allocations[_contname] = allocations[_contid]\n        containers[_contname] = containers[_contid]\n        pods[_podname] = pods[_podid]\n\"\n}\n\nresolve-template() {\n    local name=\"$1\" r=\"\" d t\n    shift\n    for d in \"$@\"; do\n        if [ -z \"$d\" ] || ! [ -d \"$d\" ]; then\n            continue\n        fi\n        t=\"$d/$name.in\"\n        if ! [ -e \"$t\" ]; then\n            continue\n        fi\n        if [ -z \"$r\" ]; then\n            r=\"$t\"\n            echo 1>&2 \"template $name resolved to file $r\"\n        else\n            echo 1>&2 \"WARNING: template file $r shadows $t\"\n        fi\n    done\n    if [ -n \"$r\" ]; then\n        echo \"$r\"\n        return 0\n    fi\n    return 1\n}\n\nis-hooked() {\n    local hook_code_var hook_code\n    hook_code_var=$1\n    hook_code=\"${!hook_code_var}\"\n    if [ -n \"${hook_code}\" ]; then\n        return 0 # logic: if is-hooked xyz; then run-hook xyz; fi\n    fi\n    return 1\n}\n\nrun-hook() {\n    local hook_code_var hook_code\n    hook_code_var=$1\n    hook_code=\"${!hook_code_var}\"\n    echo \"Running hook: $hook_code_var\"\n    eval \"${hook_code}\"\n}\n\ninstall-files() {\n    # Usage: install-files $(declare -p files_assoc_array)\n    #\n    # Parameter is a serialized associative array with\n    # key: target filepath on VM\n    # value: source URL (\"file:\", limited \"data:\" and \"dir:\" schemes supported)\n    #\n    # Example: build an associative array and install files in the array\n    #   files['/path/file1']=file:/hostpath/file\n    #   files['/path/file2']=data:,hello\n    #   files['/path/file3']=data:;base64,aGVsbG8=\n    #   files['/path/dir1']='dir:'\n    #   install-files \"$(declare -p files)\"\n    local -A files\n    eval \"files=${1#*=}\"\n    local tgt src data\n    for tgt in \"${!files[@]}\"; do\n        src=\"${files[$tgt]}\"\n        case $src in\n            \"data:,\"*)\n                data=${src#data:,}\n                ;;\n            \"data:;base64,\"*)\n                data=$(base64 -d <<< \"${src#data:;base64,}\")\n                ;;\n            \"file:\"*)\n                data=$(< \"${src#file:}\")\n                ;;\n            \"dir:\")\n                echo -n \"Creating on vm: $tgt/... \"\n                vm-command-q \"mkdir -p \\\"$tgt\\\"\" || {\n                    error \"failed to make directory to vm \\\"$tgt\\\"\"\n                }\n                echo \"ok.\"\n                continue\n                ;;\n            *)\n                error \"invalid source scheme \\\"${src}\\\", expected \\\"data:,\\\" \\\"data:;base64,\\\", \\\"file:\\\" or \\\"dir:\\\"\"\n                ;;\n        esac\n        echo -n \"Writing on vm: $tgt... \"\n        vm-write-file \"$tgt\" \"$data\" || {\n            error \"failed to write to vm file \\\"$tgt\\\"\"\n        }\n        echo \"ok.\"\n    done\n}\n\n### Test script helpers\n\ninstall() { # script API\n    # Usage: install TARGET\n    #\n    # Supported TARGETs:\n    #   cri-resmgr: install cri-resmgr to VM.\n    #               Install latest local build to VM: (the default)\n    #                 $ install cri-resmgr\n    #               Fetch github master to VM, build and install on VM:\n    #                 $ binsrc=github install cri-resmgr\n    #   cri-resmgr-webhook: install cri-resmgr-webhook to VM.\n    #               Installs from the latest webhook Docker image on the host.\n    #\n    # Example:\n    #   uninstall cri-resmgr\n    #   install cri-resmgr\n    #   launch cri-resmgr\n    local target=\"$1\"\n    case \"$target\" in\n        \"cri-resmgr\")\n            vm-install-cri-resmgr\n            ;;\n        \"cri-resmgr-agent\")\n            vm-install-cri-resmgr-agent\n            ;;\n        \"cri-resmgr-webhook\")\n            vm-install-cri-resmgr-webhook\n            ;;\n        *)\n            error \"unknown target to install \\\"$1\\\"\"\n            ;;\n    esac\n}\n\nuninstall() { # script API\n    # Usage: uninstall TARGET\n    #\n    # Supported TARGETs:\n    #   cri-resmgr: stop (kill) cri-resmgr and purge all files from VM.\n    #   cri-resmgr-webhook: stop cri-resmgr-webhook and delete webhook files from VM.\n    local target=\"$1\"\n    case $target in\n        \"cri-resmgr\")\n            terminate cri-resmgr\n            terminate cri-resmgr-agent\n            distro-remove-pkg cri-resource-manager\n            vm-command \"rm -rf /usr/local/bin/cri-resmgr /usr/bin/cri-resmgr /usr/local/bin/cri-resmgr-agent /usr/bin/cri-resmgr-agent /var/lib/cri-resmgr /etc/cri-resmgr\"\n            ;;\n        \"cri-resmgr-agent\")\n            terminate cri-resmgr-agent\n            vm-command \"rm -rf /usr/local/bin/cri-resmgr /usr/bin/cri-resmgr /usr/local/bin/cri-resmgr-agent /usr/bin/cri-resmgr-agent /var/lib/cri-resmgr /etc/cri-resmgr\"\n            ;;\n        \"cri-resmgr-webhook\")\n            terminate cri-resmgr-webhook\n            vm-command \"rm -rf webhook\"\n            ;;\n        *)\n            error \"uninstall: invalid target \\\"$target\\\"\"\n            ;;\n    esac\n}\n\nlaunch() { # script API\n    # Usage: launch TARGET\n    #\n    # Supported TARGETs:\n    #   cri-resmgr:  launch cri-resmgr on VM. Environment variables:\n    #                cri_resmgr_cfg: configuration filepath (on host)\n    #                cri_resmgr_extra_args: extra arguments on command line\n    #                cri_resmgr_config: \"force\" (default) or \"fallback\"\n    #                k8scri: if the CRI pipe starts with cri-resmgr\n    #                        this launches cri-resmgr as a proxy,\n    #                        otherwise as a dynamic NRI plugin.\n    #\n    #   cri-resmgr-systemd:\n    #                launch cri-resmgr on VM using \"systemctl start\".\n    #                Works when installed with binsrc=packages/<distro>.\n    #                Environment variables:\n    #                cri_resmgr_cfg: configuration filepath (on host)\n    #\n    #   cri-resmgr-agent:\n    #                launch cri-resmgr-agent on VM. Environment variables:\n    #                cri_resmgr_agent_extra_args: extra arguments on command line\n    #\n    #   cri-resmgr-webhook:\n    #                deploy cri-resmgr-webhook from the image on VM.\n    #\n    # Example:\n    #   cri_resmgr_cfg=/tmp/topology-aware.cfg launch cri-resmgr\n    local target=\"$1\"\n    local launch_cmd\n    local adjustment_schema=\"$HOST_PROJECT_DIR/pkg/apis/resmgr/v1alpha1/adjustment-schema.yaml\"\n    local cri_resmgr_config_option=\"-${cri_resmgr_config:-force}-config\"\n    local cri_resmgr_mode=\"\"\n    case $target in\n        \"cri-resmgr\")\n            host-command \"$SCP \\\"$cri_resmgr_cfg\\\" $VM_SSH_USER@$VM_IP:\" || {\n                command-error \"copying \\\"$cri_resmgr_cfg\\\" to VM failed\"\n            }\n            vm-command \"cat $(basename \"$cri_resmgr_cfg\")\"\n            if [[ \"$k8scri\" == cri-resmgr* ]]; then\n                # launch cri-resmgr as the top element in the k8s container runtime stack\n                cri_resmgr_mode=\"-relay-socket ${cri_resmgr_sock} -runtime-socket $cri_sock -image-socket $cri_sock\"\n            else\n                # launch cri-resmgr as an NRI plugin to running container runtime\n                cri_resmgr_mode=\"-use-nri-plugin\"\n            fi\n            launch_cmd=\"cri-resmgr $cri_resmgr_mode $cri_resmgr_config_option $(basename \"$cri_resmgr_cfg\") $cri_resmgr_extra_args\"\n            vm-command-q \"rm -f $cri_resmgr_pidfile\"\n            vm-command-q \"echo '$launch_cmd' > cri-resmgr.launch.sh ; rm -f cri-resmgr.output.txt\"\n            vm-command \"$launch_cmd  >cri-resmgr.output.txt 2>&1 &\"\n            vm-wait-process --timeout 30 --pidfile \"$cri_resmgr_pidfile\" cri-resmgr\n            vm-command \"grep 'FATAL ERROR' cri-resmgr.output.txt\" >/dev/null 2>&1 && {\n                command-error \"launching cri-resmgr failed with FATAL ERROR\"\n            }\n            vm-command \"fuser ${cri_resmgr_pidfile}\" >/dev/null 2>&1 || {\n                echo \"cri-resmgr last output line:\"\n                vm-command-q \"tail -n 1 cri-resmgr.output.txt\"\n                command-error \"launching cri-resmgr failed, cannot find cri-resmgr PID\"\n            }\n            ;;\n\n        \"cri-resmgr-agent\")\n            host-command \"$SCP \\\"$adjustment_schema\\\" $VM_SSH_USER@$VM_IP:\" ||\n                command-error \"copying \\\"$adjustment_schema\\\" to VM failed\"\n            vm-command \"kubectl delete -f $(basename \"$adjustment_schema\"); kubectl create -f $(basename \"$adjustment_schema\")\"\n            launch_cmd=\"NODE_NAME=\\$(hostname) cri-resmgr-agent -kubeconfig /root/.kube/config $cri_resmgr_agent_extra_args\"\n            vm-command-q \"echo '$launch_cmd' >cri-resmgr-agent.launch.sh; rm -f cri-resmgr-agent.output.txt\"\n            vm-command \"$launch_cmd >cri-resmgr-agent.output.txt 2>&1 &\"\n            vm-wait-process --timeout 30 cri-resmgr-agent\n            vm-command \"grep 'FATAL ERROR' cri-resmgr-agent.output.txt\" >/dev/null 2>&1 &&\n                command-error \"launching cri-resmgr-agent failed with FATAL ERROR\"\n            vm-command \"fuser ${cri_resmgr_agent_sock}\" >/dev/null 2>&1 ||\n                command-error \"launching cri-resmgr-agent failed, cannot find cri-resmgr-agent PID\"\n            ;;\n\n        \"cri-resmgr-systemd\")\n            host-command \"$SCP \\\"$cri_resmgr_cfg\\\" $VM_SSH_USER@$VM_IP:\" ||\n                command-error \"copying \\\"$cri_resmgr_cfg\\\" to VM failed\"\n            vm-command \"cp \\\"$(basename \"$cri_resmgr_cfg\")\\\" /etc/cri-resmgr/fallback.cfg\"\n            vm-command \"systemctl daemon-reload ; systemctl start cri-resource-manager\" ||\n                command-error \"systemd failed to start cri-resource-manager\"\n            vm-wait-process --timeout 30 cri-resmgr\n            vm-command \"systemctl is-active cri-resource-manager\" || {\n                vm-command \"systemctl status cri-resource-manager\"\n                command-error \"cri-resource-manager did not become active after systemctl start\"\n            }\n            ;;\n\n        \"cri-resmgr-webhook\")\n            kubectl apply -f webhook/webhook-deployment.yaml\n            kubectl wait --for=condition=Available -n cri-resmgr deployments/cri-resmgr-webhook ||\n                error \"cri-resmgr-webhook deployment did not become Available\"\n            kubectl apply -f webhook/mutating-webhook-config.yaml\n            ;;\n\n        *)\n            error \"launch: invalid target \\\"$1\\\"\"\n            ;;\n    esac\n    is-hooked on_launch && run-hook on_launch\n    return 0\n}\n\nterminate() { # script API\n    # Usage: terminate TARGET\n    #\n    # Supported TARGETs:\n    #   cri-resmgr: stop (kill) cri-resmgr.\n    #   cri-resmgr-agent: stop (kill) cri-resmgr-agent.\n    #   cri-resmgr-webhook: delete cri-resmgr-webhook from k8s.\n    local target=\"$1\"\n    case $target in\n        \"cri-resmgr\")\n            vm-command \"fuser --kill ${cri_resmgr_pidfile} 2>/dev/null\"\n            ;;\n        \"cri-resmgr-agent\")\n            vm-command \"fuser --kill ${cri_resmgr_agent_sock} 2>/dev/null\"\n            ;;\n        \"cri-resmgr-webhook\")\n            vm-command \"kubectl delete -f webhook/mutating-webhook-config.yaml; kubectl delete -f webhook/webhook-deployment.yaml\"\n            ;;\n        *)\n            error \"terminate: invalid target \\\"$target\\\"\"\n            ;;\n    esac\n}\n\nsleep() { # script API\n    # Usage: sleep PARAMETERS\n    #\n    # Run sleep PARAMETERS on host.\n    host-command \"sleep $*\"\n}\n\nextended-resources() { # script API\n    # Usage: extended-resources <add|remove> RESOURCE [VALUE]\n    #\n    # Examples:\n    #   extended-resources remove cmk.intel.com/exclusive-cpus\n    #   extended-resources add cmk.intel.com/exclusive-cpus 4\n    local action=\"$1\"\n    local resource=\"$2\"\n    local value=\"$3\"\n    local resource_escaped=\"${resource/\\//~1}\"\n    if [ -z \"$resource\" ]; then\n        error \"extended-resource: missing resource\"\n        return 1\n    fi\n    # make sure kubectl proxy is running\n    vm-command-q \"ss -ltn | grep -q 127.0.0.1:8001 || { kubectl proxy &>/dev/null </dev/null & sleep 2 ; }\"\n    case $action in\n        add)\n            if [ -z \"$value\" ]; then\n                error \"extended-resource: missing value to add to resource $resource\"\n                return 1\n            fi\n            vm-command \"curl --header 'Content-Type: application/json-patch+json' --request PATCH --data '[{\\\"op\\\": \\\"add\\\", \\\"path\\\": \\\"/status/capacity/$resource_escaped\\\", \\\"value\\\": \\\"$value\\\"}]' http://localhost:8001/api/v1/nodes/\\$(hostname)/status\"\n            ;;\n        remove)\n            vm-command \"curl --header 'Content-Type: application/json-patch+json' --request PATCH --data '[{\\\"op\\\": \\\"remove\\\", \\\"path\\\": \\\"/status/capacity/$resource_escaped\\\"}]' http://localhost:8001/api/v1/nodes/\\$(hostname)/status\"\n            ;;\n        *)\n            error \"extended-resource: invalid action \\\"$action\\\"\"\n            return 1\n            ;;\n    esac\n}\n\npyexec() { # script API\n    # Usage: pyexec [PYTHONCODE...]\n    #\n    # Run python3 -c PYTHONCODEs on host. Stops if execution fails.\n    #\n    # Variables available in PYTHONCODE:\n    #   allocations: dictionary: shorthand to cri-resmgr policy allocations\n    #                (unmarshaled cache['PolicyJSON']['allocations'])\n    #   allowed      tree: {package: {die: {node: {core: {thread: {pod}}}}}}\n    #                resource topology and pods allowed to use the resources.\n    #   packages, dies, nodes, cores, threads:\n    #                dictionaries: {podname: set-of-allowed}\n    #                Example: pyexec 'print(dies[\"pod0c0\"])'\n    #   cache:       dictionary, cri-resmgr cache\n    #\n    # Note that variables are *not* updated when pyexec is called.\n    # You can update the variables by running \"verify\" without expressions.\n    #\n    # Code in environment variable py_consts runs before PYTHONCODE.\n    #\n    # Example:\n    #   verify ; pyexec 'import pprint; pprint.pprint(allowed)'\n    PYEXEC_STATE_PY=\"$OUTPUT_DIR/pyexec_state.py\"\n    PYEXEC_PY=\"$OUTPUT_DIR/pyexec.py\"\n    PYEXEC_LOG=\"$OUTPUT_DIR/pyexec.output.txt\"\n    local last_exit_status=0\n    {\n        echo \"import pprint; pp=pprint.pprint\"\n        echo \"# \\$py_allowed:\"\n        echo -e \"$py_allowed\"\n        echo \"# \\$py_cache:\"\n        echo -e \"$py_cache\"\n        echo \"# \\$py_consts:\"\n        echo -e \"$py_consts\"\n    } > \"$PYEXEC_STATE_PY\"\n    for PYTHONCODE in \"$@\"; do\n        {\n            echo \"from pyexec_state import *\"\n            echo -e \"$PYTHONCODE\"\n        } > \"$PYEXEC_PY\"\n        PYTHONPATH=\"$OUTPUT_DIR:$PYTHONPATH:$DEMO_LIB_DIR\" python3 \"$PYEXEC_PY\" 2>&1 | tee \"$PYEXEC_LOG\"\n        last_exit_status=${PIPESTATUS[0]}\n        if [ \"$last_exit_status\" != \"0\" ]; then\n            error \"pyexec: non-zero exit status \\\"$last_exit_status\\\", see \\\"$PYEXEC_PY\\\" and \\\"$PYEXEC_LOG\\\"\"\n        fi\n    done\n    return \"$last_exit_status\"\n}\n\npp() { # script API\n    # Usage: pp EXPR\n    #\n    # Pretty-print the value of Python expression EXPR.\n    pyexec \"pp($*)\"\n}\n\nreport() { # script API\n    # Usage: report [VARIABLE...]\n    #\n    # Updates and reports current value of VARIABLE.\n    #\n    # Supported VARIABLEs:\n    #     allocations\n    #     allowed\n    #     cache\n    #\n    # Example: print cri-resmgr policy allocations. In interactive mode\n    #          you may use a pager like less.\n    #   report allocations | less\n    local varname\n    for varname in \"$@\"; do\n        if [ \"$varname\" == \"allocations\" ]; then\n            get-py-cache\n            pyexec \"\nimport pprint\npprint.pprint(allocations)\n\"\n        elif [ \"$varname\" == \"allowed\" ]; then\n            get-py-allowed\n            pyexec \"\nimport topology\nprint(topology.str_tree(allowed))\n\"\n        elif [ \"$varname\" == \"cache\" ]; then\n            get-py-cache\n            pyexec \"\nimport pprint\npprint.pprint(cache)\n\"\n        else\n            error \"report: unknown variable \\\"$varname\\\"\"\n        fi\n    done\n}\n\nverify() { # script API\n    # Usage: verify [--retry N] [EXPR...]\n    #\n    # Run python3 -c \"assert(EXPR)\" to test that every EXPR is True.\n    # Stop immediately after the first failing assertion and fail the test.\n    #\n    # If a verify is expected to fail, failing the whole test can be\n    # prevented by running the verify in a subshell (in parenthesis):\n    #   (verify 'False') || echo '...this was expected to fail.'\n    #\n    # --retry N reruns all assertions at most N times before failing\n    # the test. All assertions must hold at the same time for a\n    # successful verification. By default N=3.\n    #\n    # Variables available in EXPRs:\n    #   See variables in: help pyexec\n    #\n    # Note that all variables are updated every time verify is called\n    # before evaluating (asserting) expressions.\n    #\n    # Example: require that containers pod0c0 and pod1c0 run on separate NUMA\n    #          nodes and that pod0c0 is allowed to run on 4 CPUs:\n    #   verify 'set.intersection(nodes[\"pod0c0\"], nodes[\"pod1c0\"]) == set()' \\\n    #          'len(cpus[\"pod0c0\"]) == 4'\n    local retries=3\n    local poll_delay=1s\n    if [[ \"$1\" == \"--retry\" ]]; then\n        retries=\"$2\"\n        shift; shift\n    fi\n    while ! _verify \"$@\"; do\n        if (( retries <= 0 )); then\n            if is-hooked on_verify_fail; then\n                run-hook on_verify_fail\n            else\n                command-exit-if-not-interactive\n            fi\n            return 1\n        fi\n        out \"### Retrying verify at most $retries time(s) after $poll_delay...\"\n        sleep \"$poll_delay\"\n        retries=$(( retries - 1 ))\n    done\n    is-hooked on_verify && run-hook on_verify\n    return 0\n}\n\n_verify() {\n    get-py-allowed\n    get-py-cache\n    for py_assertion in \"$@\"; do\n        speed=1000 out \"### Verifying assertion '$py_assertion'\"\n        ( speed=1000 pyexec \"\ntry:\n    import time,sys\n    assert(${py_assertion})\nexcept KeyError as e:\n    print('WARNING: *')\n    print('WARNING: *** KeyError - %s' % str(e))\n    print('WARNING: *** Your verify expression might have a typo/thinko.')\n    print('WARNING: *')\n    sys.stdout.flush()\n    time.sleep(5)\n    raise e\nexcept IndexError as e:\n    print('WARNING: *')\n    print('WARNING: *** IndexError - %s' % str(e))\n    print('WARNING: *** Your verify expression might have a typo/thinko.')\n    print('WARNING: *')\n    sys.stdout.flush()\n    time.sleep(5)\n    raise e\n\" ) || {\n                out \"### The assertion FAILED\n### post-mortem debug help begin ###\ncd $OUTPUT_DIR\npython3\nfrom pyexec_state import *\n$py_assertion\n### post-mortem debug help end ###\"\n                echo \"verify: assertion '$py_assertion' failed.\" >> \"$SUMMARY_FILE\"\n                return 1\n        }\n        speed=1000 out \"### The assertion holds.\"\n    done\n    return 0\n}\n\nkubectl-force-delete() { # script API\n    # Usage: kubectl-force-delete RESOURCE NAME\n    #\n    # Force-deleting a \"Terminating\" namespace clears finalizers that\n    # have failed to finish. Therefore there may be resources left in the\n    # namespace NAME. Following command prints them.\n    #\n    #     kubectl api-resources --verbs=list --namespaced -o name | \\\n    #       xargs -n 1 kubectl get --show-kind --ignore-not-found -n NAME\n    #\n    # Example: delete a namespace that is stuck in the \"Terminating\" phase\n    #\n    #     kubectl-force-delete namespace my-namespace\n\n    if [ -z \"$1\" ]; then\n        error \"missing RESOURCE\"\n        return 1\n    fi\n\n    if [ -z \"$2\" ]; then\n        error \"missing resource NAME\"\n        return 1\n    fi\n\n    if [[ \"$1\" == \"namespace\" ]] || [[ \"$1\" == \"ns\" ]]; then\n        local ns=\"$2\"\n        vm-command \"\n            kubectl get namespace $ns -o json > force-delete-ns.json || exit 0\n            (\n              grep -E phase.*Terminating force-delete-ns.json || exit 0\n              tr -d '\\n' < force-delete-ns.json \\\n              | sed 's/\\\"finalizers\\\": \\[[^]]\\+\\]/\\\"finalizers\\\": []/' \\\n              | kubectl replace --raw /api/v1/namespaces/$ns/finalize -f -\n            )\n            rm -f force-delete-ns.json\n            \"\n    else\n        error \"unsupported force-delete resource: $1\"\n        return 1\n    fi\n}\n\nkubectl() { # script API\n    # Usage: kubectl parameters\n    #\n    # Runs kubectl command on virtual machine.\n    vm-command \"kubectl $*\" || {\n        command-error \"kubectl $* failed\"\n    }\n}\n\ndelete() { # script API\n    # Usage: delete PARAMETERS\n    #\n    # Run \"kubectl delete PARAMETERS\".\n    vm-command \"kubectl delete $*\" || {\n        command-error \"kubectl delete failed\"\n    }\n}\n\ninstantiate() { # script API\n    # Usage: instantiate FILENAME\n    #\n    # Produces $OUTPUT_DIR/instance/FILENAME. Prints the filename on success.\n    # Uses FILENAME.in as source (resolved from $TEST_DIR, $TOPOLOGY_DIR, ...)\n    local FILENAME=\"$1\"\n    local RESULT=\"$OUTPUT_DIR/instance/$FILENAME\"\n\n    template_file=$(resolve-template \"$FILENAME\" \"$TEST_DIR\" \"$TOPOLOGY_DIR\" \"$POLICY_DIR\" \"$SCRIPT_DIR\")\n    if [ ! -f \"$template_file\" ]; then\n        error \"error instantiating \\\"$FILENAME\\\": missing template ${template_file}\"\n    fi\n    mkdir -p \"$(dirname \"$RESULT\")\" 2>/dev/null\n    eval \"echo -e \\\"$(<\"${template_file}\")\\\"\" | grep -v '^ *$' > \"$RESULT\" ||\n        error \"instantiating \\\"$FILENAME\\\" failed\"\n    echo \"$RESULT\"\n}\n\ndeclare -a pulled_images_on_vm\ncreate() { # script API\n    # Usage: [VAR=VALUE][n=COUNT] create TEMPLATE_NAME\n    #\n    # Create n instances from TEMPLATE_NAME.yaml.in, copy each of them\n    # from host to vm, kubectl create -f them, and wait for them\n    # becoming Ready. Templates are searched in $TEST_DIR, $TOPOLOGY_DIR,\n    # $POLICY_DIR, and $SCRIPT_DIR in this order of preference. The first\n    # template found is used.\n    #\n    # Parameters:\n    #   TEMPLATE_NAME: the name of the template without extension (.yaml.in)\n    #\n    # Optional parameters (VAR=VALUE):\n    #   namespace: namespace to which instances are created\n    #   wait: condition to be waited for (see kubectl wait --for=condition=).\n    #         If empty (\"\"), skip waiting. The default is wait=\"Ready\".\n    #   wait_t: wait timeout. The default is wait_t=240s.\n    local template_file\n    template_file=$(resolve-template \"$1.yaml\" \"$TEST_DIR\" \"$TOPOLOGY_DIR\" \"$POLICY_DIR\" \"$SCRIPT_DIR\")\n    local namespace_args\n    local template_kind\n    template_kind=$(awk '/kind/{print tolower($2)}' < \"$template_file\")\n    local wait=${wait-Ready}\n    local wait_t=${wait_t-240s}\n    local images\n    local image\n    local tag\n    local errormsg\n    local default_name=${NAME:-\"\"}\n    if [ -z \"$n\" ]; then\n        local n=1\n    fi\n    if [ -n \"${namespace:-}\" ]; then\n        namespace_args=\"-n $namespace\"\n    else\n        namespace_args=\"\"\n    fi\n    if [ ! -f \"$template_file\" ]; then\n        error \"error creating from template \\\"$template_file.yaml.in\\\": template file not found\"\n    fi\n    for _ in $(seq 1 $n); do\n        kind_count[$template_kind]=$(( ${kind_count[$template_kind]} + 1 ))\n        if [ -n \"$default_name\" ]; then\n            local NAME=\"$default_name\"\n        else\n            local NAME=\"${template_kind}$(( ${kind_count[$template_kind]} - 1 ))\" # the first pod is pod0\n        fi\n        eval \"echo -e \\\"$(<\"${template_file}\")\\\"\" | grep -v '^ *$' > \"$OUTPUT_DIR/$NAME.yaml\"\n        host-command \"$SCP \\\"$OUTPUT_DIR/$NAME.yaml\\\" $VM_SSH_USER@$VM_IP:\" || {\n            command-error \"copying \\\"$OUTPUT_DIR/$NAME.yaml\\\" to VM failed\"\n        }\n        vm-command \"cat $NAME.yaml\"\n        images=\"$(grep -E '^ *image: .*$' \"$OUTPUT_DIR/$NAME.yaml\" | sed -E 's/^ *image: *([^ ]*)$/\\1/g' | sort -u)\"\n        if [ \"${#pulled_images_on_vm[@]}\" = \"0\" ]; then\n            # Initialize pulled images available on VM\n            vm-command \"crictl -i unix://${k8scri_sock} images\" >/dev/null &&\n            while read -r image tag _; do\n                if [ \"$image\" = \"IMAGE\" ]; then\n                    continue\n                fi\n                local notopdir_image=\"${image#*/}\"\n                local norepo_image=\"${image##*/}\"\n                if [ \"$tag\" = \"latest\" ]; then\n                    pulled_images_on_vm+=(\"$image\")\n                    pulled_images_on_vm+=(\"$notopdir_image\")\n                    pulled_images_on_vm+=(\"$norepo_image\")\n                fi\n                pulled_images_on_vm+=(\"$image:$tag\")\n                pulled_images_on_vm+=(\"$notopdir_image:$tag\")\n                pulled_images_on_vm+=(\"$norepo_image:$tag\")\n            done <<< \"$COMMAND_OUTPUT\"\n        fi\n        for image in $images; do\n            if ! [[ \" ${pulled_images_on_vm[*]} \" == *\" ${image} \"* ]]; then\n                if [ \"$use_host_images\" == \"1\" ] && vm-put-docker-image \"$image\"; then\n                    : # no need to pull the image to vm, it is now imported.\n                else\n                    vm-command \"crictl -i unix://${k8scri_sock} pull \\\"$image\\\"\" || {\n                        errormsg=\"pulling image \\\"$image\\\" for \\\"$OUTPUT_DIR/$NAME.yaml\\\" failed.\"\n                        if is-hooked on_create_fail; then\n                            echo \"$errormsg\"\n                            run-hook on_create_fail\n                        else\n                            command-error \"$errormsg\"\n                        fi\n                    }\n                fi\n                pulled_images_on_vm+=(\"$image\")\n            fi\n        done\n        vm-command \"kubectl create -f $NAME.yaml $namespace_args\" || {\n            if is-hooked on_create_fail; then\n                echo \"kubectl create error\"\n                run-hook on_create_fail\n            else\n                command-error \"kubectl create error\"\n            fi\n        }\n        if [ \"x$wait\" != \"x\" ]; then\n            speed=1000 vm-command \"kubectl wait --timeout=${wait_t} --for=condition=${wait} $namespace_args ${template_kind}/$NAME\" >/dev/null 2>&1 || {\n                errormsg=\"waiting for ${template_kind} \\\"$NAME\\\" to become ready timed out\"\n                if is-hooked on_create_fail; then\n                    echo \"$errormsg\"\n                    run-hook on_create_fail\n                else\n                    command-error \"$errormsg\"\n                fi\n            }\n        fi\n    done\n    is-hooked on_create && run-hook on_create\n    return 0\n}\n\nreset() { # script API\n    # Usage: reset counters\n    #\n    # Resets counters\n    if [ \"$1\" == \"counters\" ]; then\n        kind_count[pod]=0\n    else\n        error \"invalid reset \\\"$1\\\"\"\n    fi\n}\n\ninteractive() { # script API\n    # Usage: interactive\n    #\n    # Enter the interactive mode: read next script commands from\n    # the standard input until \"exit\".\n    echo \"Entering the interactive mode until \\\"exit\\\".\"\n    INTERACTIVE_MODE=$(( INTERACTIVE_MODE + 1 ))\n    # shellcheck disable=SC2162\n    while read -e -p \"run.sh> \" -a commands; do\n        if [ \"${commands[0]}\" == \"exit\" ]; then\n            break\n        fi\n        eval \"${commands[@]}\"\n    done\n    INTERACTIVE_MODE=$(( INTERACTIVE_MODE - 1 ))\n}\n\nhelp() { # script API\n    # Usage: help [FUNCTION|all]\n    #\n    # Print help on all functions or on the FUNCTION available in script.\n    awk -v f=\"$1\" \\\n        '/^[a-z].*script API/{split($1,a,\"(\");if(f==\"\"||f==a[1]||f==\"all\"){print \"\";print a[1]\":\";l=2}}\n         !/^    #/{l=l-1}\n         /^    #/{if(l>=1){split($0,a,\"#\"); print \"   \"a[2]; if (f==\"\") l=0}}' <<<\"$script_source\"\n}\n\n### End of user code helpers\n\ntest-user-code() {\n    vm-command-q \"kubectl get pods 2>&1 | grep -q NAME\" && vm-command \"kubectl delete pods --all --now --wait\"\n    ( eval \"$code\" ) || {\n        TEST_FAILURES=\"${TEST_FAILURES} test script failed\"\n    }\n}\n\n# Validate parameters\ninput_var_names=\"mode user_script_file distro k8scri k8smaster vm cgroups speed binsrc reinstall_all reinstall_containerd reinstall_crio reinstall_cri_resmgr reinstall_k8s reinstall_oneshot outdir cleanup on_verify_fail on_create_fail on_verify on_create on_launch topology cri_resmgr_cfg cri_resmgr_extra_args cri_resmgr_agent_extra_args code py_consts\"\n\nINTERACTIVE_MODE=0\nmode=$1\nuser_script_file=$2\ndistro=${distro:=$DEFAULT_DISTRO}\nk8s=${k8s:=}\nk8scri=${k8scri:=\"cri-resmgr|containerd\"}\nk8smaster=${k8smaster:=}\ncri_resmgr_pidfile=\"/var/run/cri-resmgr*.pid\"\ncri_resmgr_sock=\"/var/run/cri-resmgr/cri-resmgr.sock\"\ncri_resmgr_agent_sock=\"/var/run/cri-resmgr/cri-resmgr-agent.sock\"\ncase \"${k8scri}\" in\n    \"cri-resmgr|containerd\")\n        k8scri_sock=\"${cri_resmgr_sock}\"\n        cri_sock=\"/var/run/containerd/containerd.sock\"\n        cri=containerd\n        ;;\n    \"cri-resmgr|crio\")\n        k8scri_sock=\"${cri_resmgr_sock}\"\n        cri_sock=\"/var/run/crio/crio.sock\"\n        cri=crio\n        ;;\n    \"containerd\")\n        k8scri_sock=\"/var/run/containerd/containerd.sock\"\n        cri_sock=\"/var/run/containerd/containerd.sock\"\n        cri=containerd\n        omit_cri_resmgr=1\n        omit_agent=1\n        ;;\n    \"containerd&cri-resmgr\")\n        k8scri_sock=\"/var/run/containerd/containerd.sock\"\n        cri_sock=\"/var/run/containerd/containerd.sock\"\n        cri=containerd\n        ;;\n    \"crio\")\n        k8scri_sock=\"/var/run/crio/crio.sock\"\n        cri_sock=\"/var/run/crio/crio.sock\"\n        cri=crio\n        omit_cri_resmgr=1\n        omit_agent=1\n        ;;\n    \"crio&cri-resmgr\")\n        k8scri_sock=\"/var/run/crio/crio.sock\"\n        cri_sock=\"/var/run/crio/crio.sock\"\n        cri=crio\n        ;;\n    *)\n        error \"unsupported k8scri: \\\"${k8scri}\\\"\"\n        ;;\nesac\ndistro_binaries=${distro_binaries:=0}\ncontainerd_src=${containerd_src:=}\ncrio_src=${crio_src:=}\ncrirm_src=${crirm_src:=$HOST_PROJECT_DIR}\nrunc_src=${runc_src:=}\ncrio_version=${crio_version:=}\nif [ \"$distro_binaries\" = \"1\" ]; then\n    if [ -z \"$distro\" ]; then\n        error \"distro_binaries=1 but distro is not set\"\n    fi\n    BIN_DIR=${crirm_src}/binaries/$distro\nelse\n    BIN_DIR=${crirm_src}/bin\nfi\nTOPOLOGY_DIR=${TOPOLOGY_DIR:=e2e}\nvm=${vm:=$(basename ${TOPOLOGY_DIR})-${distro}-${cri}}\nvm_files=${vm_files:-\"\"}\ncgroups=${cgroups:-v1}\ncri_resmgr_cfg=${cri_resmgr_cfg:-\"${SCRIPT_DIR}/cri-resmgr-topology-aware.cfg\"}\ncri_resmgr_extra_args=${cri_resmgr_extra_args:-\"\"}\ncri_resmgr_agent_extra_args=${cri_resmgr_agent_extra_args:-\"\"}\ncleanup=${cleanup:-0}\nreinstall_all=${reinstall_all:-0}\nreinstall_bootstrap=${reinstall_bootstrap:-0}\nreinstall_containerd=${reinstall_containerd:-0}\nreinstall_cri_resmgr=${reinstall_cri_resmgr:-0}\nreinstall_cri_resmgr_agent=${reinstall_cri_resmgr_agent:-0}\nreinstall_crio=${reinstall_crio:-0}\nreinstall_k8s=${reinstall_k8s:-0}\nreinstall_kubeadm=${reinstall_kubeadm:-0}\nreinstall_kubectl=${reinstall_kubectl:-0}\nreinstall_kubelet=${reinstall_kubelet:-0}\nreinstall_oneshot=${reinstall_oneshot:-0}\nreinstall_runc=${reinstall_runc:-0}\nif [ \"$reinstall_all\" == \"1\" ]; then\n    for reinstall_var in ${!reinstall_*}; do\n        eval \"${reinstall_var}=1\"\n    done\nfi\nif [ \"$reinstall_k8s\" == \"1\" ]; then\n    reinstall_kubeadm=1\n    reinstall_kubectl=1\n    reinstall_kubelet=1\nfi\nif [ \"$reinstall_bootstrap\" == \"1\" ]; then\n    setup_proxies=1\nfi\nomit_agent=${omit_agent:-0}\nomit_cri_resmgr=${omit_cri_resmgr:-0}\nuse_host_images=${use_host_images:-0}\npy_consts=\"${py_consts:-''}\"\ntopology=${topology:-'[\n    {\"mem\": \"1G\", \"cores\": 1, \"nodes\": 2, \"packages\": 2, \"node-dist\": {\"4\": 28, \"5\": 28}},\n    {\"nvmem\": \"8G\", \"node-dist\": {\"5\": 28, \"0\": 17}},\n    {\"nvmem\": \"8G\", \"node-dist\": {\"2\": 17}}\n    ]'}\ncode=${code:-\"\nCPU=1 create guaranteed # creates pod 0, 1 CPU taken\nreport allowed\nCPU=2 create guaranteed # creates pod 1, 3 CPUs taken\nreport allowed\nCPU=3 create guaranteed # creates pod 2, 6 CPUs taken\nreport allowed\nverify \\\\\n    'len(cpus[\\\"pod0c0\\\"]) == 1' \\\\\n    'len(cpus[\\\"pod1c0\\\"]) == 2' \\\\\n    'len(cpus[\\\"pod2c0\\\"]) == 3' \\\\\n    'len(set.union(cpus[\\\"pod0c0\\\"], cpus[\\\"pod1c0\\\"], cpus[\\\"pod2c0\\\"])) == 6'\nn=3 create besteffort   # creates pods 3, 4 and 5\nverify \\\\\n    'set.intersection(\n       set.union(cpus[\\\"pod0c0\\\"], cpus[\\\"pod1c0\\\"], cpus[\\\"pod2c0\\\"]),\n       set.union(cpus[\\\"pod3c0\\\"], cpus[\\\"pod4c0\\\"], cpus[\\\"pod5c0\\\"])) == set()'\n\ndelete pods pod2        # deletes pod 2, 3 CPUs taken\nn=2 create besteffort   # creates pods 6 and 7\nCPU=2 n=2 create guaranteed # creates pod 8 and 9, 7 CPUs taken\nverify \\\\\n    'len(set.union(cpus[\\\"pod0c0\\\"], cpus[\\\"pod1c0\\\"], cpus[\\\"pod8c0\\\"], cpus[\\\"pod9c0\\\"])) == 7'\n\"}\nwarning_delay=${warning_delay:-5}\n\nyaml_in_defaults=\"CPU=1 MEM=100M ISO=true CPUREQ=1 CPULIM=2 MEMREQ=100M MEMLIM=200M CONTCOUNT=1\"\n\nif [ \"$mode\" == \"help\" ]; then\n    if [ \"$2\" == \"defaults\" ]; then\n        echo \"Test input defaults:\"\n        echo \"\"\n        echo \"topology=${topology}\"\n        echo \"distro=${distro}\"\n        echo \"k8s=${k8s}\"\n        echo \"\"\n        echo \"cri_resmgr_cfg=${cri_resmgr_cfg}\"\n        echo \"\"\n        echo \"cri_resmgr_extra_args=${cri_resmgr_extra_args}\"\n        echo \"\"\n        echo -e \"code=\\\"${code}\\\"\"\n        echo \"\"\n        echo \"The defaults to QOSCLASS.yaml.in variables:\"\n        echo \"    ${yaml_in_defaults}\"\n    elif [ \"$2\" == \"script\" ]; then\n        if [ \"x$3\" == \"x\" ]; then\n            help\n        else\n            help \"$3\"\n        fi\n    elif [ \"x$2\" == \"x\" ]; then\n        usage\n    else\n        echo \"invalid help page, try:\"\n        echo \"  ./run.sh help\"\n        echo \"  ./run.sh help defaults\"\n        echo \"  ./run.sh help script [FUNCTION|all]\"\n        exit 1\n    fi\n    exit 0\nelif [ \"$mode\" == \"play\" ]; then\n    speed=${speed-10}\nelif [ \"$mode\" == \"test\" ]; then\n    PV=\nelif [ \"$mode\" == \"debug\" ]; then\n    PV=\nelif [ \"$mode\" == \"interactive\" ]; then\n    PV=\nelif [ \"$mode\" == \"record\" ]; then\n    record\nelse\n    usage\n    error \"missing valid MODE\"\n    exit 1\nfi\n\nhost-require-cmd jq\nhost-require-cmd pv\n\nif [ -n \"$user_script_file\" ]; then\n    if [ ! -f \"$user_script_file\" ]; then\n        error \"cannot find test script file \\\"$user_script_file\\\"\"\n    fi\n    code=$(<\"$user_script_file\")\nfi\n\n# Prepare for test/demo\nmkdir -p \"$OUTPUT_DIR\"\nmkdir -p \"$COMMAND_OUTPUT_DIR\"\nrm -f \"$COMMAND_OUTPUT_DIR\"/0*\n( echo x > \"$OUTPUT_DIR\"/x && rm -f \"$OUTPUT_DIR\"/x ) || {\n    error \"output directory outdir=$OUTPUT_DIR is not writable\"\n}\n\nSUMMARY_FILE=\"$OUTPUT_DIR/summary.txt\"\necho -n \"\" > \"$SUMMARY_FILE\" || error \"cannot write summary to \\\"$SUMMARY_FILE\\\"\"\n\n## Save test inputs and defaults for the record\nmkdir -p \"$OUTPUT_DIR/input\"; rm -f \"$OUTPUT_DIR/input/*\"\nfor var in $input_var_names; do\n    if [ -n \"${!var}\" ]; then\n        echo -e \"${!var}\" > \"$OUTPUT_DIR/input/${var}.var\"\n    fi\ndone\n\nif [ \"$binsrc\" == \"local\" ]; then\n    if [ \"$omit_cri_resmgr\" != \"1\" ]; then\n        [ -f \"${BIN_DIR}/cri-resmgr\" ] || error \"missing \\\"${BIN_DIR}/cri-resmgr\\\"\"\n    fi\n    if [ \"$omit_agent\" != \"1\" ]; then\n        [ -f \"${BIN_DIR}/cri-resmgr-agent\" ] || error \"missing \\\"${BIN_DIR}/cri-resmgr-agent\\\"\"\n    fi\nfi\n\nhost-get-vm-config \"$vm\" || host-set-vm-config \"$vm\" \"$distro\" \"$cri\"\n\nif [ -z \"$VM_IP\" ] || [ -z \"$VM_SSH_USER\" ]; then\n    screen-create-vm\nelse\n    if [ \"$setup_proxies\" == \"1\" ]; then\n\tvm-setup-proxies\n    fi\n\n    if [ \"$reinstall_bootstrap\" == \"1\" ]; then\n\tvm-bootstrap\n    fi\nfi\n\nis-hooked \"on_vm_online\" && run-hook \"on_vm_online\"\n\nif [ \"$reinstall_oneshot\" == \"1\" ] || ! vm-command-q \"[ -f .vm-setup-oneshot ]\"; then\n    vm-setup-oneshot\n    vm-command-q \"touch .vm-setup-oneshot\"\nfi\n\nif [ -n \"$vm_files\" ]; then\n    install-files \"$vm_files\"\nfi\n\nif [ \"$reinstall_containerd\" == \"1\" ] || [ \"$reinstall_crio\" == \"1\" ] || ! vm-command-q \"( type -p containerd || type -p crio ) >/dev/null\"; then\n    vm-install-cri\n    is-hooked on_cri_install && run-hook on_cri_install\nfi\n\n# runc is installed as a dependency of containerd and crio.\n# If reinstalling runc is explictly wished for, it is safe to do\n# only after (re)installing contaienrd/crio. Otherwise\n# a custom locally built runc may be overridden from packages.\nif [ \"$reinstall_runc\" == \"1\" ] || ! vm-command-q \"type -p runc >/dev/null\"; then\n    vm-install-runc\n    is-hooked on_runc_install && run-hook on_runc_install\nfi\n\nif [ \"$reinstall_k8s\" == \"1\" ] || ! vm-command-q \"type -p kubelet >/dev/null\"; then\n    vm-install-k8s\n    is-hooked on_k8s_install && run-hook on_k8s_install\nfi\n\nif [ \"$reinstall_cri_resmgr\" == \"1\" ]; then\n    uninstall cri-resmgr\nfi\n\nif [ \"$reinstall_cri_resmgr_agent\" == \"1\" ]; then\n    uninstall cri-resmgr-agent\nfi\n\nif [[ \"$k8scri\" == cri-resmgr* ]] || [ -n \"$crirm_src\" ]; then\n    if [ \"$omit_cri_resmgr\" != \"1\" ]; then\n        if ! vm-command-q \"type -p cri-resmgr >/dev/null\"; then\n            install cri-resmgr\n        fi\n    fi\n\n    if [ \"$omit_agent\" != \"1\" ]; then\n        if ! vm-command-q \"type -p cri-resmgr-agent >/dev/null\"; then\n            install cri-resmgr-agent\n        fi\n    fi\nfi\n\nif [ \"$mode\" == \"debug\" ]; then\n    vm-command-q \"[ -x /root/go/bin/dlv ]\" || vm-install-dlv\n    if [ -d \"$crio_src\" ]; then\n        vm-dlv-add-src \"$crio_src\"\n    fi\n    if [ -d \"$containerd_src\" ]; then\n        vm-dlv-add-src \"$containerd_src\"\n    fi\n    if [ -d \"$crirm_src\" ]; then\n        vm-dlv-add-src \"$crirm_src\"\n    fi\n    if [ -d \"$runc_src\" ]; then\n        vm-dlv-add-src \"$runc_src\"\n    fi\n    echo \"How to debug cri-resmgr:\"\n    echo \"- Attach debugger to running cri-resmgr:\"\n    echo \"  ssh $VM_SSH_USER@$VM_IP\"\n    echo \"  sudo /root/go/bin/dlv attach \\$(pidof cri-resmgr)\"\n    echo \"- Relaunch cri-resmgr in debugger:\"\n    echo \"  ssh $VM_SSH_USER@$VM_IP\"\n    echo \"  sudo -i\"\n    echo \"  kill -9 \\$(pidof cri-resmgr); /root/go/bin/dlv exec /usr/local/bin/cri-resmgr -- -force-config /home/$VM_SSH_USER/*.cfg\"\n    echo \"dlv on VM is ready for use\"\n    exit 0\nfi\n\nif [ -n \"$containerd_src\" ] && [[ \"$k8scri\" == *containerd* ]]; then\n    vm-check-source-files-changed \"$containerd_src\" \"$containerd_src/bin/containerd\"\n    vm-check-running-binary \"$containerd_src/bin/containerd\"\nfi\n\nif [ -n \"$crio_src\" ] && [[ \"$k8scri\" == *crio* ]]; then\n    vm-check-source-files-changed \"$crio_src\" \"$crio_src/bin/crio\"\n    vm-check-running-binary \"$crio_src/bin/crio\"\nfi\n\n# Start cri-resmgr if not already running\nif [ \"$omit_cri_resmgr\" != \"1\" ]; then\n    if ! vm-command-q \"fuser ${cri_resmgr_pidfile}\" >/dev/null 2>&1; then\n        screen-launch-cri-resmgr\n    fi\n    if [ -n \"$crirm_src\" ]; then\n        vm-check-source-files-changed \"$crirm_src\" \"$crirm_src/bin/cri-resmgr\"\n        vm-check-running-binary \"$crirm_src/bin/cri-resmgr\"\n    fi\nfi\n\n# Create kubernetes cluster or wait that it is online\nif [ \"$reinstall_k8s\" == \"1\" ]; then\n    vm-destroy-cluster\nfi\n\nif vm-command-q \"[ ! -f /var/lib/kubelet/config.yaml ]\"; then\n    if [ -n \"$k8smaster\" ]; then\n        vm-join \"$k8smaster\"\n    else\n        screen-create-singlenode-cluster\n    fi\nelse\n    # Wait for kube-apiserver to launch (may be down if the VM was just booted)\n    vm-wait-process kube-apiserver\nfi\n\n# Start cri-resmgr-agent if not already running\nif [ \"$omit_agent\" != \"1\" ]; then\n    if ! vm-command-q \"fuser ${cri_resmgr_agent_sock}\" >/dev/null; then\n        screen-launch-cri-resmgr-agent\n    fi\nfi\n\nis-hooked \"on_k8s_online\" && run-hook \"on_k8s_online\"\n\ndeclare -A kind_count # associative arrays for counting created objects, like kind_count[pod]=1\neval \"${yaml_in_defaults}\"\nif [ \"$mode\" == \"interactive\" ]; then\n    interactive\nelse\n    # Run test/demo\n    TEST_FAILURES=\"\"\n    test-user-code\nfi\n\n# Save logs\nhost-command \"$SCP $VM_SSH_USER@$VM_IP:cri-resmgr*.output.txt \\\"$OUTPUT_DIR/\\\"\"\n\n# Cleanup\nif [ \"$cleanup\" == \"0\" ]; then\n    echo \"The VM, Kubernetes and cri-resmgr are left running. Next steps:\"\n    vm-print-usage\nelif [ \"$cleanup\" == \"1\" ]; then\n    host-stop-vm \"$vm\"\n    host-delete-vm \"$vm\"\nelif [ \"$cleanup\" == \"2\" ]; then\n    host-stop-vm \"$vm\"\nfi\n\n# Summarize results\nexit_status=0\nif [ \"$mode\" == \"test\" ]; then\n    if [ -n \"$TEST_FAILURES\" ]; then\n        echo \"Test verdict: FAIL\" >> \"$SUMMARY_FILE\"\n    else\n        echo \"Test verdict: PASS\" >> \"$SUMMARY_FILE\"\n    fi\n    cat \"$SUMMARY_FILE\"\nfi\nexit $exit_status\n"
  },
  {
    "path": "test/e2e/run_all_configurations.sh",
    "content": "#!/bin/bash\n\nRUN_SH=\"${0%/*}/run.sh\"\nPAIRWISE=\"${0%/*}/../../scripts/testing/pairwise\"\n\n\"${PAIRWISE}\" \\\n    distro={debian-sid,fedora-40,opensuse-tumbleweed} \\\n    k8scri={containerd,crio,cri-resmgr\\|containerd,cri-resmgr\\|crio} \\\n    k8scni={cilium,flannel,weavenet} | while read -r env_vars; do\n\n    eval \"export $env_vars\"\n\n    code='create besteffort'\n    # shellcheck disable=SC2154\n    # ...as it cannot know that pairwise+eval exports distro et. al.\n    vm=\"config-$distro-${k8scri/|/-}-$k8scni\"\n    outdir=\"output-configs/output-$vm\"\n    export code vm outdir\n\n    govm rm \"$vm\" >/dev/null 2>&1\n    mkdir -p \"$outdir\"\n    \"$RUN_SH\" test </dev/null >\"$outdir/run.sh.output\" 2>&1\n    govm rm \"$vm\" >/dev/null 2>&1\ndone\n"
  },
  {
    "path": "test/e2e/run_tests.sh",
    "content": "#!/bin/bash\n\nTESTS_DIR=\"$1\"\nRUN_SH=\"${0%/*}/run.sh\"\n\nDEFAULT_DISTRO=\"ubuntu-22.04\"\n\nusage() {\n    echo \"Usage: run_tests.sh TESTS_DIR\"\n    echo \"TESTS_DIR is expected to be structured as POLICY/TOPOLOGY/TEST with files:\"\n    echo \"POLICY/cri-resmgr.cfg: configuration of cri-resmgr\"\n    echo \"POLICY/TOPOLOGY/topology.var.json: contents of the topology variable for run.sh\"\n    echo \"POLICY/TOPOLOGY/TEST/code.var.sh: contents of the code var (that is, test script)\"\n}\n\nerror() {\n    (echo \"\"; echo \"error: $1\" ) >&2\n    exit 1\n}\n\nwarning() {\n    echo \"WARNING: $1\" >&2\n}\n\nexport-var-files() {\n    # export ENV_VAR from ENV_VAR.var.* file content\n    local var_file_dir=\"$1\"\n    local var_filepath\n    local var_file_name\n    local var_name\n    for var_filepath in \"$var_file_dir\"/*.var \"$var_file_dir\"/*.var.*; do\n        if ! [ -f \"$var_filepath\" ] || [[ \"$var_filepath\" == *\"~\" ]] || [[ \"$var_filepath\" == *\"#\"* ]]; then\n            continue\n        fi\n        var_file_name=$(basename \"$var_filepath\")\n        var_name=${var_file_name%%.var*}\n        if [ \"$var_name\" == \"code\" ] || [ \"$var_name\" == \"py_consts\" ]; then\n            # append values in code variables\n            echo \"exporting $var_name - appending from $var_filepath\"\n            export \"$var_name\"=\"${!var_name}\"\"\n$(< \"$var_filepath\")\"\n        else\n            # creating / replace other variables\n            if [ -z \"${!var_name}\" ]; then\n                echo \"exporting $var_name - creating from $var_filepath\"\n            else\n                echo \"exporting $var_name - overriding from $var_filepath\"\n            fi\n            if [[ \"$var_file_name\" == *.var.in.* ]]; then\n                export \"$var_name\"=\"$(eval \"echo -e \\\"$(<\"${var_filepath}\")\\\"\")\"\n            else\n                export \"$var_name\"=\"$(< \"$var_filepath\")\"\n            fi\n        fi\n    done\n}\n\nexport-vm-files() {\n    # update and export vm_files associative array from directory content\n    local vm_files_dir=\"$1\"\n    if [ ! -d \"$vm_files_dir\" ]; then\n        return\n    fi\n    if [[ \"$vm_files\" == *\"=\"* ]] ; then\n        eval \"declare -A vm_files_aa=${vm_files#*=}\"\n    else\n        declare -A vm_files_aa\n    fi\n    prefix_len=${#vm_files_dir}\n    shopt -s globstar\n    for f in \"$vm_files_dir\"/**; do\n        file_vm_name=${f:$prefix_len}\n        if [ -z \"$file_vm_name\" ] || [ \"$file_vm_name\" == \"/\" ]; then\n            continue\n        elif [ -f \"$f\" ]; then\n            if [ -n \"${vm_files_aa[$file_vm_name]}\" ]; then\n                warning \"vm file $file_vm_name: new file \\\"$f\\\" overrides \\\"${vm_files_aa[$file_vm_name]}\\\"\"\n            fi\n            vm_files_aa[$file_vm_name]=\"file:$(realpath \"$f\")\"\n        fi\n    done\n    # serialize from associative array\n    local serialized_vm_files\n    serialized_vm_files=\"$(declare -p vm_files_aa)\"\n    export vm_files=\"declare -A vm_files${serialized_vm_files#declare -A vm_files_aa}\"\n}\n\nsource-source-files() {\n    # Test execution will source *.source.* files before it executes\n    # the real test code. The files will be sourced starting from the\n    # test suite (root) directory and ending up to the test directory,\n    # which enables overriding inherited functions and variables.\n    local src_file_dir=\"$1\"\n    local src_filepath\n    for src_filepath in \"$src_file_dir\"/*.source \"$src_file_dir\"/*.source.*; do\n        if ! [ -f \"$src_filepath\" ] || [[ \"$src_filepath\" == *\"~\" ]]; then\n            continue\n        fi\n        echo \"sourcing $src_filepath before running test code\"\n        source_libs=\"${source_libs}\"\"\nsource \\\"$src_filepath\\\"\n\"\n    done\n}\n\nexport-and-source-dir() {\n    local dir=\"$1\"\n    export-var-files \"$dir\"\n    export-vm-files \"$dir/vm-files\"\n    source-source-files \"$dir\"\n}\n\nif [ -z \"$TESTS_DIR\" ] || [ \"$TESTS_DIR\" == \"help\" ] || [ \"$TESTS_DIR\" == \"--help\" ]; then\n    usage\n    error \"missing TESTS_DIR\"\nfi\n\nif ! [ -d \"$TESTS_DIR\" ]; then\n    error \"bad TESTS_DIR: \\\"$TESTS_DIR\\\"\"\nfi\n\n# Find TESTS_DIR root by looking for POLICY_DIR/*.cfg. If TESTS_DIR was not the\n# root dir, then execute tests only under TESTS_DIR.\nroot_dir_glob=\"*.test-suite\"\n# shellcheck disable=SC2053\nif [[ \"$(basename \"$TESTS_DIR\")\" == $root_dir_glob ]]; then\n    TESTS_ROOT_DIR=\"$TESTS_DIR\"\nelif [[ \"$(basename \"$(realpath \"$TESTS_DIR\"/..)\")\" == $root_dir_glob ]]; then\n    TESTS_ROOT_DIR=$(realpath \"$TESTS_DIR/..\")\n    TESTS_POLICY_FILTER=$(basename \"${TESTS_DIR}\")\nelif [[ \"$(basename \"$(realpath \"$TESTS_DIR\"/../..)\")\" == $root_dir_glob ]]; then\n    TESTS_ROOT_DIR=$(realpath \"$TESTS_DIR/../..\")\n    TESTS_POLICY_FILTER=$(basename \"$(dirname \"${TESTS_DIR}\")\")\n    TESTS_TOPOLOGY_FILTER=$(basename \"${TESTS_DIR}\")\nelif [[ \"$(basename \"$(realpath \"$TESTS_DIR\"/../../..)\")\" == $root_dir_glob ]]; then\n    TESTS_ROOT_DIR=$(realpath \"$TESTS_DIR/../../..\")\n    TESTS_POLICY_FILTER=$(basename \"$(dirname \"$(dirname \"${TESTS_DIR}\")\")\")\n    TESTS_TOPOLOGY_FILTER=$(basename \"$(dirname \"${TESTS_DIR}\")\")\n    TESTS_TEST_FILTER=$(basename \"${TESTS_DIR}\")\nelse\n    error \"TESTS_DIR=\\\"$TESTS_DIR\\\" is invalid tests/policy/topology/test dir: *.cfg not found\"\nfi\n\necho \"Running tests matching:\"\necho \"    TESTS_ROOT_DIR=$TESTS_ROOT_DIR\"\necho \"    TESTS_POLICY_FILTER=$TESTS_POLICY_FILTER\"\necho \"    TESTS_TOPOLOGY_FILTER=$TESTS_TOPOLOGY_FILTER\"\necho \"    TESTS_TEST_FILTER=$TESTS_TEST_FILTER\"\n\ncleanup() {\n    rm -rf \"$summary_dir\"\n}\nsummary_dir=$(mktemp -d)\ntrap cleanup TERM EXIT QUIT\n\nsummary_file=\"$summary_dir/summary.txt\"\necho -n \"\" > \"$summary_file\"\n\nexport-and-source-dir \"$TESTS_ROOT_DIR\"\n\nfor POLICY_DIR in \"$TESTS_ROOT_DIR\"/*; do\n    if ! [ -d \"$POLICY_DIR\" ]; then\n        continue\n    fi\n    if ! [[ \"$(basename \"$POLICY_DIR\")\" =~ .*\"$TESTS_POLICY_FILTER\".* ]]; then\n        continue\n    fi\n    # Run exports in subshells so that variables exported for previous\n    # tests do not affect any other tests.\n    (\n        for CFG_FILE in \"$POLICY_DIR\"/*.cfg; do\n            if ! [ -f \"$CFG_FILE\" ]; then\n                continue\n            fi\n            export cri_resmgr_cfg=$CFG_FILE\n        done\n        export-and-source-dir \"$POLICY_DIR\"\n        for TOPOLOGY_DIR in \"$POLICY_DIR\"/*; do\n            if ! [ -d \"$TOPOLOGY_DIR\" ]; then\n                continue\n            fi\n            if ! [[ \"$(basename \"$TOPOLOGY_DIR\")\" =~ .*\"$TESTS_TOPOLOGY_FILTER\".* ]]; then\n                continue\n            fi\n            if [ \"$(basename \"$TOPOLOGY_DIR\")\" == \"vm-files\" ]; then\n                continue\n            fi\n            (\n                distro=${distro:=$DEFAULT_DISTRO}\n                export distro\n                # Create name for the vm.\n                # Needs topology, distro and container runtime stack.\n                k8scri=${k8scri:-\"cri-resmgr|containerd\"}\n                case \"${k8scri}\" in\n                    \"cri-resmgr|containerd\")\n                        criname=crirm-containerd\n                        ;;\n                    \"cri-resmgr|crio\")\n                        criname=crirm-crio\n                        ;;\n                    \"containerd\")\n                        criname=containerd\n                        ;;\n                    \"containerd&cri-resmgr\")\n                        criname=nrirm-containerd\n                        ;;\n                    \"crio\")\n                        criname=crio\n                        ;;\n                    \"crio&cri-resmgr\")\n                        criname=nrirm-crio\n                        ;;\n                    *)\n                        error \"unsupported k8scri: \\\"${k8scri}\\\"\"\n                        ;;\n                esac\n                vm=\"$(basename \"$TOPOLOGY_DIR\")-${distro}-${criname}\"\n                export vm\n                export-and-source-dir \"$TOPOLOGY_DIR\"\n                for TEST_DIR in \"$TOPOLOGY_DIR\"/*; do\n                    if ! [ -d \"$TEST_DIR\" ]; then\n                        continue\n                    fi\n                    if ! [[ \"$(basename \"$TEST_DIR\")\" =~ .*\"$TESTS_TEST_FILTER\".* ]]; then\n                        continue\n                    fi\n                    if [ \"$(basename \"$TEST_DIR\")\" == \"vm-files\" ]; then\n                        continue\n                    fi\n                    (\n                        export outdir=\"$TEST_DIR/output\"\n                        export-and-source-dir \"$TEST_DIR\"\n                        export code=\"${source_libs}\"\"\n${code}\"\n                        mkdir -p \"$outdir\"\n                        echo \"Run $(basename \"$TEST_DIR\")\"\n                        TEST_DIR=$TEST_DIR TOPOLOGY_DIR=$TOPOLOGY_DIR POLICY_DIR=$POLICY_DIR \\\n                            \"$RUN_SH\" test 2>&1 | tee \"$outdir/run.sh.output\"\n                        test_name=\"$(basename \"$POLICY_DIR\")/$(basename \"$TOPOLOGY_DIR\")/$(basename \"$TEST_DIR\")\"\n                        if grep -q \"Test verdict: PASS\" \"$outdir/run.sh.output\"; then\n                            echo \"PASS $test_name\" >> \"$summary_file\"\n                        elif grep -q \"Test verdict: FAIL\" \"$outdir/run.sh.output\"; then\n                            echo \"FAIL $test_name\" >> \"$summary_file\"\n                        else\n                            echo \"ERROR $test_name\" >> \"$summary_file\"\n                        fi\n                    )\n                done\n            )\n        done\n    )\ndone\n\necho \"\"\necho \"Tests summary:\"\ncat \"$summary_file\"\nif grep -q ERROR \"$summary_file\" || grep -q FAIL \"$summary_file\"; then\n    exit 1\nfi\n"
  },
  {
    "path": "test/functional/e2e_test.go",
    "content": "// Copyright 2020 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage e2e\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n\n\tresmgr \"github.com/intel/cri-resource-manager/pkg/cri/resource-manager\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/cache\"\n\t\"github.com/intel/cri-resource-manager/pkg/cri/resource-manager/kubernetes\"\n\t\"github.com/intel/cri-resource-manager/pkg/dump\"\n\t\"google.golang.org/grpc\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n\n\tlogger \"github.com/intel/cri-resource-manager/pkg/log\"\n)\n\nconst (\n\ttestDir = \"/tmp/cri-rm-test\"\n)\n\nfunc init() {\n\trate := logger.Rate{Limit: logger.Every(1 * time.Minute)}\n\tlogger.SetGrpcLogger(\"grpc\", &rate)\n\n\tif err := os.MkdirAll(testDir, 0700); err != nil {\n\t\tfmt.Printf(\"unable to create %q: %+v\\n\", testDir, err)\n\t}\n}\n\ntype testEnv struct {\n\tt           *testing.T\n\thandlers    map[string]interface{}\n\tclient      criv1.RuntimeServiceClient\n\tforceConfig string\n\tmgr         resmgr.ResourceManager\n\tcache       cache.Cache\n}\n\nfunc (env *testEnv) Run(name string, testFunction func(context.Context, *testEnv)) {\n\tt := env.t\n\toverriddenCriHandlers := env.handlers\n\n\tt.Helper()\n\tt.Run(name, func(t *testing.T) {\n\t\ttmpDir, err := os.MkdirTemp(testDir, \"requests-\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"unable to create temp directory: %+v\", err)\n\t\t}\n\t\tdefer os.RemoveAll(tmpDir)\n\n\t\tif err := flag.Set(\"runtime-socket\", filepath.Join(tmpDir, \"fakecri.sock\")); err != nil {\n\t\t\tt.Fatalf(\"unable to set runtime-socket\")\n\t\t}\n\t\tif err := flag.Set(\"image-socket\", filepath.Join(tmpDir, \"fakecri.sock\")); err != nil {\n\t\t\tt.Fatalf(\"unable to set image-socket\")\n\t\t}\n\t\tif err := flag.Set(\"relay-socket\", filepath.Join(tmpDir, \"relay.sock\")); err != nil {\n\t\t\tt.Fatalf(\"unable to set relay-socket\")\n\t\t}\n\t\tif err := flag.Set(\"relay-dir\", filepath.Join(tmpDir, \"relaystorage\")); err != nil {\n\t\t\tt.Fatalf(\"unable to set relay-dir\")\n\t\t}\n\t\tif err := flag.Set(\"agent-socket\", filepath.Join(tmpDir, \"agent.sock\")); err != nil {\n\t\t\tt.Fatalf(\"unable to set agent-socket\")\n\t\t}\n\t\tif err := flag.Set(\"config-socket\", filepath.Join(tmpDir, \"config.sock\")); err != nil {\n\t\t\tt.Fatalf(\"unable to set config-socket\")\n\t\t}\n\t\tif err := flag.Set(\"allow-untested-runtimes\", \"true\"); err != nil {\n\t\t\tt.Fatalf(\"unable to allow untested runtimes: %v\", err)\n\t\t}\n\n\t\tif env.forceConfig != \"\" {\n\t\t\tpath := filepath.Join(tmpDir, \"forcedconfig.cfg\")\n\t\t\tif err := os.WriteFile(path, []byte(env.forceConfig), 0644); err != nil {\n\t\t\t\tt.Fatalf(\"failed to create configuration file %s: %v\", path, err)\n\t\t\t}\n\t\t\tif err := flag.Set(\"force-config\", path); err != nil {\n\t\t\t\tt.Fatalf(\"unable to set force-config\")\n\t\t\t}\n\t\t}\n\n\t\tflag.Parse()\n\n\t\tfakeCri := newFakeCriServer(t, filepath.Join(tmpDir, \"fakecri.sock\"), overriddenCriHandlers)\n\t\tdefer fakeCri.stop()\n\n\t\tresMgr, err := resmgr.NewResourceManager()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"unable to create resource manager: %+v\", err)\n\t\t}\n\t\tif err := resMgr.Start(); err != nil {\n\t\t\tt.Fatalf(\"unable to start resource manager: %+v\", err)\n\t\t}\n\t\tdefer resMgr.Stop()\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\t\tdefer cancel()\n\n\t\tconn, err := grpc.DialContext(ctx, filepath.Join(tmpDir, \"relay.sock\"), grpc.WithInsecure(), grpc.WithBlock(),\n\t\t\tgrpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {\n\t\t\t\tif deadline, ok := ctx.Deadline(); ok {\n\t\t\t\t\treturn net.DialTimeout(\"unix\", addr, time.Until(deadline))\n\t\t\t\t}\n\t\t\t\treturn net.DialTimeout(\"unix\", addr, 0)\n\t\t\t}),\n\t\t)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"unable to connect to relay: %+v\", err)\n\t\t}\n\t\tdefer conn.Close()\n\n\t\tclient := criv1.NewRuntimeServiceClient(conn)\n\n\t\tenv.client = client\n\t\tenv.mgr = resMgr\n\t\tenv.cache = resMgr.GetCache()\n\n\t\ttestFunction(ctx, env)\n\n\t\t// until pkg/log fixes gets merged: wait until pkg/dump is done with\n\t\t// logging before we run next test (and consequently do a reconfig)\n\t\tdump.Sync()\n\t})\n}\n\nfunc TestListPodSandbox(t *testing.T) {\n\ttcases := []struct {\n\t\tname         string\n\t\tpods         []*criv1.PodSandbox\n\t\texpectedPods int\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t},\n\t\t{\n\t\t\tname:         \"list one pod\",\n\t\t\tpods:         []*criv1.PodSandbox{{}},\n\t\t\texpectedPods: 1,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tcriHandlers := map[string]interface{}{\n\t\t\t\"ListPodSandbox\": func(*fakeCriServer, context.Context, *criv1.ListPodSandboxRequest) (*criv1.ListPodSandboxResponse, error) {\n\t\t\t\treturn &criv1.ListPodSandboxResponse{\n\t\t\t\t\tItems: tc.pods,\n\t\t\t\t}, nil\n\t\t\t},\n\t\t}\n\t\tenv := &testEnv{\n\t\t\tt:        t,\n\t\t\thandlers: criHandlers,\n\t\t}\n\t\tenv.Run(tc.name, func(ctx context.Context, env *testEnv) {\n\t\t\tt := env.t\n\t\t\tclient := env.client\n\t\t\tresp, err := client.ListPodSandbox(ctx, &criv1.ListPodSandboxRequest{})\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Unexpected error: %+v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif len(resp.Items) != tc.expectedPods {\n\t\t\t\tt.Errorf(\"Expected %d pods, got %d\", tc.expectedPods, len(resp.Items))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestListContainers(t *testing.T) {\n\ttcases := []struct {\n\t\tname               string\n\t\tcontainers         []*criv1.Container\n\t\texpectedContainers int\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t},\n\t\t{\n\t\t\tname:               \"list one container\",\n\t\t\tcontainers:         []*criv1.Container{{}},\n\t\t\texpectedContainers: 1,\n\t\t},\n\t}\n\tfor _, tc := range tcases {\n\t\tcriHandlers := map[string]interface{}{\n\t\t\t\"ListContainers\": func(*fakeCriServer, context.Context, *criv1.ListContainersRequest) (*criv1.ListContainersResponse, error) {\n\t\t\t\treturn &criv1.ListContainersResponse{\n\t\t\t\t\tContainers: tc.containers,\n\t\t\t\t}, nil\n\t\t\t},\n\t\t}\n\t\tenv := &testEnv{\n\t\t\tt:        t,\n\t\t\thandlers: criHandlers,\n\t\t}\n\t\tenv.Run(tc.name, func(ctx context.Context, env *testEnv) {\n\t\t\tt := env.t\n\t\t\tclient := env.client\n\t\t\tresp, err := client.ListContainers(ctx, &criv1.ListContainersRequest{})\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Unexpected error: %+v\", err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif len(resp.Containers) != tc.expectedContainers {\n\t\t\t\tt.Errorf(\"Expected %d pods, got %d\", tc.expectedContainers, len(resp.Containers))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLingeringPodCleanup(t *testing.T) {\n\tcfg := `\npolicy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\n`\n\ttcases := []struct {\n\t\tname         string\n\t\treqs         []*criv1.RunPodSandboxRequest\n\t\texpectedPods int\n\t}{\n\t\t{\n\t\t\tname: \"create Pod #1\",\n\t\t\treqs: []*criv1.RunPodSandboxRequest{\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t},\n\t\t\texpectedPods: 1,\n\t\t},\n\t\t{\n\t\t\tname: \"create Pods #1 and #2\",\n\t\t\treqs: []*criv1.RunPodSandboxRequest{\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#2\", \"UID#2\", \"\", nil, nil, \"\"),\n\t\t\t},\n\t\t\texpectedPods: 2,\n\t\t},\n\t\t{\n\t\t\tname: \"create Pods #1, #2, and #3\",\n\t\t\treqs: []*criv1.RunPodSandboxRequest{\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#2\", \"UID#2\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#3\", \"UID#3\", \"\", nil, nil, \"\"),\n\t\t\t},\n\t\t\texpectedPods: 3,\n\t\t},\n\t\t{\n\t\t\tname: \"create Pods #1, #2, #3, #4, '1, '2, '3\",\n\t\t\treqs: []*criv1.RunPodSandboxRequest{\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#2\", \"UID#2\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#3\", \"UID#3\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#4\", \"UID#4\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#2\", \"UID'2\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#3\", \"UID'3\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#2\", \"UID#2\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#3\", \"UID#3\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID'1\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#2\", \"UID'2\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#3\", \"UID'3\", \"\", nil, nil, \"\"),\n\t\t\t\tcreatePodRequest(\"Pod#4\", \"UID#4\", \"\", nil, nil, \"\"),\n\t\t\t},\n\t\t\texpectedPods: 7,\n\t\t},\n\t}\n\n\tnumPods := 0\n\tfor _, tc := range tcases {\n\t\tcriHandlers := map[string]interface{}{\n\t\t\t\"RunPodSandbox\": func(*fakeCriServer, context.Context, *criv1.RunPodSandboxRequest) (*criv1.RunPodSandboxResponse, error) {\n\t\t\t\tnumPods++\n\t\t\t\treturn &criv1.RunPodSandboxResponse{\n\t\t\t\t\tPodSandboxId: fmt.Sprintf(\"Pod#%d\", numPods),\n\t\t\t\t}, nil\n\t\t\t},\n\t\t}\n\t\tenv := &testEnv{\n\t\t\tt:           t,\n\t\t\thandlers:    criHandlers,\n\t\t\tforceConfig: cfg,\n\t\t}\n\t\tenv.Run(tc.name, func(ctx context.Context, env *testEnv) {\n\t\t\tt := env.t\n\t\t\tclient := env.client\n\t\t\tcache := env.cache\n\t\t\tfor _, req := range tc.reqs {\n\t\t\t\t_, err := client.RunPodSandbox(ctx, req)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"failed to create pod %+v: %v\", req, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tpods := cache.GetPods()\n\t\t\tif len(pods) != tc.expectedPods {\n\t\t\t\tt.Errorf(\"expected %d pods in cache, got %d (%v)\", tc.expectedPods, len(pods), pods)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLingeringContainerCleanup(t *testing.T) {\n\tcfg := `\npolicy:\n  Active: topology-aware\n  ReservedResources:\n    CPU: 750m\n`\n\ttype pod struct {\n\t\tUID string\n\t\tID  string\n\t\treq *criv1.RunPodSandboxRequest\n\t}\n\n\ttype container struct {\n\t\tpod    string\n\t\tname   string\n\t\texpect int\n\t\treq    *criv1.CreateContainerRequest\n\t\tID     string\n\t}\n\n\ttcases := []struct {\n\t\tname       string\n\t\tpods       []*criv1.RunPodSandboxRequest\n\t\tcontainers []*container\n\t}{\n\t\t{\n\t\t\tname: \"create containers per one pod\",\n\t\t\tpods: []*criv1.RunPodSandboxRequest{\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t},\n\t\t\tcontainers: []*container{\n\t\t\t\t{pod: \"UID#1\", name: \"Container#1\", expect: 1},\n\t\t\t\t{pod: \"UID#1\", name: \"Container#2\", expect: 2},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"create lingering containers per one pod\",\n\t\t\tpods: []*criv1.RunPodSandboxRequest{\n\t\t\t\tcreatePodRequest(\"Pod#1\", \"UID#1\", \"\", nil, nil, \"\"),\n\t\t\t},\n\t\t\tcontainers: []*container{\n\t\t\t\t{pod: \"UID#1\", name: \"Container#1\", expect: 1},\n\t\t\t\t{pod: \"UID#1\", name: \"Container#2\", expect: 2},\n\t\t\t\t{pod: \"UID#1\", name: \"Container#3\", expect: 3},\n\t\t\t\t{pod: \"UID#1\", name: \"Container#3\", expect: 3},\n\t\t\t\t{pod: \"UID#1\", name: \"Container#2\", expect: 3},\n\t\t\t\t{pod: \"UID#1\", name: \"Container#1\", expect: 3},\n\t\t\t},\n\t\t},\n\t}\n\n\tnumPods := 0\n\tnumContainers := 0\n\tfor _, tc := range tcases {\n\t\tcriHandlers := map[string]interface{}{\n\t\t\t\"RunPodSandbox\": func(*fakeCriServer, context.Context, *criv1.RunPodSandboxRequest) (*criv1.RunPodSandboxResponse, error) {\n\t\t\t\tnumPods++\n\t\t\t\treturn &criv1.RunPodSandboxResponse{\n\t\t\t\t\tPodSandboxId: fmt.Sprintf(\"Pod#%d\", numPods),\n\t\t\t\t}, nil\n\t\t\t},\n\t\t\t\"CreateContainer\": func(*fakeCriServer, context.Context, *criv1.CreateContainerRequest) (*criv1.CreateContainerResponse, error) {\n\t\t\t\tnumContainers++\n\t\t\t\treturn &criv1.CreateContainerResponse{\n\t\t\t\t\tContainerId: fmt.Sprintf(\"Container#%d\", numContainers),\n\t\t\t\t}, nil\n\t\t\t},\n\t\t}\n\t\tenv := &testEnv{\n\t\t\tt:           t,\n\t\t\thandlers:    criHandlers,\n\t\t\tforceConfig: cfg,\n\t\t}\n\t\tenv.Run(tc.name, func(ctx context.Context, env *testEnv) {\n\t\t\tt := env.t\n\t\t\tclient := env.client\n\t\t\tcache := env.cache\n\t\t\tpods := map[string]*pod{}\n\n\t\t\tfor _, req := range tc.pods {\n\t\t\t\trpl, err := client.RunPodSandbox(ctx, req)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"failed to create pod %+v: %v\", req, err)\n\t\t\t\t} else {\n\t\t\t\t\tid := rpl.PodSandboxId\n\t\t\t\t\tuid := req.Config.Metadata.Uid\n\t\t\t\t\tpods[uid] = &pod{\n\t\t\t\t\t\tUID: uid,\n\t\t\t\t\t\tID:  id,\n\t\t\t\t\t\treq: req,\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, c := range tc.containers {\n\t\t\t\tpod, ok := pods[c.pod]\n\t\t\t\tif !ok {\n\t\t\t\t\tt.Errorf(\"failed to find pod by UID %s\", c.pod)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tc.req = createContainerRequest(pod.ID, c.name, pod.req)\n\t\t\t\trpl, err := client.CreateContainer(ctx, c.req)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"failed to create container %+v: %v\", c.req, err)\n\t\t\t\t} else {\n\t\t\t\t\tc.ID = rpl.ContainerId\n\t\t\t\t\tcached := cache.GetContainers()\n\t\t\t\t\tif len(cached) != c.expect {\n\t\t\t\t\t\tt.Errorf(\"pod %s, container %s: expected %d containers in cache, got %d\",\n\t\t\t\t\t\t\tc.pod, c.name, c.expect, len(cached))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc createPodRequest(name, uid, namespace string,\n\tlabels, annotations map[string]string,\n\tcgroupParent string) *criv1.RunPodSandboxRequest {\n\tif namespace == \"\" {\n\t\tnamespace = \"default\"\n\t}\n\tif labels == nil {\n\t\tlabels = map[string]string{}\n\t}\n\tlabels[kubernetes.PodUIDLabel] = uid\n\treturn &criv1.RunPodSandboxRequest{\n\t\tConfig: &criv1.PodSandboxConfig{\n\t\t\tMetadata: &criv1.PodSandboxMetadata{\n\t\t\t\tName:      name,\n\t\t\t\tUid:       uid,\n\t\t\t\tNamespace: namespace,\n\t\t\t},\n\t\t\tLabels:      labels,\n\t\t\tAnnotations: annotations,\n\t\t\tLinux: &criv1.LinuxPodSandboxConfig{\n\t\t\t\tCgroupParent: cgroupParent,\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc createContainerRequest(podID, name string,\n\tpodReq *criv1.RunPodSandboxRequest) *criv1.CreateContainerRequest {\n\treturn &criv1.CreateContainerRequest{\n\t\tPodSandboxId: podID,\n\t\tConfig: &criv1.ContainerConfig{\n\t\t\tMetadata: &criv1.ContainerMetadata{\n\t\t\t\tName: name,\n\t\t\t},\n\t\t\tLinux: &criv1.LinuxContainerConfig{},\n\t\t},\n\t\tSandboxConfig: podReq.Config,\n\t}\n}\n"
  },
  {
    "path": "test/functional/fake_cri_server_test.go",
    "content": "// Copyright 2019 Intel Corporation. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage e2e\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/intel/cri-resource-manager/pkg/utils\"\n\t\"google.golang.org/grpc\"\n\tcriv1 \"k8s.io/cri-api/pkg/apis/runtime/v1\"\n)\n\nconst (\n\tfakeKubeAPIVersion    = \"0.1.0\"\n\tfakeRuntimeName       = \"fake-CRI-runtime\"\n\tfakeRuntimeVersion    = \"v0.0.0\"\n\tfakeRuntimeAPIVersion = \"v1\"\n)\n\ntype fakeCriServer struct {\n\tt            *testing.T\n\tsocket       string\n\tgrpcServer   *grpc.Server\n\tfakeHandlers map[string]interface{}\n}\n\nfunc newFakeCriServer(t *testing.T, socket string, fakeHandlers map[string]interface{}) *fakeCriServer {\n\tt.Helper()\n\n\tif !filepath.IsAbs(socket) {\n\t\tt.Fatalf(\"invalid socket %q, absolute path expected\", socket)\n\t}\n\n\tif err := os.MkdirAll(filepath.Dir(socket), 0700); err != nil {\n\t\tt.Fatalf(\"failed to create directory for socket %q: %v\", socket, err)\n\t}\n\n\tsrv := &fakeCriServer{\n\t\tt:            t,\n\t\tsocket:       socket,\n\t\tgrpcServer:   grpc.NewServer(),\n\t\tfakeHandlers: fakeHandlers,\n\t}\n\n\tcriv1.RegisterRuntimeServiceServer(srv.grpcServer, srv)\n\tcriv1.RegisterImageServiceServer(srv.grpcServer, srv)\n\n\tlis, err := net.Listen(\"unix\", socket)\n\tif err != nil {\n\t\tif ls, err := utils.IsListeningSocket(socket); ls || err != nil {\n\t\t\tt.Fatalf(\"failed to create fake server: socket %s already exists\", socket)\n\t\t}\n\t\tos.Remove(socket)\n\t\tlis, err = net.Listen(\"unix\", socket)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to create fake server on socket %q: %v\", socket, err)\n\t\t}\n\t}\n\n\tgo func() {\n\t\tif err := srv.grpcServer.Serve(lis); err != nil {\n\t\t\tfmt.Printf(\"unable to start gRPC server: %+v\\n\", err)\n\t\t}\n\t}()\n\n\tif err := utils.WaitForServer(socket, time.Second); err != nil {\n\t\tt.Fatalf(\"starting fake CRI server failed: %v\", err)\n\t}\n\n\treturn srv\n}\n\nfunc (s *fakeCriServer) stop() {\n\ts.t.Helper()\n\ts.grpcServer.Stop()\n\tos.Remove(s.socket)\n}\n\nfunc (s *fakeCriServer) callHandler(ctx context.Context, request interface{}, defaultHandler interface{}) (interface{}, error) {\n\tvar err error\n\n\tpc, _, _, _ := runtime.Caller(1)\n\tnameFull := runtime.FuncForPC(pc).Name()\n\tnameEnd := filepath.Ext(nameFull)\n\tname := strings.TrimPrefix(nameEnd, \".\")\n\n\thandler, found := s.fakeHandlers[name]\n\tif !found {\n\t\tif defaultHandler == nil {\n\t\t\tmethod := reflect.ValueOf(s).MethodByName(name)\n\t\t\treturnType := method.Type().Out(0)\n\t\t\treturn reflect.New(returnType).Elem().Interface(), fmt.Errorf(\"%s() not implemented\", name)\n\t\t}\n\n\t\thandler = defaultHandler\n\t}\n\n\tin := make([]reflect.Value, 3)\n\tin[0] = reflect.ValueOf(s)\n\tin[1] = reflect.ValueOf(ctx)\n\tin[2] = reflect.ValueOf(request)\n\tout := reflect.ValueOf(handler).Call(in)\n\n\tif !out[1].IsNil() {\n\t\terr = out[1].Interface().(error)\n\t}\n\n\treturn out[0].Interface(), err\n}\n\n// Implementation of criv1.RuntimeServiceServer\n\nfunc (s *fakeCriServer) Version(ctx context.Context, req *criv1.VersionRequest) (*criv1.VersionResponse, error) {\n\tresponse, err := s.callHandler(ctx, req,\n\t\tfunc(*fakeCriServer, context.Context, *criv1.VersionRequest) (*criv1.VersionResponse, error) {\n\t\t\treturn &criv1.VersionResponse{\n\t\t\t\tVersion:           fakeKubeAPIVersion,\n\t\t\t\tRuntimeName:       fakeRuntimeName,\n\t\t\t\tRuntimeVersion:    fakeRuntimeVersion,\n\t\t\t\tRuntimeApiVersion: fakeRuntimeAPIVersion,\n\t\t\t}, nil\n\t\t},\n\t)\n\treturn response.(*criv1.VersionResponse), err\n}\n\nfunc (s *fakeCriServer) RunPodSandbox(ctx context.Context, req *criv1.RunPodSandboxRequest) (*criv1.RunPodSandboxResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.RunPodSandboxResponse), err\n}\n\nfunc (s *fakeCriServer) StopPodSandbox(ctx context.Context, req *criv1.StopPodSandboxRequest) (*criv1.StopPodSandboxResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.StopPodSandboxResponse), err\n}\n\nfunc (s *fakeCriServer) RemovePodSandbox(ctx context.Context, req *criv1.RemovePodSandboxRequest) (*criv1.RemovePodSandboxResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.RemovePodSandboxResponse), err\n}\n\nfunc (s *fakeCriServer) PodSandboxStatus(ctx context.Context, req *criv1.PodSandboxStatusRequest) (*criv1.PodSandboxStatusResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.PodSandboxStatusResponse), err\n}\n\nfunc (s *fakeCriServer) ListPodSandbox(ctx context.Context, req *criv1.ListPodSandboxRequest) (*criv1.ListPodSandboxResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, func(*fakeCriServer, context.Context, *criv1.ListPodSandboxRequest) (*criv1.ListPodSandboxResponse, error) {\n\t\treturn &criv1.ListPodSandboxResponse{}, nil\n\t})\n\treturn response.(*criv1.ListPodSandboxResponse), err\n}\n\nfunc (s *fakeCriServer) CreateContainer(ctx context.Context, req *criv1.CreateContainerRequest) (*criv1.CreateContainerResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.CreateContainerResponse), err\n}\n\nfunc (s *fakeCriServer) StartContainer(ctx context.Context, req *criv1.StartContainerRequest) (*criv1.StartContainerResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.StartContainerResponse), err\n}\n\nfunc (s *fakeCriServer) StopContainer(ctx context.Context, req *criv1.StopContainerRequest) (*criv1.StopContainerResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.StopContainerResponse), err\n}\n\nfunc (s *fakeCriServer) RemoveContainer(ctx context.Context, req *criv1.RemoveContainerRequest) (*criv1.RemoveContainerResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.RemoveContainerResponse), err\n}\n\nfunc (s *fakeCriServer) ListContainers(ctx context.Context, req *criv1.ListContainersRequest) (*criv1.ListContainersResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, func(*fakeCriServer, context.Context, *criv1.ListContainersRequest) (*criv1.ListContainersResponse, error) {\n\t\treturn &criv1.ListContainersResponse{}, nil\n\t})\n\treturn response.(*criv1.ListContainersResponse), err\n}\n\nfunc (s *fakeCriServer) ContainerStatus(ctx context.Context, req *criv1.ContainerStatusRequest) (*criv1.ContainerStatusResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ContainerStatusResponse), err\n}\n\nfunc (s *fakeCriServer) UpdateContainerResources(ctx context.Context, req *criv1.UpdateContainerResourcesRequest) (*criv1.UpdateContainerResourcesResponse, error) {\n\tresponse, err := s.callHandler(ctx, req,\n\t\tfunc(*fakeCriServer, context.Context, *criv1.UpdateContainerResourcesRequest) (*criv1.UpdateContainerResourcesResponse, error) {\n\t\t\treturn &criv1.UpdateContainerResourcesResponse{}, nil\n\t\t},\n\t)\n\treturn response.(*criv1.UpdateContainerResourcesResponse), err\n}\n\nfunc (s *fakeCriServer) ReopenContainerLog(ctx context.Context, req *criv1.ReopenContainerLogRequest) (*criv1.ReopenContainerLogResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ReopenContainerLogResponse), err\n}\n\nfunc (s *fakeCriServer) ExecSync(ctx context.Context, req *criv1.ExecSyncRequest) (*criv1.ExecSyncResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ExecSyncResponse), err\n}\n\nfunc (s *fakeCriServer) Exec(ctx context.Context, req *criv1.ExecRequest) (*criv1.ExecResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ExecResponse), err\n}\n\nfunc (s *fakeCriServer) Attach(ctx context.Context, req *criv1.AttachRequest) (*criv1.AttachResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.AttachResponse), err\n}\n\nfunc (s *fakeCriServer) PortForward(ctx context.Context, req *criv1.PortForwardRequest) (*criv1.PortForwardResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.PortForwardResponse), err\n}\n\nfunc (s *fakeCriServer) ContainerStats(ctx context.Context, req *criv1.ContainerStatsRequest) (*criv1.ContainerStatsResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ContainerStatsResponse), err\n}\n\nfunc (s *fakeCriServer) ListContainerStats(ctx context.Context, req *criv1.ListContainerStatsRequest) (*criv1.ListContainerStatsResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ListContainerStatsResponse), err\n}\n\nfunc (s *fakeCriServer) PodSandboxStats(ctx context.Context, req *criv1.PodSandboxStatsRequest) (*criv1.PodSandboxStatsResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.PodSandboxStatsResponse), err\n}\n\nfunc (s *fakeCriServer) ListPodSandboxStats(ctx context.Context, req *criv1.ListPodSandboxStatsRequest) (*criv1.ListPodSandboxStatsResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ListPodSandboxStatsResponse), err\n}\n\nfunc (s *fakeCriServer) UpdateRuntimeConfig(ctx context.Context, req *criv1.UpdateRuntimeConfigRequest) (*criv1.UpdateRuntimeConfigResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.UpdateRuntimeConfigResponse), err\n}\n\nfunc (s *fakeCriServer) Status(ctx context.Context, req *criv1.StatusRequest) (*criv1.StatusResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.StatusResponse), err\n}\n\nfunc (s *fakeCriServer) CheckpointContainer(ctx context.Context, req *criv1.CheckpointContainerRequest) (*criv1.CheckpointContainerResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.CheckpointContainerResponse), err\n}\n\nfunc (s *fakeCriServer) GetContainerEvents(_ *criv1.GetEventsRequest, _ criv1.RuntimeService_GetContainerEventsServer) error {\n\treturn nil\n}\n\nfunc (s *fakeCriServer) ListMetricDescriptors(ctx context.Context, req *criv1.ListMetricDescriptorsRequest) (*criv1.ListMetricDescriptorsResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ListMetricDescriptorsResponse), err\n}\n\nfunc (s *fakeCriServer) ListPodSandboxMetrics(ctx context.Context, req *criv1.ListPodSandboxMetricsRequest) (*criv1.ListPodSandboxMetricsResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ListPodSandboxMetricsResponse), err\n}\n\nfunc (s *fakeCriServer) RuntimeConfig(ctx context.Context, req *criv1.RuntimeConfigRequest) (*criv1.RuntimeConfigResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.RuntimeConfigResponse), err\n}\n\n// Implementation of criv1.ImageServiceServer\n\nfunc (s *fakeCriServer) ListImages(ctx context.Context, req *criv1.ListImagesRequest) (*criv1.ListImagesResponse, error) {\n\tresponse, err := s.callHandler(ctx, req,\n\t\tfunc(*fakeCriServer, context.Context, *criv1.ListImagesRequest) (*criv1.ListImagesResponse, error) {\n\t\t\treturn &criv1.ListImagesResponse{}, nil\n\t\t},\n\t)\n\treturn response.(*criv1.ListImagesResponse), err\n}\n\nfunc (s *fakeCriServer) ImageStatus(ctx context.Context, req *criv1.ImageStatusRequest) (*criv1.ImageStatusResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ImageStatusResponse), err\n}\n\nfunc (s *fakeCriServer) PullImage(ctx context.Context, req *criv1.PullImageRequest) (*criv1.PullImageResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.PullImageResponse), err\n}\n\nfunc (s *fakeCriServer) RemoveImage(ctx context.Context, req *criv1.RemoveImageRequest) (*criv1.RemoveImageResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.RemoveImageResponse), err\n}\n\nfunc (s *fakeCriServer) ImageFsInfo(ctx context.Context, req *criv1.ImageFsInfoRequest) (*criv1.ImageFsInfoResponse, error) {\n\tresponse, err := s.callHandler(ctx, req, nil)\n\treturn response.(*criv1.ImageFsInfoResponse), err\n}\n"
  }
]