[
  {
    "path": ".dockerignore",
    "content": "dist/*.aci\ndist/*.docker\ndist/*.tar.gz\n.git\nvendor\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE.md",
    "content": "<!--- Provide a general summary of the issue in the Title above -->\n\n## Expected Behavior\n<!--- If you're describing a bug, tell us what should happen -->\n<!--- If you're suggesting a change/improvement, tell us how it should work -->\n\n## Current Behavior\n<!--- If describing a bug, tell us what happens instead of the expected behavior -->\n<!--- If suggesting a change/improvement, explain the difference from current behavior -->\n\n## Possible Solution\n<!--- Not obligatory, but suggest a fix/reason for the bug, -->\n<!--- or ideas how to implement the addition or change -->\n\n## Steps to Reproduce (for bugs)\n<!--- Provide a link to a live example, or an unambiguous set of steps to -->\n<!--- reproduce this bug. Include code to reproduce, if relevant -->\n1.\n2.\n3.\n4.\n\n## Context\n<!--- How has this issue affected you? What are you trying to accomplish? -->\n<!--- Providing context helps us come up with a solution that is most useful in the real world -->\n\n## Your Environment\n<!--- Include as many relevant details about the environment you experienced the bug in -->\n* Flannel version:\n* Backend used (e.g. vxlan or udp):\n* Etcd version:\n* Kubernetes version (if used):\n* Operating System and version:\n* Link to your project (optional):\n\n"
  },
  {
    "path": ".github/PULL_REQUEST_TEMPLATE.md",
    "content": "## Description\n<!-- A few sentences describing the overall goals of the pull request's commits. \nPlease include \n- the type of fix - (e.g. bug fix, new feature, documentation)\n- some details on _why_ this PR should be merged\n- the details of the testing you've done on it (both manual and automated)\n- which components are affected by this PR\n-->\n\n## Todos\n- [ ] Tests\n- [ ] Documentation\n- [ ] Release note\n\n## Release Note\n<!-- Writing a release note:\n- By default, no release note action is required.\n- If you're unsure whether or not your PR needs a note, ask your reviewer for guidance.\n- If this PR requires a release note, update the block below to include a concise note describing\n  the change and any important impacts this PR may have.\n-->\n\n```release-note\nNone required\n```\n"
  },
  {
    "path": ".github/dependabot.yml",
    "content": "# To get started with Dependabot version updates, you'll need to specify which\n# package ecosystems to update and where the package manifests are located.\n# Please see the documentation for all configuration options:\n# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file\n\nversion: 2\nupdates:\n  - package-ecosystem: \"docker\" \n    directory: \"images\" \n    schedule:\n      interval: \"weekly\"\n  - package-ecosystem: \"gomod\" \n    directory: \"/\" \n    schedule:\n      interval: \"weekly\"\n    groups:\n      k8s:\n        patterns: [ \"k8s.io/*\", \"sigs.k8s.io/*\" ]\n        update-types: [ \"major\", \"minor\", \"patch\" ]\n      etcd:\n        patterns: [ \"go.etcd.io/*\" ]\n        update-types: [ \"major\", \"minor\", \"patch\" ]\n      tencent:\n        patterns: [ \"github.com/tencentcloud/*\" ]\n        update-types: [ \"major\", \"minor\", \"patch\" ]\n      other-go-modules:\n        patterns: [ \"*\" ]\n        exclude-patterns: \n          - \"k8s.io/*\"\n          - \"sigs.k8s.io/*\"\n          - \"go.etcd.io/*\"\n          - \"github.com/tencentcloud/*\"\n  - package-ecosystem: \"github-actions\" \n    directory: \"/\"\n    schedule:\n      interval: \"weekly\"\n\n  - package-ecosystem: docker\n    directory: /e2e\n    schedule:\n      interval: daily\n\n  - package-ecosystem: docker\n    directory: /images\n    schedule:\n      interval: daily\n\n  - package-ecosystem: docker\n    directory: /images/iperf3\n    schedule:\n      interval: daily\n"
  },
  {
    "path": ".github/stale.yml",
    "content": "# Number of days of inactivity before an issue becomes stale\ndaysUntilStale: 180\n# Number of days of inactivity before a stale issue is closed\ndaysUntilClose: 21\n# Issues with these labels will never be considered stale\nexemptLabels:\n  - pinned\n  - security\n# Label to use when marking an issue as stale\nstaleLabel: wontfix\n# Comment to post when marking an issue as stale. Set to `false` to disable\nmarkComment: >\n  This issue has been automatically marked as stale because it has not had\n  recent activity. It will be closed if no further activity occurs. Thank you\n  for your contributions.\n# Comment to post when closing a stale issue. Set to `false` to disable\ncloseComment: false\n"
  },
  {
    "path": ".github/workflows/build.yaml",
    "content": "name: build flannel\n\non: pull_request\n\nenv:\n  GO_VERSION: \"1.24\"\n  LINUX_ARCHES: \"amd64 arm arm64 s390x ppc64le riscv64\"\n  REPOSITORY: flannel/flannel\n\npermissions:\n  contents: read\n\njobs:\n  build-images:\n    runs-on: ubuntu-latest\n\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: set tag \n        run: echo \"GIT_TAG=$(git describe --tags --always)\" >> $GITHUB_ENV\n\n      - name: Set up Go 1.x\n        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: ${{ env.GO_VERSION }}\n\n      - name: go mod vendor\n        run: go mod vendor\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0\n\n      - name: Docker meta\n        id: meta\n        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf #v6.0.0\n        with:\n          images: ${{ env.REPOSITORY }}\n          flavor: latest=false\n          tags: |\n            type=ref, event=branch\n\n      - name: Build multi-arch Docker image\n        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 #v6.15.0\n        with:\n          context: .\n          file: images/Dockerfile\n          push: false\n          platforms: linux/amd64,linux/arm64,linux/arm,linux/s390x,linux/ppc64le,linux/riscv64\n          tags: ${{ steps.meta.outputs.tags }}\n          build-args: TAG=${{ env.GIT_TAG }}\n\n      - name: build for windows\n        run: make dist/flanneld.exe\n  \n  "
  },
  {
    "path": ".github/workflows/codeql-analysis.yml",
    "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n  pull_request:\n    # The branches below must be a subset of the branches above\n    branches: [ \"master\" ]\n  schedule:\n    - cron: '30 20 * * 0'\n\nenv:\n  GO_VERSION: \"1.24\"\n\npermissions:\n  contents: read\n\njobs:\n  analyze:\n    name: Analyze\n    runs-on: ubuntu-latest\n    permissions:\n      actions: read\n      contents: read\n      security-events: write\n\n    strategy:\n      fail-fast: false\n      matrix:\n        language: [ 'go' ]\n        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]\n        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support\n\n    steps:\n    - name: Checkout repository\n      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n    - name: Set up Go 1.x\n      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n      with:\n        go-version: ${{ env.GO_VERSION }}\n\n    # Initializes the CodeQL tools for scanning.\n    - name: Initialize CodeQL\n      uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 #v4.32.6\n      with:\n        languages: ${{ matrix.language }}\n        # If you wish to specify custom queries, you can do so here or in a config file.\n        # By default, queries listed here will override any specified in a config file.\n        # Prefix the list here with \"+\" to use these queries and those in the config file.\n        \n        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs\n        # queries: security-extended,security-and-quality\n\n        \n\n    # ℹ️ Command-line programs to run using the OS shell.\n    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun\n\n    #   If the Autobuild fails above, remove it and uncomment the following three lines. \n    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.\n\n    - run: |\n       echo \"Run, Build Application using script\"\n       make dist/flanneld\n\n    - name: Perform CodeQL Analysis\n      uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 #v4.32.6\n"
  },
  {
    "path": ".github/workflows/e2eTests.yaml",
    "content": "name: e2e tests for flannel\n\non: pull_request\n\npermissions:\n  contents: read\n\njobs:\n  e2e-test:\n    name: test\n    runs-on: ubuntu-latest\n    timeout-minutes: 90\n    steps:\n    - name: Set up Go 1.x\n      uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n      with:\n        go-version: ^1.24\n\n    - name: Check out code into the Go module directory\n      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n    - name: set up modules\n      run: sudo modprobe br_netfilter overlay\n\n    - name: Run tests\n      id: testing\n      continue-on-error: true\n      run: git fetch --unshallow --all --tags && make test 2>&1 > errors.txt\n\n    - name: Show additional logs\n      if: steps.testing.outcome != 'success'\n      run: |\n            cat errors.txt\n            exit 1\n"
  },
  {
    "path": ".github/workflows/golangci-lint.yaml",
    "content": "name: run golangci-lint\n\non: pull_request\n\npermissions:\n  contents: read\n\njobs:\n  golangci:\n    permissions:\n      contents: read  # for actions/checkout to fetch code\n      pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests\n    name: lint\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n      - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: \"1.24\"\n          cache: false\n      - name: golangci-lint\n        uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 #v9.2.0\n        with:\n          version: v2.7.2\n          args: \"--timeout=5m\"\n"
  },
  {
    "path": ".github/workflows/k3s-e2eTests.yml",
    "content": "name: k3s e2e tests for flannel\n\non: pull_request\n\nenv:\n  ARCH: amd64\n  GO_VERSION: \"1.24\"\n  KUBECONFIG: ${HOME}/.kube/config\n\npermissions:\n  contents: read\n\njobs:\n  k3s-e2e-tests:\n    name: test\n    runs-on: ubuntu-latest\n    timeout-minutes: 90\n    steps:\n      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: Set up Go 1.x\n        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: ${{ env.GO_VERSION }}\n\n      - name: set up modules\n        run: sudo modprobe br_netfilter overlay\n\n      - name: build flannel image\n        run: make image\n        \n      - name: run e2e tests with k3s\n        run: make k3s-e2e-test\n"
  },
  {
    "path": ".github/workflows/release.yml",
    "content": "name: release flannel and upload docker images\n  \non:\n  release:\n    types: [published]\n\nenv:\n  GO_VERSION: \"1.24\"\n  LINUX_ARCHES: \"amd64 arm arm64 s390x ppc64le riscv64\"\n  REPOSITORY: flannel/flannel\n  IMAGE_NAME: flannel-io/flannel\n  REGISTRY: ghcr.io\n\npermissions:\n  contents: read\n\njobs:\n  build-and-push-images:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read\n      packages: write\n\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: set tag \n        run: echo \"GIT_TAG=$(git describe --tags --always)\" >> $GITHUB_ENV\n\n      - name: Set up Go 1.x\n        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: ${{ env.GO_VERSION }}\n\n      - name: go mod vendor\n        run: go mod vendor\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a #v4.0.0\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0\n\n      - name: Docker meta\n        id: meta\n        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf #v6.0.0\n        with:\n          images: ${{ env.REPOSITORY }}\n          flavor: latest=false\n          tags: |\n            type=ref,event=tag\n\n      - name: Log in to Docker Hub\n        if: github.repository_owner == 'flannel-io' && success()\n        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #v4.0.0\n        with:\n          username: ${{ secrets.DOCKER_USERNAME }}\n          password: ${{ secrets.DOCKER_PASSWORD }}\n\n      - name: Build and push Docker image\n        if: github.repository_owner == 'flannel-io' && success()\n        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 #v7.0.0\n        with:\n          context: .\n          file: images/Dockerfile\n          push: true\n          platforms: linux/amd64,linux/arm64,linux/arm,linux/s390x,linux/ppc64le,linux/riscv64\n          tags: ${{ steps.meta.outputs.tags }}\n          build-args: TAG=${{ env.GIT_TAG }}\n\n  build-and-push-images-github-registry:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: read\n      packages: write\n      attestations: write\n      id-token: write\n\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: set tag\n        run: echo \"GIT_TAG=$(git describe --tags --always)\" >> $GITHUB_ENV\n\n      - name: Set up Go 1.x\n        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: ${{ env.GO_VERSION }}\n\n      - name: go mod vendor\n        run: go mod vendor\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a #v4.0.0\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0\n\n      - name: Log in to the Container registry\n        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #v4.0.0\n        with:\n          registry: ${{ env.REGISTRY }}\n          username: ${{ github.actor }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n\n      - name: Extract metadata (tags, labels) for Docker\n        id: meta\n        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf #v6.0.0\n        with:\n          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\n\n      - name: Build and push Docker image\n        id: push\n        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 #v7.0.0\n        with:\n          context: .\n          file: images/Dockerfile\n          push: true\n          platforms: linux/amd64,linux/arm64,linux/arm,linux/s390x,linux/ppc64le,linux/riscv64\n          tags: ${{ steps.meta.outputs.tags }}\n          labels: ${{ steps.meta.outputs.labels }}\n          build-args: TAG=${{ env.GIT_TAG }}\n\n      - name: Generate artifact attestation\n        uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 #v4.1.0\n        with:\n          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\n          subject-digest: ${{ steps.push.outputs.digest }}\n          push-to-registry: true\n\n  build-and-push-artifacts:\n    runs-on: ubuntu-latest\n    permissions:\n      contents: write\n      packages: write\n\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: set tag\n        run: echo \"GIT_TAG=$(git describe --tags --always)\" >> $GITHUB_ENV\n\n      - name: Set up Go 1.x\n        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: ${{ env.GO_VERSION }}\n\n      - name: go mod vendor\n        run: go mod vendor\n      \n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a #v4.0.0\n\n      - name: Build release artifacts\n        run: make release\n\n      - name: Upload flannel binaries to the release page\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: gh release upload ${{ env.GIT_TAG }} dist/flannel* \n\n  publish-chart:\n    permissions:\n      contents: write\n      packages: write\n      pages: write      # to deploy to Pages\n      id-token: write   # to verify the deployment originates from an appropriate source\n\n    environment:\n      name: github-pages\n      url: ${{ steps.deployment.outputs.page_url }}\n\n    needs: build-and-push-images\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout repository\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: set tag\n        run: echo \"GIT_TAG=$(git describe --tags --always)\" >> $GITHUB_ENV\n\n      - name: Package chart\n        run: make release-manifest release-helm\n\n      - name: Upload chart and manifests to the release page\n        env:\n          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        run: gh release upload ${{ env.GIT_TAG }} dist/flannel.tgz dist/kube-flannel.yml\n\n      - name: Setup Pages\n        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b #v5.0.0\n\n      - name: Upload artifact\n        uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b #v4.0.0\n        with:\n          path: 'chart/'\n\n      - name: Deploy to GitHub Pages\n        id: deployment\n        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e #v4.0.5\n\n"
  },
  {
    "path": ".github/workflows/scorecard.yml",
    "content": "# This workflow uses actions that are not certified by GitHub. They are provided\n# by a third-party and are governed by separate terms of service, privacy\n# policy, and support documentation.\n\nname: Scorecard supply-chain security\non:\n  # For Branch-Protection check. Only the default branch is supported. See\n  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection\n  branch_protection_rule:\n  # To guarantee Maintained check is occasionally updated. See\n  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained\n  schedule:\n    - cron: '19 11 * * 3'\n  push:\n    branches: [ \"master\" ]\n\n# Declare default permissions as read only.\npermissions: read-all\n\njobs:\n  analysis:\n    name: Scorecard analysis\n    runs-on: ubuntu-latest\n    # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled.\n    if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request'\n    permissions:\n      # Needed to upload the results to code-scanning dashboard.\n      security-events: write\n      # Needed to publish results and get a badge (see publish_results below).\n      id-token: write\n      # Uncomment the permissions below if installing in a private repository.\n      # contents: read\n      # actions: read\n\n    steps:\n      - name: \"Checkout code\"\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n        with:\n          persist-credentials: false\n\n      - name: \"Run analysis\"\n        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3\n        with:\n          results_file: results.sarif\n          results_format: sarif\n          # (Optional) \"write\" PAT token. Uncomment the `repo_token` line below if:\n          # - you want to enable the Branch-Protection check on a *public* repository, or\n          # - you are installing Scorecard on a *private* repository\n          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.\n          # repo_token: ${{ secrets.SCORECARD_TOKEN }}\n\n          # Public repositories:\n          #   - Publish results to OpenSSF REST API for easy access by consumers\n          #   - Allows the repository to include the Scorecard badge.\n          #   - See https://github.com/ossf/scorecard-action#publishing-results.\n          # For private repositories:\n          #   - `publish_results` will always be set to `false`, regardless\n          #     of the value entered here.\n          publish_results: true\n\n          # (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore\n          # file_mode: git\n\n      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF\n      # format to the repository Actions tab.\n      - name: \"Upload artifact\"\n        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n        with:\n          name: SARIF file\n          path: results.sarif\n          retention-days: 5\n\n      # Upload the results to GitHub's code scanning dashboard (optional).\n      # Commenting out will disable upload of results to your repo's Code Scanning dashboard\n      - name: \"Upload to code-scanning\"\n        uses: github/codeql-action/upload-sarif@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6\n        with:\n          sarif_file: results.sarif\n"
  },
  {
    "path": ".github/workflows/trivy.yml",
    "content": "# This workflow uses actions that are not certified by GitHub.\n# They are provided by a third-party and are governed by\n# separate terms of service, privacy policy, and support\n# documentation.\n\nname: Vulnerability scan\n\non:\n  pull_request:\n    # The branches below must be a subset of the branches above\n    branches: [ \"master\" ]\n  schedule:\n    - cron: '34 5 * * 2'\n\nenv:\n  GO_VERSION: \"1.24\"\n  REPOSITORY: flannel/flannel\n\npermissions:\n  contents: read\n\njobs:\n  build:\n    permissions:\n      contents: read # for actions/checkout to fetch code\n      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results\n    name: Build\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2\n\n      - name: Set up Go 1.x\n        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 #v6.3.0\n        with:\n          go-version: ${{ env.GO_VERSION }}\n\n      - name: Build an image from Dockerfile\n        run: |\n          ARCH=amd64 TAG=${{ github.sha }} make image\n\n      - name: Run Trivy vulnerability scanner in tarball mode\n        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 #v0.35.0\n        with:\n          input: ./dist/flanneld-${{ github.sha }}-amd64.docker\n          severity: 'CRITICAL,HIGH'\n          format: 'sarif'\n          output: 'trivy-results.sarif'\n\n      - name: Upload Trivy scan results to GitHub Security tab\n        uses: github/codeql-action/upload-sarif@0d579ffd059c29b07949a3cce3983f0780820c98 \n        with:\n          sarif_file: 'trivy-results.sarif'\n"
  },
  {
    "path": ".gitignore",
    "content": "dist/*.tar.gz\ndist/flanneld*\ndist/*.docker\ndist/here.txt\ndist/flannel_oci.tar\ncover.out\n.editorconfig\n.idea/\ndefault.etcd/\nflannel.exe\nbash_unit\ndist/qemu-*\n.vscode/*\nvendor/*\ne2e/scratch/*\ne2e/kube-flannel.yml\n"
  },
  {
    "path": "ACTIVITY_SUMMARY_2025.md",
    "content": "# Flannel Repository Activity Summary - 2025\n\n## Overview\nThis document provides a comprehensive summary of the flannel-io/flannel repository activity during the year 2025.\n\n---\n\n## 📊 Key Statistics\n\n### Releases\n- **Total Releases in 2025:** 10\n- **Release Details:**\n  1. v0.26.3 - January 8, 2025\n  2. v0.26.4 - February 4, 2025\n  3. v0.26.5 - March 6, 2025\n  4. v0.26.6 - April 8, 2025\n  5. v0.26.7 - April 15, 2025\n  6. v0.27.0 - June 4, 2025\n  7. v0.27.1 - July 10, 2025\n  8. v0.27.2 - July 21, 2025\n  9. v0.27.3 - September 1, 2025\n  10. v0.27.4 - October 2, 2025\n\n### Pull Requests\n- **Total PRs Merged in 2025:** 111\n\n### Issues\n- **Total Issues Created in 2025:** 40\n- **Total Issues Resolved in 2025:** 52\n\n---\n\n## 👥 Most Active Contributors (by PRs merged in 2025)\n\n| Rank | Contributor | PRs Merged |\n|------|-------------|------------|\n| 1 | dependabot[bot] | 54 |\n| 2 | thomasferrandiz | 20 |\n| 3 | rbrtbnfgl | 10 |\n| 4 | pratikjagrut | 2 |\n| 5 | lilioid | 2 |\n| 6 | tyholling | 1 |\n| 7 | sudheerv | 1 |\n| 8 | philips | 1 |\n| 9 | pgonin | 1 |\n| 10 | np-13 | 1 |\n\n*Note: Dependabot automated dependency updates account for approximately 49% of merged PRs.*\n\n---\n\n## 📈 Activity Highlights\n\n- **Release Cadence:** The project maintained a consistent release schedule with 10 releases throughout the year, averaging about 1 release per month.\n- **Version Progress:** The project progressed from v0.26.x to v0.27.x series during 2025.\n- **Issue Resolution Rate:** 52 issues were resolved while 40 new issues were created, showing a positive net resolution (130% resolution rate).\n- **PR Activity:** 111 PRs were merged, demonstrating active development and maintenance.\n- **Automation:** Dependabot contributed significantly to keeping dependencies up to date with 54 automated PRs.\n- **Human Contributors:** Excluding automated contributions, the top human contributors were thomasferrandiz (20 PRs) and rbrtbnfgl (10 PRs).\n\n---\n\n## 🔍 Summary\n\nThe flannel repository showed healthy activity in 2025 with:\n- Consistent release schedule (10 releases)\n- Active maintenance (111 PRs merged)\n- Positive issue resolution trend (52 resolved vs 40 created)\n- Strong community contributions from both automated tooling and human maintainers\n- Two major contributors (thomasferrandiz and rbrtbnfgl) leading the development efforts\n\n---\n\n*Report generated on: January 21, 2026*\n*Data source: GitHub API via flannel-io/flannel repository*\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# How to Contribute\n\nCoreOS projects are [Apache 2.0 licensed](LICENSE) and accept contributions via\nGitHub pull requests.  This document outlines some of the conventions on\ndevelopment workflow, commit message formatting, contact points and other\nresources to make it easier to get your contribution accepted.\n\n# Certificate of Origin\n\nBy contributing to this project you agree to the Developer Certificate of\nOrigin (DCO). This document was created by the Linux Kernel community and is a\nsimple statement that you, as a contributor, have the legal right to make the\ncontribution. See the [DCO](DCO) file for details.\n\n## Getting Started\n\n- Fork the repository on GitHub\n- Read the [README](README.md) for build and test instructions\n- Play with the project, submit bugs, submit patches!\n\n## Contribution Flow\n\nThis is a rough outline of what a contributor's workflow looks like:\n\n- Create a topic branch from where you want to base your work (usually master).\n- Make commits of logical units.\n- Make sure your commit messages are in the proper format (see below).\n- Push your changes to a topic branch in your fork of the repository.\n- Make sure the tests pass, and add any new tests as appropriate.\n- Submit a pull request to the original repository.\n\nThanks for your contributions!\n\n### Format of the Commit Message\n\nWe follow a rough convention for commit messages that is designed to answer two\nquestions: what changed and why. The subject line should feature the what and\nthe body of the commit should describe the why.\n\n```\nscripts: add the test-cluster command\n\nthis uses tmux to setup a test cluster that you can easily kill and\nstart for debugging.\n\nFixes #38\n```\n\nThe format can be described more formally as follows:\n\n```\n<subsystem>: <what changed>\n<BLANK LINE>\n<why this change was made>\n<BLANK LINE>\n<footer>\n```\n\nThe first line is the subject and should be no longer than 70 characters, the\nsecond line is always blank, and other lines should be wrapped at 80 characters.\nThis allows the message to be easier to read on GitHub as well as in various\ngit tools.\n"
  },
  {
    "path": "DCO",
    "content": "Developer Certificate of Origin\nVersion 1.1\n\nCopyright (C) 2004, 2006 The Linux Foundation and its contributors.\n660 York Street, Suite 102,\nSan Francisco, CA 94110 USA\n\nEveryone is permitted to copy and distribute verbatim copies of this\nlicense document, but changing it is not allowed.\n\n\nDeveloper's Certificate of Origin 1.1\n\nBy making a contribution to this project, I certify that:\n\n(a) The contribution was created in whole or in part by me and I\n    have the right to submit it under the open source license\n    indicated in the file; or\n\n(b) The contribution is based upon previous work that, to the best\n    of my knowledge, is covered under an appropriate open source\n    license and I have the right under that license to submit that\n    work with modifications, whether created in whole or in part\n    by me, under the same open source license (unless I am\n    permitted to submit under a different license), as indicated\n    in the file; or\n\n(c) The contribution was provided directly to me by some other\n    person who certified (a), (b) or (c) and I have not modified\n    it.\n\n(d) I understand and agree that this project and the contribution\n    are public and that a record of the contribution (including all\n    personal information I submit with it, including my sign-off) is\n    maintained indefinitely and may be redistributed consistent with\n    this project or the open source license(s) involved.\n"
  },
  {
    "path": "Documentation/adrs/add-nftables-implementation.md",
    "content": "# Add nftables implementation to flannel\n\nDate: 2024-02-01\n\n## Status\n\nWriting\n\n## Context\nAt the moment, flannel uses iptables to mask and route packets.\nOur implementation is  based on the library from coreos (https://github.com/coreos/go-iptables).\n\nThere are several issues with using iptables in flannel:\n* performance: packets are matched using a list so performance is O(n). This isn't very important for flannel because use few iptables rules anyway.\n* stability: \n** rules must be purged then updated every time flannel needs to change a rule to keep the correct order\n** there can be interferences with other k8s components using iptables as well (kube-proxy, kube-router...)\n* deprecation: nftables is pushed as a replacement for iptables in the kernel and in future distros including the future RHEL.\n\nReferences:\n- https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/3866-nftables-proxy/README.md#motivation\n\n## Current state\nIn flannel code, all references to iptables are wrapped in the `iptables` package.\n\nThe package provides the type `IPTableRule` to represent an individual rule. This type is almost entirely internal to the package so it would be easy to refactor the code to hide in favor of a more abstract type that would work for both iptables and nftables rules.\n\nUnfortunately the package doesn't provide an interface so in order to provide both an iptables-based and an nftables-based implementation this needs to be refactored.\n\nThis package includes several Go interfaces (`IPTables`, `IPTablesError`) that are used for testing.\n\n## Requirements\nIdeally, flannel will include both iptables and nftables implementation. These need to coexist in the code but will be mutually exclusive at runtime.\n\nThe choice of which implementation to use will be triggered by an optional CLI flag.\niptables will remain the default for the time being.\n\nUsing nftables is an opportunity for optimising the rules deployed by flannel but we need to be careful about retro-compatibility with the current backend.\n\nStarting flannel in either mode should reset the other mode as best as possible to ensure that users don't need to reboot if they need to change mode.\n\n## Architecture\nCurrently, flannel uses two dedicated tables for its own rules: `FLANNEL-POSTRTG` and `FLANNEL-FWD`.\n* flannel adds rules to the `FORWARD` and `POSTROUTING` tables to direct traffic to its own tables.\n* rules in `FLANNEL-POSTRTG` are used to manage masquerading of the traffic to/from the pods\n* rules in `FLANNEL-FWD` are used to ensure that traffic to and from the flannel network can be forwarded\n\nWith nftables, flannel would have its own dedicated table (`flannel`) with arbitrary chains and rules as needed.\n\nsee https://wiki.nftables.org/wiki-nftables/index.php/Performing_Network_Address_Translation_(NAT)\n```\n# !! untested example\ntable flannel {\n    chain flannel-postrtg {\n        type nat hook postrouting priority 0; \n        # kube-proxy\n        meta mark 0x4000/0x4000 return\n        # don't NAT traffic within overlay network \n        ip saddr $pod_cidr ip daddr $cluster_cidr return \n        ip saddr $cluster_cidr ip daddr $pod_cidr return \n        # Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address\n        ip saddr != $pod_cidr ip daddr $cluster_cidr return \n        # NAT if it's not multicast traffic\n        ip saddr $cluster_cidr ip daddr != 224.0.0.0/4 nat \n        # Masquerade anything headed towards flannel from the host\n        ip saddr != $cluster_cidr ip daddr $cluster_cidr nat \n    }\n\n    chain flannel-fwd {\n        type filter hook input priority 0; policy drop;\n        # allow traffic to be forwarded if it is to or from the flannel network range\n        ip saddr flannelNetwork accept\n        ip daddr flannelNetwork accept\n    }\n}\n```\n\n## nftables library\nWe can either:\n* call the `nft` executable directly\n* use https://github.com/kubernetes-sigs/knftables which is developed for kube-proxy and should cover our use case\n\n## Implementation steps\n* refactor current iptables code to better encapsulate iptables calls in the dedicated package\n* implement nftables mode that is the exact equivalent of the current iptables code\n* add similar unit tests and e2e test coverage\n* try to optimize the code using nftables-specific feature\n* integrate the new flag in k3s\n\n\n## Decision\n"
  },
  {
    "path": "Documentation/backends.md",
    "content": "# Backends\n\nFlannel may be paired with several different backends. Once set, the backend should not be changed at runtime.\n\nVXLAN is the recommended choice. host-gw is recommended for more experienced users who want the performance improvement and whose infrastructure support it (typically it can't be used in cloud environments). UDP is suggested for debugging only or for very old kernels that don't support VXLAN.\n\nIn case `firewalld` is enabled on the node the port used by the backend needs to be enabled with `firewall-cmd`:\n```\nfirewall-cmd --permanent --zone=public --add-port=[port]/udp\n```\n\nFor more information on configuration options for Tencent see [TencentCloud VPC Backend for Flannel][tencentcloud-vpc]\n\n## Recommended backends\n\n### VXLAN\n\nUse in-kernel VXLAN to encapsulate the packets.\n\nType and options:\n* `Type` (string): `vxlan`\n* `VNI` (number): VXLAN Identifier (VNI) to be used. On Linux, defaults to 1. On Windows should be greater than or equal to 4096. \n* `Port` (number): UDP port to use for sending encapsulated packets. On Linux, defaults to kernel default, currently 8472, but on Windows, must be 4789.\n* `GBP` (Boolean): Enable [VXLAN Group Based Policy](https://github.com/torvalds/linux/commit/3511494ce2f3d3b77544c79b87511a4ddb61dc89).  Defaults to `false`. GBP is not supported on Windows\n* `DirectRouting` (Boolean): Enable direct routes (like `host-gw`) when the hosts are on the same subnet. VXLAN will only be used to encapsulate packets to hosts on different subnets. Defaults to `false`. DirectRouting is not supported on Windows.\n* `MTU` (number): Desired MTU for the outgoing packets if not defined the MTU of the external interface is used.\n* `MacPrefix` (String): Only use on Windows, set to the MAC prefix. Defaults to `0E-2A`.\n\nStarting with Ubuntu 21.10, vxlan support on Raspberry Pi has been moved into a separate kernel module. \n```\nsudo apt install linux-modules-extra-raspi\n```\n\n### host-gw\n\nUse host-gw to create IP routes to subnets via remote machine IPs. Requires direct layer2 connectivity between hosts running flannel.\n\nhost-gw provides good performance, with few dependencies, and easy set up.\n\nType:\n* `Type` (string): `host-gw`\n\n### WireGuard\n\nUse in-kernel [WireGuard](https://www.wireguard.com) to encapsulate and encrypt the packets.\n\nType:\n* `Type` (string): `wireguard`\n* `PSK` (string): Optional. The pre shared key to use. Use `wg genpsk` to generate a key.\n* `ListenPort` (int): Optional. The udp port to listen on. Default is `51820`.\n* `ListenPortV6` (int): Optional. The udp port to listen on for ipv6. Default is `51821`.\n* `MTU` (number): Desired MTU for the outgoing packets if not defined the MTU of the external interface is used.\n* `Mode` (string): Optional.\n    * separate - Use separate wireguard tunnels for ipv4 and ipv6 (default)\n    * auto - Single wireguard tunnel for both address families; autodetermine the preferred peer address\n    * ipv4 - Single wireguard tunnel for both address families; use ipv4 for\n      the peer addresses\n    * ipv6 - Single wireguard tunnel for both address families; use ipv6 for\n      the peer addresses\n* `PersistentKeepaliveInterval` (int): Optional. Default is 0 (disabled).\n\nIf no private key was generated before the private key is written to `/run/flannel/wgkey`. You can use environment `WIREGUARD_KEY_FILE` to change this path.\n\nThe static names of the interfaces are `flannel-wg` and `flannel-wg-v6`. WireGuard tools like `wg show` can be used to debug interfaces and peers.\n\nUsers of kernels < 5.6 need to [install](https://www.wireguard.com/install/) an additional Wireguard package.\n\n### UDP\n\nUse UDP only for debugging if your network and kernel prevent you from using VXLAN or host-gw.\n\nType and options:\n* `Type` (string): `udp`\n* `Port` (number): UDP port to use for sending encapsulated packets. Defaults to 8285.\n\n## Experimental backends\n\nThe following options are experimental and unsupported at this time.\n\n### Alloc\n\nAlloc performs subnet allocation with no forwarding of data packets.\n\nType:\n* `Type` (string): `alloc`\n\n### TencentCloud VPC\n\nUse TencentCloud VPC to create IP routes in a [TencentCloud VPC route table](https://intl.cloud.tencent.com/product/vpc) when running in an TencentCloud VPC. This mitigates the need to create a separate flannel interface.\n\nRequirements:\n* Running on an CVM instance that is in an TencentCloud VPC.\n* Permission require `accessid` and `keysecret`.\n    * `Type` (string): `tencent-vpc`\n    * `AccessKeyID` (string): API access key ID. Can also be configured with environment ACCESS_KEY_ID.\n    * `AccessKeySecret` (string): API access key secret. Can also be configured with environment ACCESS_KEY_SECRET.\n\nRoute Limits: TencentCloud VPC limits the number of entries per route table to 50.\n\n\n[tencentcloud-vpc]: https://github.com/flannel-io/flannel/blob/master/Documentation/tencentcloud-vpc-backend.md\n\n\n### IPIP\n\nUse in-kernel IPIP to encapsulate the packets.\n\nIPIP kind of tunnels is the simplest one. It has the lowest overhead, but can incapsulate only IPv4 unicast traffic, so you will not be able to setup OSPF, RIP or any other multicast-based protocol.\n\nType:\n* `Type` (string): `ipip`\n* `DirectRouting` (Boolean): Enable direct routes (like `host-gw`) when the hosts are on the same subnet. IPIP will only be used to encapsulate packets to hosts on different subnets. Defaults to `false`.\n\nNote that there may exist two ipip tunnel device `tunl0` and `flannel.ipip`, this is expected and it's not a bug.\n`tunl0` is automatically created per network namespace by ipip kernel module on modprobe ipip module. It is the namespace default IPIP device with attributes local=any and remote=any.\nWhen receiving IPIP protocol packets, kernel will forward them to tunl0 as a fallback device if it can't find an option whose local/remote attribute matches their src/dst ip address more precisely.\n`flannel.ipip` is created by flannel to achieve one to many ipip network.\n\n### IPSec\n\nUse in-kernel IPSec to encapsulate and encrypt the packets.\n\n[Strongswan](https://www.strongswan.org) is used at the IKEv2 daemon. A single pre-shared key is used for the initial key exchange between hosts and then Strongswan ensures that keys are rotated at regular intervals. \n\nType:\n* `Type` (string): `ipsec`\n* `PSK` (string): Required. The pre shared key to use. It needs to be at least 96 characters long. One method for generating this key is to run `dd if=/dev/urandom count=48 bs=1 status=none | xxd -p -c 48`\n* `UDPEncap` (Boolean): Optional, defaults to false. Forces the use UDP encapsulation of packets which can help with some NAT gateways.\n* `ESPProposal` (string): Optional, defaults to `aes128gcm16-sha256-prfsha256-ecp256`. Change this string to choose another ESP Proposal.\n\nHint: \nAdd rules to your firewall: Open ports 50 (for ESP protocol), UDP 500 (for IKE, to manage encryption keys) and UDP 4500 (for IPSEC NAT-Traversal mode).\n\n#### Troubleshooting\nLogging\n* When flannel is run from a container, the Strongswan tools are installed. `swanctl` can be used for interacting with the charon and it provides a logs command. \n* Charon logs are also written to the stdout of the flannel process. \n\nTroubleshooting\n* `ip xfrm state` can be used to interact with the kernel's security association database. This can be used to show the current security associations (SA) and whether a host is successfully establishing ipsec connections to other hosts.\n* `ip xfrm policy` can be used to show the installed policies. Flannel installs three policies for each host it connects to. \n\nFlannel will not restore policies that are manually deleted (unless flannel is restarted). It will also not delete stale policies on startup. They can be removed by rebooting your host or by removing all ipsec state with `ip xfrm state flush && ip xfrm policy flush` and restarting flannel.\n"
  },
  {
    "path": "Documentation/building.md",
    "content": "# Building flannel\n\nThe most reliable way to build flannel is by using Docker.\n\n## Building in a Docker container\n\nTo build flannel in a container run `make dist/flanneld-amd64`.\nYou will now have a `flanneld-amd64` binary in the `dist` directory.\n\n## Building for other platforms\n\nIf you're not running `amd64` then you need to manually set `ARCH` before running `make`. For example, to produce a \n`flanneld-s390x` binary and image, run\n* ARCH=s390x make image\n\nIf you want to cross-compile for a different platform (e.g. you're running `amd64` but you want to produce `arm` binaries) then you need the qemu-static binaries to be present in `/usr/bin`. They can be installed on Ubuntu with\n* `sudo apt-get install qemu-user-static`\n\nThen you should be able to set the ARCH as above\n* ARCH=arm make image\n\n## Building a multi-arch image\n\nTo build the multi-arch image of flannel locally, you need to install [Docker buildx](https://github.com/docker/buildx).\nThen you can use the following target:\n```\nmake build-multi-arch\n```\n\nIf you don't already have a builder running locally, you can this target to start it:\n```\nmake buildx-create-builder\n```\n\nSee the [buildx documentation](https://docs.docker.com/reference/cli/docker/buildx/) for more details.\n\n## Running the tests locally\nTo run the end-to-end tests locally, you need to installl [Docker compose](https://docs.docker.com/compose/install/).\n\n## Building manually\n\n1. Make sure you have required dependencies installed on your machine.\n    * On Ubuntu, run `sudo apt-get install linux-libc-dev golang gcc`. \n      If the golang version installed is not 1.7 or higher. Download the newest golang and install manually.\n      To build the flannel.exe on windows, mingw-w64 is also needed. Run command `sudo apt-get install mingw-w64`\n    * On Fedora/Redhat, run `sudo yum install kernel-headers golang gcc glibc-static`.\n2. Git clone the flannel repo. It MUST be placed in your GOPATH under `github.com/flannel-io/flannel`: `cd $GOPATH/src; git clone https://github.com/flannel-io/flannel.git`\n3. Run the build script, ensuring that `CGO_ENABLED=1`: `cd flannel; CGO_ENABLED=1 make dist/flanneld` for linux usage.\n   Run the build script, ensuring that `CGO_ENABLED=1`: `cd flannel; CGO_ENABLED=1 make dist/flanneld.exe` for windows usage.\n\n"
  },
  {
    "path": "Documentation/configuration.md",
    "content": "# Configuration\n\nIf the --kube-subnet-mgr argument is true, flannel reads its configuration from `/etc/kube-flannel/net-conf.json`.\n\nIf the --kube-subnet-mgr argument is false, flannel reads its configuration from etcd.\nBy default, it will read the configuration from `/coreos.com/network/config` (which can be overridden using `--etcd-prefix`).\n\nUse the `etcdctl` utility to set values in etcd.\n\nThe value of the config is a JSON dictionary with the following keys:\n\n* `Network` (string): IPv4 network in CIDR format to use for the entire flannel network. (Mandatory if EnableIPv4 is true)\n\n* `IPv6Network` (string): IPv6 network in CIDR format to use for the entire flannel network. (Mandatory if EnableIPv6 is true)\n\n* `EnableIPv4` (bool): Enables ipv4 support\n  Defaults to `true`\n\n* `EnableIPv6` (bool): Enables ipv6 support\n  Defaults to `false`\n\n* `EnableNFTables` (bool): (EXPERIMENTAL) If set to true, flannel uses nftables instead of iptables to masquerade the traffic.\n   Default to `false`\n\n* `SubnetLen` (integer): The size of the subnet allocated to each host.\n   Defaults to 24 (i.e. /24) unless `Network` was configured to be smaller than a /22 in which case it is two less than the network.\n\n* `SubnetMin` (string): The beginning of IP range which the subnet allocation should start with.\n   Defaults to the second subnet of `Network`.\n\n* `SubnetMax` (string): The end of the IP range at which the subnet allocation should end with.\n   Defaults to the last subnet of `Network`.\n\n* `IPv6SubnetLen` (integer): The size of the ipv6 subnet allocated to each host.\n   Defaults to 64 (i.e. /64) unless `Ipv6Network` was configured to be smaller than a /62 in which case it is two less than the network.\n\n* `IPv6SubnetMin` (string): The beginning of IPv6 range which the subnet allocation should start with.\n   Defaults to the second subnet of `Ipv6Network`.\n\n* `IPv6SubnetMax` (string): The end of the IPv6 range at which the subnet allocation should end with.\n   Defaults to the last subnet of `Ipv6Network`.\n\n* `Backend` (dictionary): Type of backend to use and specific configurations for that backend.\n   The list of available backends and the keys that can be put into the this dictionary are listed in [Backends](backends.md).\n   Defaults to `vxlan` backend.\n\nSubnet leases have a duration of 24 hours. Leases are renewed within 1 hour of their expiration,\nunless a different renewal margin is set with the ``--subnet-lease-renew-margin`` option.\n\n## Example configuration JSON\n\nThe following configuration illustrates the use of most options with `udp` backend.\n\n```json\n{\n\t\"Network\": \"10.0.0.0/8\",\n\t\"SubnetLen\": 20,\n\t\"SubnetMin\": \"10.10.0.0\",\n\t\"SubnetMax\": \"10.99.0.0\",\n\t\"Backend\": {\n\t\t\"Type\": \"udp\",\n\t\t\"Port\": 7890\n\t}\n}\n```\n\n## Key command line options\n\n```bash\n--public-ip=\"\": IP accessible by other nodes for inter-host communication. Defaults to the IP of the interface being used for communication.\n--etcd-endpoints=http://127.0.0.1:4001: a comma-delimited list of etcd endpoints.\n--etcd-prefix=/coreos.com/network: etcd prefix.\n--etcd-keyfile=\"\": SSL key file used to secure etcd communication.\n--etcd-certfile=\"\": SSL certification file used to secure etcd communication.\n--etcd-cafile=\"\": SSL Certificate Authority file used to secure etcd communication.\n--kube-subnet-mgr: Contact the Kubernetes API for subnet assignment instead of etcd.\n--iface=\"\": interface to use (IP or name) for inter-host communication. Defaults to the interface for the default route on the machine. This can be specified multiple times to check each option in order. Returns the first match found.\n--iface-regex=\"\": regex expression to match the first interface to use (IP or name) for inter-host communication. If unspecified, will default to the interface for the default route on the machine. This can be specified multiple times to check each regex in order. Returns the first match found. This option is superseded by the iface option and will only be used if nothing matches any option specified in the iface options.\n--iface-can-reach=\"\": detect interface to use (IP or name) for inter-host communication based on which will be used for provided IP. This is exactly the interface to use of command \"ip route get <ip-address>\" (example: --iface-can-reach=192.168.1.1 results the interface can be reached to 192.168.1.1 will be selected)\n--iptables-forward-rules: Adds default ACCEPT rules to the iptables FORWARD chain to allow network traffic forwarding (default: true).\n--iptables-resync=5: resync period for iptables rules, in seconds. Defaults to 5 seconds, if you see a large amount of contention for the iptables lock increasing this will probably help.\n--subnet-file=/run/flannel/subnet.env: filename where env variables (subnet and MTU values) will be written to.\n--net-config-path=/etc/kube-flannel/net-conf.json: path to the network configuration file to use\n--subnet-lease-renew-margin=60: subnet lease renewal margin, in minutes.\n--ip-masq=false: setup IP masquerade for traffic destined for outside the flannel network. Flannel assumes that the default policy is ACCEPT in the NAT POSTROUTING chain.\n-v=0: log level for V logs. Set to 1 to see messages related to data path.\n--healthz-ip=\"0.0.0.0\": The IP address for healthz server to listen (default \"0.0.0.0\")\n--healthz-port=0: The port for healthz server to listen(0 to disable)\n--version: print version and exit\n```\n\nMTU is calculated and set automatically by flannel. It then reports that value in `subnet.env`. This value can be changed as [backend](backends.md) config.\n\n## Environment variables\n\nThe command line options outlined above can also be specified via environment variables.\nFor example `--etcd-endpoints=http://10.0.0.2:2379` is equivalent to `FLANNELD_ETCD_ENDPOINTS=http://10.0.0.2:2379` environment variable.\nAny command line option can be turned into an environment variable by prefixing it with `FLANNELD_`, stripping leading dashes, converting to uppercase and replacing all other dashes to underscores.\n\n`EVENT_QUEUE_DEPTH` is another environment variable to indicate the kubernetes scale. Set `EVENT_QUEUE_DEPTH` to adapter your cluster node numbers. If not set, default value is 5000. \n\n`CONT_WHEN_CACHE_NOT_READY` is environment variable to indicate if flanneld should continue even when the node informer cache is not fully sync'd yet. This can happen for large clusters (clusters with node capacity higher than `EVENT_QUEUE_DEPTH`). Set `CONT_WHEN_CACHE_NOT_READY` to \"true\" to let flanneld not fail startup for such large capacity clusters.\n\n## Health Check\n\nFlannel provides a health check http endpoint `healthz`. Currently this endpoint will blindly\nreturn http status ok(i.e. 200) when flannel is running. This feature is by default disabled.\nSet `healthz-port` to a non-zero value will enable a healthz server for flannel.\n\n## Dual-stack\n\nFlannel supports dual-stack mode. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for vxlan, wireguard or host-gw(linux) backends.\n\nRequirements:\n* v1.0.1 of flannel binary from [containernetworking/plugins](https://github.com/containernetworking/plugins)\n* Nodes must have an ipv4 and ipv6 address in the main interface\n* Nodes must have an ipv4 and ipv6 address default route\n* vxlan support ipv6 tunnel require kernel version >= 3.12\n* When public IPv6 addresses are used, routing of the \"IPv6Network\" towards the cluster must be configured externally from flannel (see [#2289](https://github.com/flannel-io/flannel/issues/2289) for details)\n\nConfiguration:\n* Set \"EnableIPv6\": true and the \"IPv6Network\", for example \"IPv6Network\": * \"2001:cafe:42:0::/56\" in the net-conf.json of the kube-flannel-cfg ConfigMap or in `/coreos.com/network/config` for etcd\n\nIf everything works as expected, flanneld should generate a `/run/flannel/subnet.env` file with IPV6 subnet and network. For example:\n\n```bash\nFLANNEL_NETWORK=10.42.0.0/16\nFLANNEL_SUBNET=10.42.0.1/24\nFLANNEL_IPV6_NETWORK=2001:cafe:42::/56\nFLANNEL_IPV6_SUBNET=2001:cafe:42::1/64\nFLANNEL_MTU=1450\nFLANNEL_IPMASQ=true\n```\n\n## IPv6 only\n\nTo use an IPv6-only environment use the same configuration of the Dual-stack section to enable IPv6 and add \"EnableIPv4\": false in the net-conf.json of the kube-flannel-cfg ConfigMap. In case of IPv6-only setup, please use the docker.io IPv6-only endpoint as described in the following link: https://www.docker.com/blog/beta-ipv6-support-on-docker-hub-registry/\n\n## nftables mode\nTo enable `nftables` mode in flannel, set `EnableNFTables` to true in flannel configuration.\n\nNote: to test with kube-proxy, use kubeadm with the following configuration:\n```yaml\napiVersion: kubeadm.k8s.io/v1beta3\nkind: ClusterConfiguration\nkubernetesVersion: v1.29.0\ncontrollerManager:\n  extraArgs:\n    feature-gates: NFTablesProxyMode=true\n---\napiVersion: kubeproxy.config.k8s.io/v1alpha1\nkind: KubeProxyConfiguration\nmode: \"nftables\"\nfeatureGates:\n  NFTablesProxyMode: true\n```\n"
  },
  {
    "path": "Documentation/extension.md",
    "content": "# Extension\n\nThe `extension` backend provides an easy way for prototyping new backend types for flannel.\n\nIt is _not_ recommended for production use, for example it doesn't have a built in retry mechanism.\n\nThis backend has the following configuration\n* `Type` (string): `extension`\n* `PreStartupCommand`  (string): Command to run before allocating a network to this host\n    * The stdout of the process is captured and passed to the stdin of the SubnetAdd/Remove commands.\n* `PostStartupCommand`  (string): Command to run after allocating a network to this host\n    * The following environment variable is set\n            * SUBNET - The subnet of the remote host that was added.\n* `SubnetAddCommand`   (string): Command to run when a subnet is added\n    * stdin - The output from `PreStartupCommand` is passed in.\n    * The following environment variables are set\n        * SUBNET - The ipv4 subnet of the remote host that was added.\n        * IPV6SUBNET - The ipv6 subnet of the remote host that was added.\n        * PUBLIC_IP - The public IP of the remote host.\n        * PUBLIC_IPV6 - The public IPv6 of the remote host.\n* `SubnetRemoveCommand`(string): Command to run when a subnet is removed\n    * stdin - The output from `PreStartupCommand` is passed in.\n      * The following environment variables are set\n          * SUBNET - The ipv4 subnet of the remote host that was removed.\n          * IPV6SUBNET - The ipv6 subnet of the remote host that was removed.\n          * PUBLIC_IP - The public IP of the remote host.\n          * PUBLIC_IPV6 - The public IPv6 of the remote host.\n\nAll commands are run through the `sh` shell and are run with the same permissions as the flannel daemon.\n\n\n## Simple example (host-gw)\nTo replicate the functionality of the host-gw plugin, there's no need for a startup command.\n\nThe backend just needs to manage the route to subnets when they are added or removed.\n\nAn example\n```json\n{\n  \"Network\": \"10.0.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"extension\",\n    \"SubnetAddCommand\": \"ip route add $SUBNET via $PUBLIC_IP\",\n    \"SubnetRemoveCommand\": \"ip route del $SUBNET via $PUBLIC_IP\"\n  }\n}\n```\n\n\n## Complex example (vxlan)\nVXLAN is more complex. It needs to store the MAC address of the vxlan device when it's created and to make it available to the flannel daemon running on other hosts.\nThe address of the vxlan device also needs to be set _after_ the subnet has been allocated.\n\nAn example\n```json\n{\n  \"Network\": \"10.50.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"extension\",\n    \"PreStartupCommand\": \"export VNI=1; export IF_NAME=flannel-vxlan; ip link del $IF_NAME 2>/dev/null; ip link add $IF_NAME type vxlan id $VNI dstport 8472 && cat /sys/class/net/$IF_NAME/address\",\n    \"PostStartupCommand\": \"export IF_NAME=flannel-vxlan; export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; ip addr add $SUBNET_IP/32 dev $IF_NAME && ip link set $IF_NAME up\",\n    \"SubnetAddCommand\": \"export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; export IF_NAME=flannel-vxlan; read VTEP; ip route add $SUBNET nexthop via $SUBNET_IP dev $IF_NAME onlink && ip neigh replace $SUBNET_IP dev $IF_NAME lladdr $VTEP && bridge fdb add $VTEP dev $IF_NAME self dst $PUBLIC_IP\"\n  }\n}\n```\n"
  },
  {
    "path": "Documentation/integrations.md",
    "content": "# Integrations\n\nThis document tracks projects that integrate with flannel. [Join the community](https://github.com/flannel-io/flannel/) and help us keep the list current.\n\n## Projects\n\n[kube-network-policies](https://github.com/kubernetes-sigs/kube-network-policies): Network policies controller that can be deployed alongside Flannel.\n\n[Canal](https://projectcalico.docs.tigera.io/getting-started/kubernetes/flannel/flannel): Kubernetes CNI plugin that uses Calico for network policies and intra-node communications and Flannel for inter-node communications.\n\n[K3s](https://k3s.io/): Kubernetes distribution with flannel embedded as CNI.\n\n[RKE2](https://docs.rke2.io/): Kubernetes distribution packed with Canal as default CNI and can be configured with Flannel with support for windows.\n"
  },
  {
    "path": "Documentation/kube-flannel.yml",
    "content": "---\nkind: Namespace\napiVersion: v1\nmetadata:\n  name: kube-flannel\n  labels:\n    k8s-app: flannel\n    pod-security.kubernetes.io/enforce: privileged\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  labels:\n    k8s-app: flannel\n  name: flannel\nrules:\n- apiGroups:\n  - \"\"\n  resources:\n  - pods\n  verbs:\n  - get\n- apiGroups:\n  - \"\"\n  resources:\n  - nodes\n  verbs:\n  - get\n  - list\n  - watch\n- apiGroups:\n  - \"\"\n  resources:\n  - nodes/status\n  verbs:\n  - patch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  labels:\n    k8s-app: flannel\n  name: flannel\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: flannel\nsubjects:\n- kind: ServiceAccount\n  name: flannel\n  namespace: kube-flannel\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  labels:\n    k8s-app: flannel\n  name: flannel\n  namespace: kube-flannel\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: kube-flannel-cfg\n  namespace: kube-flannel\n  labels:\n    tier: node\n    k8s-app: flannel\n    app: flannel\ndata:\n  cni-conf.json: |\n    {\n      \"name\": \"cbr0\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\": [\n        {\n          \"type\": \"flannel\",\n          \"delegate\": {\n            \"hairpinMode\": true,\n            \"isDefaultGateway\": true\n          }\n        },\n        {\n          \"type\": \"portmap\",\n          \"capabilities\": {\n            \"portMappings\": true\n          }\n        }\n      ]\n    }\n  net-conf.json: |\n    {\n      \"Network\": \"10.244.0.0/16\",\n      \"EnableNFTables\": false,\n      \"Backend\": {\n        \"Type\": \"vxlan\"\n      }\n    }\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: kube-flannel-ds\n  namespace: kube-flannel\n  labels:\n    tier: node\n    app: flannel\n    k8s-app: flannel\nspec:\n  selector:\n    matchLabels:\n      app: flannel\n  template:\n    metadata:\n      labels:\n        tier: node\n        app: flannel\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: kubernetes.io/os\n                operator: In\n                values:\n                - linux\n      hostNetwork: true\n      priorityClassName: system-node-critical\n      tolerations:\n      - operator: Exists\n        effect: NoSchedule\n      serviceAccountName: flannel\n      initContainers:\n      - name: install-cni-plugin\n        image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.0-flannel1\n        command:\n        - cp\n        args:\n        - -f\n        - /flannel\n        - /opt/cni/bin/flannel\n        volumeMounts:\n        - name: cni-plugin\n          mountPath: /opt/cni/bin\n      - name: install-cni\n        image: ghcr.io/flannel-io/flannel:v0.28.1\n        command:\n        - cp\n        args:\n        - -f\n        - /etc/kube-flannel/cni-conf.json\n        - /etc/cni/net.d/10-flannel.conflist\n        volumeMounts:\n        - name: cni\n          mountPath: /etc/cni/net.d\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n      containers:\n      - name: kube-flannel\n        image: ghcr.io/flannel-io/flannel:v0.28.1\n        command:\n        - /opt/bin/flanneld\n        args:\n        - --ip-masq\n        - --kube-subnet-mgr\n        resources:\n          requests:\n            cpu: \"100m\"\n            memory: \"50Mi\"\n        securityContext:\n          privileged: false\n          capabilities:\n            add: [\"NET_ADMIN\", \"NET_RAW\"]\n        env:\n        - name: POD_NAME\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.namespace\n        - name: EVENT_QUEUE_DEPTH\n          value: \"5000\"\n        - name: CONT_WHEN_CACHE_NOT_READY\n          value: \"false\"\n        volumeMounts:\n        - name: run\n          mountPath: /run/flannel\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n        - name: xtables-lock\n          mountPath: /run/xtables.lock\n      volumes:\n      - name: run\n        hostPath:\n          path: /run/flannel\n      - name: cni-plugin\n        hostPath:\n          path: /opt/cni/bin\n      - name: cni\n        hostPath:\n          path: /etc/cni/net.d\n      - name: flannel-cfg\n        configMap:\n          name: kube-flannel-cfg\n      - name: xtables-lock\n        hostPath:\n          path: /run/xtables.lock\n          type: FileOrCreate\n"
  },
  {
    "path": "Documentation/kubernetes.md",
    "content": "# kubeadm\n\nFor information on deploying flannel manually, using the Kubernetes installer toolkit kubeadm, see [Installing Kubernetes on Linux with kubeadm][kubeadm].\n\nNOTE: If `kubeadm` is used, then pass `--pod-network-cidr=10.244.0.0/16` to `kubeadm init` to ensure that the `podCIDR` is set.\n\n# kube-flannel.yaml\n\nThe `flannel` manifest defines five things:\n1. A `kube-flannel` with PodSecurity level set to *privileged*. \n2. A ClusterRole and ClusterRoleBinding for Role Based Access Control (RBAC).\n3. A service account for `flannel` to use.\n4. A ConfigMap containing both a CNI configuration and a `flannel` configuration. The `network` in the `flannel` configuration should match the pod network CIDR. The choice of `backend` is also made here and defaults to VXLAN.\n5. A DaemonSet for every architecture to deploy the `flannel` pod on each Node. The pod has two containers 1) the `flannel` daemon itself, and 2) an initContainer for deploying the CNI configuration to a location that the `kubelet` can read.\n\nWhen you run pods, they will be allocated IP addresses from the pod network CIDR. No matter which node those pods end up on, they will be able to communicate with each other.\n\n# Notes on securing flannel deployment\nAs of Kubernetes v1.21, the [PodSecurityPolicy API was deprecated](https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/) and it will be removed in v1.25. Thus, the `flannel` manifest does not use PodSecurityPolicy anymore. \n\nIf you wish to use the [Pod Security Admission Controller](https://kubernetes.io/docs/concepts/security/pod-security-admission/) which was introduced to [replace PodSecurityPolicy](https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/), you will need to deploy `flannel` in a namespace which allows the deployment of pods with `privileged` level. The `baseline` level is insufficient to deploy `flannel` and you will see the following error message:\n```\nError creating: non-default capabilities (container \"kube-flannel\" must not include \"NET_ADMIN\", \"NET_RAW\" in securityContext.capabilities.add), host namespaces (hostNetwork=true), hostPath volumes (volumes \"run\", \"cni-plugin\", \"cni\", \"xtables-lock\")\n```\n\nThe `kube-flannel.yaml` manifest deploys `flannel` in the `kube-flannel` namespace and enables the `privileged` level for this namespace. \nThus, you will need to restrict access to this namespace if you wish to secure your cluster.\n\nIf you want to deploy `flannel` securely in a shared namespace or want more fine-grained control over the pods deployed in your cluster, you can use a 3rd-party admission controller like [Kubewarden](https://kubewarden.io). Kubewarden provides policies that can replace features of PodSecurityPolicy like [capabilities-psp-policy](https://github.com/kubewarden/capabilities-psp-policy) and [hostpaths-psp-policy](https://github.com/kubewarden/hostpaths-psp-policy).\n\nOther options include [Kyverno](https://kyverno.io/policies/pod-security/) and [OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper).\n# Annotations\n\nAdditional annotations can be configured on a specific node as parameters used when Flannel starts on that specific node\n*  `flannel.alpha.coreos.com/node-public-ip`, `flannel.alpha.coreos.com/node-public-ipv6`: Define the used IP of the node in case the node has multiple interface it selects the interface with the configured IP for the backend tunnel. If configured when Flannel starts it'll be used as the `public-ip` and `public-ipv6` flag.\n*  `flannel.alpha.coreos.com/public-ip-overwrite`, `flannel.alpha.coreos.com/public-ipv6-overwrite`: Allows to overwrite the public IP of a node that IP can be not configured on the node. Useful if the public IP can not determined from the node, e.G. because it is behind a NAT and the other nodes need to use it to create the tunnel. It can be automatically set to a nodes `ExternalIP` using the [flannel-node-annotator](https://github.com/alvaroaleman/flannel-node-annotator).\n   See also the \"NAT\" section in [troubleshooting](./troubleshooting.md) if UDP checksums seem corrupted.\n\n## Older versions of Kubernetes\n\n`kube-flannel.yaml` has some features that aren't compatible with older versions of Kubernetes, though flanneld itself should work with any version of Kubernetes.\n\n### For Kubernetes v1.6~v1.15\n\nIf you see errors saying `found invalid field...` when you try to apply `kube-flannel.yaml` then you can try the \"legacy\" manifest file\n* `kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-old-manifests/kube-flannel-legacy.yml`\n\nThis file does not bundle RBAC permissions. If you need those, run\n* `kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-old-manifests/kube-flannel-rbac.yml`\n\nIf you didn't apply the `kube-flannel-rbac.yml` manifest and you need to, you'll see errors in your flanneld logs about failing to connect.\n* `Failed to create SubnetManager: error retrieving pod spec...`\n\n### For Kubernetes v1.16\n\n`kube-flannel.yaml` uses `ClusterRole` & `ClusterRoleBinding` of `rbac.authorization.k8s.io/v1`. When you use Kubernetes v1.16, you should replace `rbac.authorization.k8s.io/v1` to `rbac.authorization.k8s.io/v1beta1` because `rbac.authorization.k8s.io/v1` had become GA from Kubernetes v1.17.\n\n### For Kubernetes <= v1.24\nAs of Kubernetes v1.21, the [PodSecurityPolicy API was deprecated](https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/) and it will be removed in v1.25. Thus, the `flannel` manifest does not use PodSecurityPolicy anymore.\n\nIf you still wish to use it, you can use `kube-flannel-psp.yaml` instead of `kube-flannel.yaml`. Please note that if you use a Kubernetes version >= 1.21, you will see a deprecation warning for the PodSecurityPolicy API.\n\n# Troubleshooting\n\nSee [troubleshooting](troubleshooting.md)\n\n[kubeadm]: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/\n"
  },
  {
    "path": "Documentation/kustomization/kube-flannel/kube-flannel.yml",
    "content": "---\nkind: Namespace\napiVersion: v1\nmetadata:\n  name: kube-flannel\n  labels:\n    pod-security.kubernetes.io/enforce: privileged\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: flannel\nrules:\n- apiGroups:\n  - \"\"\n  resources:\n  - pods\n  verbs:\n  - get\n- apiGroups:\n  - \"\"\n  resources:\n  - nodes\n  verbs:\n  - get\n  - list\n  - watch\n- apiGroups:\n  - \"\"\n  resources:\n  - nodes/status\n  verbs:\n  - patch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: flannel\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: flannel\nsubjects:\n- kind: ServiceAccount\n  name: flannel\n  namespace: kube-flannel\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: flannel\n  namespace: kube-flannel\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: kube-flannel-cfg\n  namespace: kube-flannel\n  labels:\n    tier: node\n    app: flannel\ndata:\n  cni-conf.json: |\n    {\n      \"name\": \"cbr0\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\": [\n        {\n          \"type\": \"flannel\",\n          \"delegate\": {\n            \"hairpinMode\": true,\n            \"isDefaultGateway\": true\n          }\n        },\n        {\n          \"type\": \"portmap\",\n          \"capabilities\": {\n            \"portMappings\": true\n          }\n        }\n      ]\n    }\n  net-conf.json: |\n    {\n      \"Network\": \"10.244.0.0/16\",\n      \"EnableNFTables\": false,\n      \"Backend\": {\n        \"Type\": \"vxlan\"\n      }\n    }\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: kube-flannel-ds\n  namespace: kube-flannel\n  labels:\n    tier: node\n    app: flannel\nspec:\n  selector:\n    matchLabels:\n      app: flannel\n  template:\n    metadata:\n      labels:\n        tier: node\n        app: flannel\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: kubernetes.io/os\n                operator: In\n                values:\n                - linux\n      hostNetwork: true\n      priorityClassName: system-node-critical\n      tolerations:\n      - operator: Exists\n        effect: NoSchedule\n      serviceAccountName: flannel\n      initContainers:\n      - name: install-cni-plugin\n        image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.0-flannel1\n        command:\n        - cp\n        args:\n        - -f\n        - /flannel\n        - /opt/cni/bin/flannel\n        volumeMounts:\n        - name: cni-plugin\n          mountPath: /opt/cni/bin\n      - name: install-cni\n        image: ghcr.io/flannel-io/flannel:v0.28.1\n        command:\n        - cp\n        args:\n        - -f\n        - /etc/kube-flannel/cni-conf.json\n        - /etc/cni/net.d/10-flannel.conflist\n        volumeMounts:\n        - name: cni\n          mountPath: /etc/cni/net.d\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n      containers:\n      - name: kube-flannel\n        image: ghcr.io/flannel-io/flannel:v0.28.1\n        command:\n        - /opt/bin/flanneld\n        args:\n        - --ip-masq\n        - --kube-subnet-mgr\n        resources:\n          requests:\n            cpu: \"100m\"\n            memory: \"50Mi\"\n        securityContext:\n          privileged: false\n          capabilities:\n            add: [\"NET_ADMIN\", \"NET_RAW\"]\n        env:\n        - name: POD_NAME\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.namespace\n        - name: EVENT_QUEUE_DEPTH\n          value: \"5000\"\n        - name: CONT_WHEN_CACHE_NOT_READY\n          value: \"false\"\n        volumeMounts:\n        - name: run\n          mountPath: /run/flannel\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n        - name: xtables-lock\n          mountPath: /run/xtables.lock\n      volumes:\n      - name: run\n        hostPath:\n          path: /run/flannel\n      - name: cni-plugin\n        hostPath:\n          path: /opt/cni/bin\n      - name: cni\n        hostPath:\n          path: /etc/cni/net.d\n      - name: flannel-cfg\n        configMap:\n          name: kube-flannel-cfg\n      - name: xtables-lock\n        hostPath:\n          path: /run/xtables.lock\n          type: FileOrCreate\n"
  },
  {
    "path": "Documentation/kustomization/kube-flannel/kustomization.yaml",
    "content": "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\ncommonLabels:\n  k8s-app: flannel\nresources:\n- kube-flannel.yml\nimages:\n- name: ghcr.io/flannel-io/flannel\n  newTag: v0.28.1\n"
  },
  {
    "path": "Documentation/minikube.yml",
    "content": "# This manifest is intended for dev work, so there are some differences from the \"normal\" manifest\n#   - no namespace (make kubectl simpler)\n#   - special image name (flannel-minikube)\n#   - never pull the image\n#   - host-gw backend (since vxlan doesn't work in minikube)\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: flannel\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n  name: kube-flannel-cfg\n  labels:\n    tier: node\n    app: flannel\ndata:\n  cni-conf.json: |\n    {\n      \"name\": \"cbr0\",\n      \"cniVersion\": \"0.3.1\",\n      \"type\": \"flannel\",\n      \"delegate\": {\n        \"hairpinMode\": true,\n        \"isDefaultGateway\": true\n      }\n    }\n  net-conf.json: |\n    {\n      \"Network\": \"10.33.0.0/16\",\n      \"Backend\": {\n        \"Type\": \"host-gw\"\n      }\n    }\n---\napiVersion: extensions/v1beta1\nkind: DaemonSet\nmetadata:\n  name: kube-flannel-ds\n  labels:\n    tier: node\n    app: flannel\nspec:\n  template:\n    metadata:\n      labels:\n        tier: node\n        app: flannel\n    spec:\n      hostNetwork: true\n      serviceAccountName: flannel\n      containers:\n      - name: kube-flannel\n        image: flannel/minikube\n        imagePullPolicy: Never\n        command:\n        - /opt/bin/flanneld\n        args:\n        - --ip-masq\n        - --kube-subnet-mgr\n        resources:\n          requests:\n            cpu: \"100m\"\n            memory: \"50Mi\"\n        securityContext:\n          privileged: true\n        env:\n        - name: POD_NAME\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.namespace\n        volumeMounts:\n        - name: run\n          mountPath: /run\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n      - name: install-cni\n        image: flannel/minikube\n        imagePullPolicy: Never\n        command: [ \"/bin/sh\", \"-c\", \"set -e -x; cat /etc/kube-flannel/cni-conf.json; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done\" ]\n        volumeMounts:\n        - name: cni\n          mountPath: /etc/cni/net.d\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n      volumes:\n        - name: run\n          hostPath:\n            path: /run\n        - name: cni\n          hostPath:\n            path: /etc/cni/net.d\n        - name: flannel-cfg\n          configMap:\n            name: kube-flannel-cfg\n"
  },
  {
    "path": "Documentation/netpol.md",
    "content": "# Network policy controller\n\nFrom v0.25.5 it is possible to deploy Flannel with https://github.com/kubernetes-sigs/kube-network-policies controller to provide a network policy controller within the Flannel CNI.\n\nWhen deployed with the Helm chart it is enough to enable the `netpol.enabled` value.\n```bash\nhelm install flannel --set netpol.enabled=true --namespace kube-flannel flannel/flannel\n```\n\nFlannel pod should start with an additional container and it is possible to configure Network policies.\nUse the kube-network-poilicies documentation to find additional info.\n"
  },
  {
    "path": "Documentation/reporting_bugs.md",
    "content": "# Reporting bugs\n\nIf any part of the flannel project has bugs or documentation mistakes, please let us know by [opening an issue][flannel-issue]. Before creating a bug report, please check that an issue reporting the same problem does not already exist.\n\nTo make the bug report accurate and easy to understand, please try to create bug reports that are:\n\n- Specific. Include as much details as possible: which version, what environment, what configuration, etc.\n\n- Reproducible. Include the steps to reproduce the problem. We understand some issues might be hard to reproduce, please includes the steps that might lead to the problem.\n\n- Isolated. Please try to isolate and reproduce the bug with minimum dependencies. It would significantly slow down the speed to fix a bug if too many dependencies are involved in a bug report. Debugging external systems that rely on flannel is out of scope, but we are happy to provide guidance in the right direction or help with using flannel itself.\n\n- Unique. Do not duplicate an existing bug report.\n\n- Scoped. One bug per report. Do not follow up with another bug inside one report.\n\nIt may be worthwhile to read [Elika Etemad’s article on filing good bug reports][filing-good-bugs] before creating a bug report.\n\nWe might ask for further information to locate a bug. A duplicated bug report will be closed.\n\n## Frequently asked questions\n\n### How to get a stack trace\n\n``` bash\n$ kill -QUIT $PID\n```\n\n### How to get flannel version\n\n``` bash\n$ flannel --version\n```\n\n[flannel-issue]: https://github.com/flannel-io/flannel/issues/new\n[filing-good-bugs]: http://fantasai.inkedblade.net/style/talks/filing-good-bugs/\n"
  },
  {
    "path": "Documentation/reservations.md",
    "content": "# Leases and Reservations\n\n## Leases\n\nWhen flannel starts up, it ensures that the host has a subnet lease. If there is\nan existing lease then it's used, otherwise one is assigned.\n\nLeases can be viewed by checking the contents of etcd. e.g.\n\n```\n$ export ETCDCTL_API=3\n$ etcdctl get /coreos.com/network/subnets --prefix --keys-only         \n/coreos.com/network/subnets/10.5.52.0-24\n$ etcdctl get /coreos.com/network/subnets/10.5.52.0-24\n/coreos.com/network/subnets/10.5.52.0-24\n{\"PublicIP\":\"192.168.64.3\",\"PublicIPv6\":null,\"BackendType\":\"vxlan\",\"BackendData\":{\"VNI\":1,\"VtepMAC\":\"c6:d2:32:6f:8f:44\"}}\n$ etcdctl lease list\nfound 1 leases\n694d854330fc5110\n$ etcdctl lease timetolive --keys 694d854330fc5110\nlease 694d854330fc5110 granted with TTL(86400s), remaining(74737s), attached keys([/coreos.com/network/subnets/10.5.52.0-24])\n```\n\nThis shows that there is a single lease (`10.5.52.0/24`) which will expire in 74737 seconds. flannel will attempt to renew the lease before it expires, but if flannel is not running for an extended period then the lease will be lost.\n\nThe `\"PublicIP\"` value is how flannel knows to reuse this lease when restarted. \nThis means that if the public IP changes, then the flannel subnet will change too.\n\nIn case a host is unable to renew its lease before the lease expires (e.g. a host takes a long time to restart and the timing lines up with when the lease would normally be renewed), flannel will then attempt to renew the last lease that it has saved in its subnet config file (which, unless specified, is located at `/var/run/flannel/subnet.env`)\n```bash\ncat /var/run/flannel/subnet.env\nFLANNEL_NETWORK=10.5.0.0/16\nFLANNEL_SUBNET=10.5.52.1/24\nFLANNEL_MTU=1450\nFLANNEL_IPMASQ=false\n```\nIn this case, if flannel fails to retrieve an existing lease from etcd, it will attempt to renew lease specified in `FLANNEL_SUBNET` (`10.5.52.1/24`).  It will only renew this lease if the subnet specified is valid for the current etcd network configuration otherwise it will allocate a new lease.\n\n## Reservations\n\nflannel also supports reservations for the subnet assigned to a host. Reservations\nallow a fixed subnet to be used for a given host.\n\nThe only difference between a lease and reservation is the etcd TTL value. Simply \nremoving the TTL from a lease will convert it to a reservation. e.g.\n\n```\n# update the value without any lease option (--lease). \n$ export ETCDCTL_API=3\n$ etcdctl put /coreos.com/network/subnets/10.5.1.0-24 $(etcdctl get /coreos.com/network/subnets/10.5.1.0-24)\n```\n"
  },
  {
    "path": "Documentation/running.md",
    "content": "# Running flannel\n\nOnce you have pushed configuration JSON to `etcd`, you can start `flanneld`. If you published your config at the default location, you can start `flanneld` with no arguments.\n\nFlannel will acquire a subnet lease, configure its routes based on other leases in the overlay network and start routing packets.\n\nIt will also monitor `etcd` for new members of the network and adjust the routes accordingly.\n\nAfter flannel has acquired the subnet and configured backend, it will write out an environment variable file (`/run/flannel/subnet.env` by default) with subnet address and MTU that it supports.\n\nFor more information on checking the IP range for a specific host, see [Leases and Reservations](https://github.com/flannel-io/flannel/blob/master/Documentation/reservations.md).\n\n## Multiple networks\n\nFlanneld does not support running multiple networks from a single daemon (it did previously as an experimental feature).\nHowever, it does support running multiple daemons on the same host with different configurations. The `-subnet-file` and `-etcd-prefix` options should be used to \"namespace\" the different daemons.\nFor example\n```\nflanneld -subnet-file /vxlan.env -etcd-prefix=/vxlan/network\n```\n\n## Running manually\n\n1. Download a `flannel` binary.\n```bash\nwget https://github.com/flannel-io/flannel/releases/latest/download/flanneld-amd64 && chmod +x flanneld-amd64\n```\n2. Run the binary.\n```bash\nsudo ./flanneld-amd64 # it will hang waiting to talk to etcd\n```\n3. Run `etcd`. Follow the instructions on the [etcd page](https://etcd.io/docs/v3.5/quickstart/), or, if you have docker just do\n```bash\ndocker run --rm --net=host quay.io/coreos/etcd\n```\n4. Observe that `flannel` can now talk to `etcd`, but can't find any config. So write some config. Either get `etcdctl` from the [etcd page](https://etcd.io/docs/v3.5/quickstart/), or use `docker` again.\n```bash\ndocker run --rm -e ETCDCTL_API=3 --net=host quay.io/coreos/etcd etcdctl put /coreos.com/network/config '{ \"Network\": \"10.5.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}'\n```\nNow `flannel` is running, it has created a VXLAN tunnel device on the host and written a subnet config file\n\n```bash\ncat /run/flannel/subnet.env\nFLANNEL_NETWORK=10.5.0.0/16\nFLANNEL_SUBNET=10.5.72.1/24\nFLANNEL_MTU=1450\nFLANNEL_IPMASQ=false\n```\nEach time flannel is restarted, it will attempt to access the `FLANNEL_SUBNET` value written in this subnet config file. This prevents each host from needing to update its network information in case a host is unable to renew its lease before it expires (e.g. a host was restarting during the time flannel would normally renew its lease).\n\nThe `FLANNEL_SUBNET` value is also only used if it is valid for the etcd network config. For instance, a `FLANNEL_SUBNET` value of `10.5.72.1/24` will not be used if the etcd network value is set to `10.6.0.0/16` since it is not within that network range.\n\nSubnet config value is `10.5.72.1/24`\n```bash\ncat /run/flannel/subnet.env\nFLANNEL_NETWORK=10.5.0.0/16\nFLANNEL_SUBNET=10.5.72.1/24\nFLANNEL_MTU=1450\nFLANNEL_IPMASQ=false\n```\netcd network value is `10.6.0.0/16`. Since `10.5.72.1/24` is outside of this network, a new lease will be allocated.\n```bash\nexport ETCDCTL_API=3\netcdctl get /coreos.com/network/config\n{ \"Network\": \"10.6.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}\n```\n\n## Interface selection\n\nFlannel uses the interface selected to register itself in the datastore.\n\nThe important options are:\n* `-iface string`: Interface to use (IP or name) for inter-host communication.\n* `-public-ip string`: IP accessible by other nodes for inter-host communication.\n\nThe combination of the defaults, the autodetection and these two flags ultimately result in the following being determined:\n* An interface (used for MTU detection and selecting the VTEP MAC in VXLAN).\n* An IP address for that interface.\n* A public IP that can be used for reaching this node. In `host-gw` it should match the interface address.\n\n## Making changes at runtime\n\nPlease be aware of the following flannel runtime limitations.\n* The datastore type cannot be changed.\n* The backend type cannot be changed. (It can be changed if you stop all workloads and restart all flannel daemons.)\n* You can change the subnetlen/subnetmin/subnetmax with a daemon restart. (Subnets can be changed with caution. If pods are already using IP addresses outside the new range they will stop working.)\n* The clusterwide network range cannot be changed (without downtime).\n\n## Docker integration\n\nDocker daemon accepts `--bip` argument to configure the subnet of the docker0 bridge.\nIt also accepts `--mtu` to set the MTU for docker0 and veth devices that it will be creating.\n\nBecause flannel writes out the acquired subnet and MTU values into a file, the script starting Docker can source in the values and pass them to Docker daemon:\n```bash\nsource /run/flannel/subnet.env\ndocker daemon --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} &\n```\n\nSystemd users can use `EnvironmentFile` directive in the `.service` file to pull in `/run/flannel/subnet.env`\n\nIf you want to leave default docker0 network as it is and instead create a new network that will be using flannel you do so like this:\n```bash\nsource /run/flannel/subnet.env\ndocker network create --attachable=true --subnet=${FLANNEL_SUBNET} -o \"com.docker.network.driver.mtu\"=${FLANNEL_MTU} flannel\n```\n\n## Running on Vagrant\n\nVagrant has a tendency to give the default interface (one with the default route) a non-unique IP (often 10.0.2.15).\n\nThis causes flannel to register multiple nodes with the same IP.\n\nTo work around this issue, use `--iface` option to specify the interface that has a unique IP.\n\n## Zero-downtime restarts\n\nWhen running with a backend other than `udp`, the kernel is providing the data path with `flanneld` acting as the control plane.\n\nAs such, `flanneld` can be restarted (even to do an upgrade) without disturbing existing flows.\n\nHowever in the case of `vxlan` backend, this needs to be done within a few seconds as ARP entries can start to timeout requiring the flannel daemon to refresh them.\n\nAlso, to avoid interruptions during restart, the configuration must not be changed (e.g. VNI, --iface values).\n"
  },
  {
    "path": "Documentation/tencentcloud-vpc-backend.md",
    "content": "# TencentCloud VPC Backend for Flannel\n\nThere are only two differences between the usage method and Alibaba Cloud:\n1. Tencent Cloud needs to create a routing table, while Alibaba Cloud creates a switch\n2. In network/config, backend-type is \"tencent-vpc\"\n\n\n"
  },
  {
    "path": "Documentation/troubleshooting.md",
    "content": "# Troubleshooting\n\n# General\n\n## Connectivity\nIn Docker v1.13 and later, the default iptables forwarding policy was changed to `DROP`. For more detail on the Docker change, see the Docker [documentation](https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#container-communication-between-hosts).\n\nThis problems manifests itself as connectivity problems between containers running on different hosts. To resolve it upgrade to the latest version of flannel.\n\n\n## Logging\nFlannel uses the `klog` library but only supports logging to stderr. The severity level can't be changed but the verbosity can be changed with the `-v` option. Flannel does not make extensive use of the verbosity level but increasing the value from `0` (the default) will result in some additional logs. To get the most detailed logs, use `-v=10`\n\n```\n-v value\n    \tlog level for V logs\n-vmodule value\n    \tcomma-separated list of pattern=N settings for file-filtered logging\n-log_backtrace_at value\n    \twhen logging hits line file:N, emit a stack trace\n```\n\nWhen running under systemd (e.g. on CoreOS Container Linux) the logs can be viewed with `journalctl -u flanneld`\n\nWhen flannel is running as a pod on Kubernetes, the logs can be viewed with `kubectl logs --namespace kube-flannel <POD_ID> -c kube-flannel`. You can find the pod IDs with `kubectl get pod --namespace kube-flannel -l app=flannel`\n\n## Interface selection and the public IP.\nMost backends require that each node has a unique \"public IP\" address. This address is chosen when flannel starts. Because leases are tied to the public address, if the address changes, flannel must be restarted.\n\nThe interface chosen and the public IP in use is logged out during startup, e.g.\n```\nI0629 14:28:35.866793    5522 main.go:386] Determining IP address of default interface\nI0629 14:28:35.866987    5522 main.go:399] Using interface with name enp62s0u1u2 and address 172.24.17.174\nI0629 14:28:35.867000    5522 main.go:412] Using 10.10.10.10 as external address\n```\n\n### Vagrant\nVagrant typically assigns two interfaces to all VMs. The first, for which all hosts are assigned the IP address `10.0.2.15`, is for external traffic that gets NATed.\n\nThis may lead to problems with flannel. By default, flannel selects the first interface on a host. This leads to all hosts thinking they have the same public IP address. To prevent this issue, pass the `--iface=eth1` flag to flannel so that the second interface is chosen.\n\n## NAT\nWhen the public IP is behind NAT, the UDP checksum fields of the VXLAN packets can be corrupted.\nIn that case, try running the following commands to avoid corrupted checksums:\n\n```bash\n/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off\n```\n\nTo automate the command above via udev, create `/etc/udev/rules.d/90-flannel.rules` as follows:\n\n```\nSUBSYSTEM==\"net\", ACTION==\"add|change|move\", ENV{INTERFACE}==\"flannel.1\", RUN+=\"/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off\"\n```\n\n<!--\nref:\n- https://github.com/flannel-io/flannel/issues/1279\n- https://github.com/kubernetes/kops/pull/9074\n- https://github.com/karmab/kcli/commit/b1a8eff658d17cf4e28162f0fa2c8b2b10e5ad00\n-->\n\n## Permissions\nDepending on the backend being used, flannel may need to run with super user permissions. Examples include creating VXLAN devices or programming routes.  If you see errors similar to the following, confirm that the user running flannel has the right permissions (or try running with `sudo)`.\n * `Error adding route...`\n * `Add L2 failed`\n * `Failed to set up IP Masquerade`\n * `Error registering network: operation not permitted`\n\n## Performance\n\n### Control plane\nFlannel is known to scale to a very large number of hosts. A delay in contacting pods in a newly created host may indicate control plane problems. Flannel doesn't need much CPU or RAM but the first thing to check would be that it has adequate resources available. Flannel is also reliant on the performance of the datastore, either etcd or the Kubernetes API server. Check that they are performing well.\n\n### Data plane\nFlannel relies on the underlying network so that's the first thing to check if you're seeing poor data plane performance.\n\nThere are two flannel specific choices that can have a big impact on performance\n1) The type of backend. For example, if encapsulation is used, `vxlan` will always perform better than `udp`. For maximum data plane performance, avoid encapsulation.\n2) The size of the MTU can have a large impact. To achieve maximum raw bandwidth, a network supporting a large MTU should be used. Flannel writes an MTU setting to the `subnet.env` file. This file is read by either the Docker daemon or the CNI flannel plugin which does the networking for individual containers. To troubleshoot, first ensure that the network interface that flannel is using has the right MTU. Then check that the correct MTU is written to the `subnet.env`. Finally, check that the containers have the correct MTU on their virtual ethernet device.\n\n\n## Firewalls\nWhen using `udp` backend, flannel uses UDP port 8285 for sending encapsulated packets.\n\nWhen using `vxlan` backend, kernel uses UDP port 8472 for sending encapsulated packets.\n\nMake sure that your firewall rules allow this traffic for all hosts participating in the overlay network.\n\nMake sure that your firewall rules allow traffic from pod network cidr visit your kubernetes master node.\n\n# Kubernetes Specific\nThe flannel kube subnet manager relies on the fact that each node already has a `podCIDR` defined.\n\nYou can check the podCidr for your nodes with one of the following two commands\n* `kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}'`\n* `kubectl get nodes -o template --template={{.spec.podCIDR}}`\n\nIf your nodes do not have a podCIDR, then either use the `--pod-cidr` kubelet command-line option or the `--allocate-node-cidrs=true --cluster-cidr=<cidr>` controller-manager command-line options.\n\nIf `kubeadm` is being used then pass `--pod-network-cidr=10.244.0.0/16` to `kubeadm init` which will ensure that all nodes are automatically assigned a `podCIDR`.\n\nIt's possible (but not generally recommended) to manually set the `podCIDR` to a fixed value for each node. The node subnet ranges must not overlap.\n* `kubectl patch node <NODE_NAME> -p '{\"spec\":{\"podCIDR\":\"<SUBNET>\"}}'`\n\n## Log messages\n\n* `failed to read net conf` - flannel expects to be able to read the net conf from \"/etc/kube-flannel/net-conf.json\". In the provided manifest, this is set up in the `kube-flannel-cfg` ConfigMap.\n* `error parsing subnet config` - The net conf is malformed. Double check that it has the right content and is valid JSON.\n* `node <NODE_NAME> pod cidr not assigned` - The node doesn't have a `podCIDR` defined. See above for more info.\n* `Failed to create SubnetManager: error retrieving pod spec for 'kube-system/kube-flannel-ds-abc123': the server does not allow access to the requested resource` - The kubernetes cluster has RBAC enabled. Run `https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-old-manifests/kube-flannel-rbac.yml`\n"
  },
  {
    "path": "Documentation/upgrade.md",
    "content": "# Upgrade\n\nFlannel upgrade/downgrade procedure\n \nThere are different ways of changing flannel version in the running cluster:\n \n## Remove old resources definitions and install a new one.\n* Pros: Cleanest way of managing resources of the flannel deployment and no manual validation required as long as no additional resources was created by administrators/operators\n* Cons: Massive networking outage within a cluster during the version change\n\n*1. Delete all the flannel resources using kubectl*\n```bash\nkubectl -n kube-flannel delete daemonset kube-flannel-ds\nkubectl -n kube-flannel delete configmap kube-flannel-cfg\nkubectl -n kube-flannel delete serviceaccount flannel\nkubectl delete clusterrolebinding.rbac.authorization.k8s.io flannel\nkubectl delete clusterrole.rbac.authorization.k8s.io flannel\nkubectl delete namespace kube-flannel\n```\n\n*2. Install the newer version of flannel and reboot the nodes*\n\n## On the fly version\n* Pros: Less disruptive way of changing flannel version, easier to do\n* Cons: Some version may have changes which can't be just replaced and may need resources cleanup and/or rename, manual resources comparison required\n\nIf the update is done from newer version as 0.20.2 it can be done using kubectl\n```bash\nkubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml\n```\nIn case of error on the labeling follow the previous way.\n\n## Using the helm repository\n\nFrom version 0.21.4 flannel is deployed on an helm repository at `https://flannel-io.github.io/flannel/` it will be possible to manage the update directly with helm.\n```bash\nhelm upgrade flannel --set podCidr=\"10.244.0.0/16\" --namespace kube-flannel flannel/flannel\n```\n"
  },
  {
    "path": "LICENSE",
    "content": "\n                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "Makefile",
    "content": ".PHONY: test unit-test e2e-test deps cover gofmt gofmt-fix license-check clean tar.gz release buildx-create-builder build-multi-arch release-manifest release-helm\n\n# Registry used for publishing images\nREGISTRY?=quay.io/coreos/flannel\nQEMU_VERSION=v3.0.0\nBASH_UNIT_VERSION=v2.3.0\n\n# Default tag and architecture. Can be overridden\nTAG?=$(shell git describe --tags --always)\nARCH?=amd64\n# Only enable CGO (and build the UDP backend) on AMD64\nifeq ($(ARCH),amd64)\n\tCGO_ENABLED=1\nelse\n\tCGO_ENABLED=0\nendif\n\n# Go version to use for builds\nGO_VERSION=1.24.7\n\n# K8s version used for Makefile helpers\nK8S_VERSION=1.32.5\n\nGOARM=7\n\n# These variables can be overridden by setting an environment variable.\nTEST_PACKAGES?=pkg/ip pkg/subnet pkg/subnet/etcd pkg/subnet/kube pkg/trafficmngr pkg/backend\nTEST_PACKAGES_EXPANDED=$(TEST_PACKAGES:%=github.com/flannel-io/flannel/%)\nPACKAGES?=$(TEST_PACKAGES)\nPACKAGES_EXPANDED=$(PACKAGES:%=github.com/flannel-io/flannel/%)\n\n### BUILDING\nclean:\n\trm -f dist/flanneld*\n\trm -f dist/*.aci\n\trm -f dist/*.docker\n\trm -f dist/*.tar.gz\n\trm -f dist/qemu-*\n\ndist/flanneld: $(shell find . -type f  -name '*.go')\n\tCGO_ENABLED=$(CGO_ENABLED) go build -o dist/flanneld \\\n\t  -ldflags '-s -w -X github.com/flannel-io/flannel/pkg/version.Version=$(TAG) -extldflags \"-static\"'\n\ndist/flanneld.exe: $(shell find . -type f  -name '*.go')\n\tCXX=x86_64-w64-mingw32-g++ CC=x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows go build -o dist/flanneld.exe \\\n\t  -ldflags '-s -w -X github.com/flannel-io/flannel/pkg/version.Version=$(TAG) -extldflags \"-static\"'\n\n# This will build flannel natively using golang image\ndist/flanneld-$(ARCH): deps dist/qemu-$(ARCH)-static\n\t# valid values for ARCH are [amd64 arm arm64 ppc64le s390x riscv64]\n\tdocker run --rm -e CGO_ENABLED=$(CGO_ENABLED) -e GOARCH=$(ARCH) -e GOCACHE=/go \\\n\t\t-u $(shell id -u):$(shell id -g) \\\n\t\t-v $(CURDIR)/dist/qemu-$(ARCH)-static:/usr/bin/qemu-$(ARCH)-static \\\n\t\t-v $(CURDIR):/go/src/github.com/flannel-io/flannel:ro \\\n\t\t-v $(CURDIR)/dist:/go/src/github.com/flannel-io/flannel/dist \\\n\t\tgolang:$(GO_VERSION) /bin/bash -c '\\\n\t\tcd /go/src/github.com/flannel-io/flannel && \\\n\t\tmake -e dist/flanneld && \\\n\t\tmv dist/flanneld dist/flanneld-$(ARCH)'\n\n## Create a docker image on disk for a specific arch and tag\nimage:\tdeps dist/flanneld-$(TAG)-$(ARCH).docker\ndist/flanneld-$(TAG)-$(ARCH).docker:\n\tdocker buildx build -f images/Dockerfile --platform=$(ARCH) --build-arg TAG=$(TAG) -t $(REGISTRY):$(TAG)-$(ARCH) --load .\n\tdocker save -o dist/flanneld-$(TAG)-$(ARCH).docker $(REGISTRY):$(TAG)-$(ARCH)\n\n# amd64 gets an image without the suffix too (i.e. it's the default)\nifeq ($(ARCH),amd64)\n\tdocker build -f images/Dockerfile --platform=$(ARCH) --build-arg TAG=$(TAG) -t $(REGISTRY):$(TAG) .\nendif\n\n### TESTING\ntest: license-check gofmt deps verify-modules\n\tmake unit-test\n\n\t# Test the docker-opts script\n\tcd dist; ./mk-docker-opts_tests.sh\n\n\t# Run the functional tests\n\tmake e2e-test\n\nunit-test: \n\t# Run the unit tests\n\t# NET_ADMIN capacity is required to do some network operation\n\t# SYS_ADMIN capacity is required to create network namespace\n\tdocker run --cap-add=NET_ADMIN \\\n\t\t--cap-add=SYS_ADMIN --rm \\\n\t\t-v $(shell pwd):/go/src/github.com/flannel-io/flannel \\\n\t\tgolang:$(GO_VERSION) \\\n\t\t/bin/bash -c 'cd /go/src/github.com/flannel-io/flannel && go test -v -cover -timeout 5m $(TEST_PACKAGES_EXPANDED)'\n\ne2e-test: bash_unit dist/flanneld-e2e-$(TAG)-$(ARCH).docker\n\t$(MAKE) -C images/iperf3 ARCH=$(ARCH)\n\tFLANNEL_DOCKER_IMAGE=$(REGISTRY):$(TAG)-$(ARCH) ./bash_unit dist/functional-test.sh\n\tFLANNEL_DOCKER_IMAGE=$(REGISTRY):$(TAG)-$(ARCH) ./bash_unit dist/functional-test-k8s.sh\n\nk3s-e2e-test: bash_unit dist/flanneld-e2e-$(TAG)-$(ARCH).docker\n\t$(MAKE) -C images/iperf3 ARCH=$(ARCH)\n\t./bash_unit ./e2e/run-e2e-tests.sh\n\ncover:\n\t# A single package must be given - e.g. 'PACKAGES=pkg/ip make cover'\n\tgo test -coverprofile cover.out $(PACKAGES_EXPANDED)\n\tgo tool cover -html=cover.out\n\nlicense-check:\n\t# run license-check script\n\tdist/license-check.sh\n\n# Throw an error if gofmt finds problems.\n# \"read\" will return a failure return code if there is no output. This is inverted wth the \"!\"\ngofmt:\n\t# Running gofmt... \n\tdocker run --rm -e CGO_ENABLED=$(CGO_ENABLED) -e GOARCH=$(ARCH) \\\n\t\t-u $(shell id -u):$(shell id -g) \\\n\t\t-v $(CURDIR):/go/src/github.com/flannel-io/flannel \\\n\t\t-v $(CURDIR)/dist:/go/src/github.com/flannel-io/flannel/dist \\\n\t\tgolang:$(GO_VERSION) /bin/bash -c '\\\n\t\tcd /go/src/github.com/flannel-io/flannel && \\\n\t\t! gofmt -d $(PACKAGES) 2>&1 | read'\n\nverify-modules:\n\t# Running verify-modules...\n\tdocker run --rm -e CGO_ENABLED=$(CGO_ENABLED) -e GOARCH=$(ARCH) \\\n                -u $(shell id -u):$(shell id -g) \\\n                -v $(CURDIR):/go/src/github.com/flannel-io/flannel \\\n                -v $(CURDIR)/dist:/go/src/github.com/flannel-io/flannel/dist \\\n                golang:$(GO_VERSION) /bin/bash -c '\\\n                cd /go/src/github.com/flannel-io/flannel && \\\n\t\t!go mod tidy 2>&1|read && \\\n\t\t!go vet 2>&1|read'\n\n\nbash_unit:\n\twget https://raw.githubusercontent.com/pgrange/bash_unit/$(BASH_UNIT_VERSION)/bash_unit\n\tchmod +x bash_unit\n\n# This will build flannel natively using golang image\ndist/flanneld-e2e-$(TAG)-$(ARCH).docker:\nifneq ($(ARCH),amd64)\n\t$(MAKE) dist/qemu-$(ARCH)-static\nendif\n\t# valid values for ARCH are [amd64 arm arm64 ppc64le s390x riscv64]\n\tdocker run --rm -e GOARM=$(GOARM) -e CGO_ENABLED=$(CGO_ENABLED) -e GOCACHE=/go \\\n\t\t-u $(shell id -u):$(shell id -g) \\\n\t\t-v $(CURDIR):/go/src/github.com/flannel-io/flannel:ro \\\n\t\t-v $(CURDIR)/dist:/go/src/github.com/flannel-io/flannel/dist \\\n\t\tgolang:$(GO_VERSION) /bin/bash -c '\\\n\t\tcd /go/src/github.com/flannel-io/flannel && \\\n\t\tmake -e dist/flanneld && \\\n\t\tmv dist/flanneld dist/flanneld-$(ARCH)'\n\tdocker build -f images/Dockerfile --platform=$(ARCH) --build-arg TAG=$(TAG) -t $(REGISTRY):$(TAG)-$(ARCH) .\n\n# Make a release after creating a tag\n# To build cross platform Docker images, the qemu-static binaries are needed. On ubuntu \"apt-get install  qemu-user-static\"\nrelease: tar.gz dist/qemu-s390x-static dist/qemu-ppc64le-static dist/qemu-arm64-static dist/qemu-arm-static dist/qemu-riscv64-static\n\tARCH=amd64 make dist/flanneld-$(TAG)-amd64.docker\n\tARCH=arm make dist/flanneld-$(TAG)-arm.docker\n\tARCH=arm64 make dist/flanneld-$(TAG)-arm64.docker\n\tARCH=ppc64le make dist/flanneld-$(TAG)-ppc64le.docker\n\tARCH=s390x make dist/flanneld-$(TAG)-s390x.docker\n\tARCH=riscv64 make dist/flanneld-$(TAG)-riscv64.docker\n\t@echo \"Everything should be built for $(TAG)\"\n\t@echo \"Add all flanneld-* and *.tar.gz files from dist/ to the Github release\"\n\t@echo \"Use make docker-push-all to push the images to a registry\"\n\nrelease-manifest:\n\tsed -i 's/^  newTag: .*/  newTag: $(TAG)/' Documentation/kustomization/kube-flannel/kustomization.yaml\n\tkubectl kustomize ./Documentation/kustomization/kube-flannel/ > dist/kube-flannel.yml\n\nrelease-helm:\n\tsed -i '0,/^    tag: .*/s//    tag: $(TAG)/' ./chart/kube-flannel/values.yaml\n\thelm package ./chart/kube-flannel/ --destination chart/ --version $(TAG) --app-version $(TAG)\n\tcp chart/flannel-$(TAG).tgz dist/flannel.tgz\n\tmv chart/flannel-$(TAG).tgz chart/flannel.tgz\n\twget https://flannel-io.github.io/flannel/index.yaml -O chart/index.yaml\n\thelm repo index --merge chart/index.yaml --url https://github.com/flannel-io/flannel/releases/download/$(TAG)/ chart/\n\ndist/qemu-%-static:\n\tif [ \"$(@F)\" = \"qemu-amd64-static\" ]; then \\\n\t\twget -O dist/qemu-amd64-static https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-x86_64-static; \\\n\telif [ \"$(@F)\" = \"qemu-arm64-static\" ]; then \\\n\t\twget -O dist/qemu-arm64-static https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-aarch64-static; \\\n\telse \\\n\t\twget -O dist/$(@F) https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/$(@F); \\\n\tfi \n\n## Build a .tar.gz for the amd64 ppc64le arm arm64 riscv64 flanneld binary\ntar.gz:\n\tARCH=amd64 make dist/flanneld-amd64\n\ttar --transform='flags=r;s|-amd64||' -zcvf dist/flannel-$(TAG)-linux-amd64.tar.gz -C dist flanneld-amd64 mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-linux-amd64.tar.gz\n\tARCH=amd64 make dist/flanneld.exe\n\ttar --transform='flags=r;s|-amd64||' -zcvf dist/flannel-$(TAG)-windows-amd64.tar.gz -C dist flanneld.exe mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-windows-amd64.tar.gz\n\tARCH=ppc64le make dist/flanneld-ppc64le\n\ttar --transform='flags=r;s|-ppc64le||' -zcvf dist/flannel-$(TAG)-linux-ppc64le.tar.gz -C dist flanneld-ppc64le mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-linux-ppc64le.tar.gz\n\tARCH=arm make dist/flanneld-arm\n\ttar --transform='flags=r;s|-arm||' -zcvf dist/flannel-$(TAG)-linux-arm.tar.gz -C dist flanneld-arm mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-linux-arm.tar.gz\n\tARCH=arm64 make dist/flanneld-arm64\n\ttar --transform='flags=r;s|-arm64||' -zcvf dist/flannel-$(TAG)-linux-arm64.tar.gz -C dist flanneld-arm64 mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-linux-arm64.tar.gz\n\tARCH=s390x make dist/flanneld-s390x\n\ttar --transform='flags=r;s|-s390x||' -zcvf dist/flannel-$(TAG)-linux-s390x.tar.gz -C dist flanneld-s390x mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-linux-s390x.tar.gz\n\tARCH=riscv64 make dist/flanneld-riscv64\n\ttar --transform='flags=r;s|-riscv64||' -zcvf dist/flannel-$(TAG)-linux-riscv64.tar.gz -C dist flanneld-riscv64 mk-docker-opts.sh ../README.md\n\ttar -tvf dist/flannel-$(TAG)-linux-riscv64.tar.gz\n\ninstall:\n\t# This is intended as just a developer convenience to help speed up non-containerized builds\n\t# It is NOT how you install flannel\n\tCGO_ENABLED=$(CGO_ENABLED) go install -v github.com/flannel-io/flannel\n\ndeps:\n\tgo mod tidy\n\tgo mod vendor\n\nbuildx-create-builder:\n\tdocker buildx create --name mybuilder --use --bootstrap\n\nbuild-multi-arch:\n\tdocker buildx build  --platform linux/amd64,linux/arm64,linux/arm,linux/s390x,linux/ppc64le,linux/riscv64 -t $(REGISTRY):$(TAG) -f images/Dockerfile --build-arg TAG=$(TAG) -o type=oci,dest=dist/flannel_oci.tar --progress plain .\n\n"
  },
  {
    "path": "OWNERS",
    "content": "reviewers:\n  - manuelbuil (Manuel Buil)\n  - mgfritch (Michael Fritch)\n  - rbrtbnfgl (Roberto Bonafiglia)\n  - thomasferrandiz (Thomas Ferrandiz)\napprovers:\n  - manuelbuil\n  - mgfritch \n  - rbrtbnfgl \n  - thomasferrandiz\n"
  },
  {
    "path": "README.md",
    "content": "# flannel\n\n![flannel Logo](logos/flannel-horizontal-color.png)\n\n![Build Status](https://github.com/flannel-io/flannel/actions/workflows/build.yaml/badge.svg?branch=master)\n[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/flannel-io/flannel/badge)](https://scorecard.dev/viewer/?uri=github.com%2Fflannel-io%2Fflannel)\n\nFlannel is a simple and easy way to configure a layer 3 network fabric designed for Kubernetes.\n\n## How it works\n\nFlannel runs a small, single binary agent called `flanneld` on each host, and is responsible for allocating a subnet lease to each host out of a larger, preconfigured address space.\nFlannel uses either the Kubernetes API or [etcd][etcd] directly to store the network configuration, the allocated subnets, and any auxiliary data (such as the host's public IP).\nPackets are forwarded using one of several [backend mechanisms][backends] including VXLAN and various cloud integrations.\n\n### Networking details\n\nPlatforms like Kubernetes assume that each container (pod) has a unique, routable IP inside the cluster.\nThe advantage of this model is that it removes the port mapping complexities that come from sharing a single host IP.\n\nFlannel is responsible for providing a layer 3 IPv4 network between multiple nodes in a cluster. Flannel does not control how containers are networked to the host, only how the traffic is transported between hosts. However, flannel does provide a CNI plugin for Kubernetes and a guidance on integrating with Docker.\n\nFlannel is focused on networking. For network policy, other projects such as [Calico][calico] can be used.\n\n## Getting started on Kubernetes\n\nThe easiest way to deploy flannel with Kubernetes is to use one of several deployment tools and distributions that network clusters with flannel by default. For example, [K3s][k3s] sets up flannel in the Kubernetes clusters it creates using the open source [K3s Installer][k3s-installer] to drive the setup process.\n\nThough not required, it's recommended that flannel uses the Kubernetes API as its backing store which avoids the need to deploy a discrete `etcd` cluster for `flannel`. This `flannel` mode is known as the *kube subnet manager*.\n\n### Deploying flannel manually\n\nFlannel can be added to any existing Kubernetes cluster though it's simplest to add `flannel` before any pods using the pod network have been started.\n\nFor Kubernetes v1.17+\n\n#### Deploying Flannel with kubectl\n\n```bash\nkubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml\n```\n\nIf you use custom `podCIDR` (not `10.244.0.0/16`) you first need to download the above manifest and modify the network to match your one.\n\n#### Deploying Flannel with Helm\n\n```bash\n# Needs manual creation of namespace to avoid Helm error\nkubectl create ns kube-flannel\nkubectl label --overwrite ns kube-flannel pod-security.kubernetes.io/enforce=privileged\n\nhelm repo add flannel https://flannel-io.github.io/flannel/\nhelm install flannel --set podCidr=\"10.244.0.0/16\" --namespace kube-flannel flannel/flannel\n```\n\nSee [Kubernetes](Documentation/kubernetes.md) for more details.\n\nIn case a firewall is configured ensure to enable the right port used by the configured [backend][backends].\n\nFlannel uses `portmap` as CNI network plugin by default; when deploying Flannel ensure that the [CNI Network plugins][Network-plugins] are installed in `/opt/cni/bin` the latest binaries can be downloaded with the following commands:\n\n```bash\nARCH=$(uname -m)\n  case $ARCH in\n    armv7*) ARCH=\"arm\";;\n    aarch64) ARCH=\"arm64\";;\n    x86_64) ARCH=\"amd64\";;\n  esac\nmkdir -p /opt/cni/bin\ncurl -O -L https://github.com/containernetworking/plugins/releases/download/v1.7.1/cni-plugins-linux-$ARCH-v1.7.1.tgz\ntar -C /opt/cni/bin -xzf cni-plugins-linux-$ARCH-v1.7.1.tgz\n```\n\nFlannel requires the br_netfilter module to start and from version 1.30 kubeadm doesn't check if the module is installed and Flannel will not rightly start in case the module is missing.\n\n## Getting started on Docker\n\nflannel is also widely used outside of Kubernetes. When deployed outside of Kubernetes, etcd is always used as the datastore. For more details integrating flannel with Docker see [Running](Documentation/running.md)\n\n## Documentation\n\n- [Building (and releasing)](Documentation/building.md)\n- [Configuration](Documentation/configuration.md)\n- [Backends](Documentation/backends.md)\n- [Running](Documentation/running.md)\n- [Troubleshooting](Documentation/troubleshooting.md)\n- [Projects integrating with flannel](Documentation/integrations.md)\n\n## Contact\n\n- Slack:\n  - #k3s on [Rancher Users Slack](https://slack.rancher.io)\n  - #flannel-users on [Calico Users Slack](https://slack.projectcalico.org)\n- Planning/Roadmap: [milestones][milestones], [roadmap][roadmap]\n- Bugs: [issues][flannel-issues]\n\n## Community Meeting\n\nThe Flannel Maintainer Community runs a meeting on the third Thursday of each month at 8:30 AM PST (16:30 UTC). This meeting is used to discuss issues, open pull requests, and other topics related to Flannel should the need arise.\n\nThe meeting agenda and Teams link can be found here: [Flannel Community Meeting Agenda](https://docs.google.com/document/d/1kPMMFDhljWL8_CUZajrfL8Q9sdntd9vvUpe-UGhX5z8)\n\n## Contributing\n\nSee [CONTRIBUTING][contributing] for details on submitting patches and the contribution workflow.\n\n## Reporting bugs & security vulnerabilities\n\nSee [reporting bugs][reporting] for details about reporting any issues.\n\nFor security issues, please first check our [security policy](SECURITY.md).\n\n## Licensing\n\nFlannel is under the Apache 2.0 license. See the [LICENSE][license] file for details.\n\n[calico]: http://www.projectcalico.org\n[etcd]: https://go.etcd.io/etcd/v3\n[contributing]: CONTRIBUTING.md\n[license]: https://github.com/flannel-io/flannel/blob/master/LICENSE\n[milestones]: https://github.com/flannel-io/flannel/milestones\n[flannel-issues]: https://github.com/flannel-io/flannel/issues\n[backends]: Documentation/backends.md\n[roadmap]: https://github.com/kubernetes/kubernetes/milestones\n[reporting]: Documentation/reporting_bugs.md\n[k3s-installer]: https://github.com/k3s-io/k3s/#quick-start---install-script\n[k3s]: https://k3s.io/\n[Network-plugins]: https://github.com/containernetworking/plugins\n"
  },
  {
    "path": "SECURITY.md",
    "content": "# Security Policy\n\n## Supported Versions\n\nThe flannel project maintains security fixes for the **latest release** only.\nOlder releases are not actively patched. Users are encouraged to stay on the\nlatest stable release.\n\n| Version        | Supported |\n|----------------|-----------|\n| Latest stable  | ✅ Yes    |\n| Older versions | ❌ No     |\n\n## Reporting a Vulnerability\n\nThe flannel maintainers take security vulnerabilities seriously and appreciate\nresponsible disclosure.\n\n**Please do not report security vulnerabilities through public GitHub issues.**\n\nTo report a vulnerability, use **GitHub private vulnerability reporting**:\nhttps://github.com/flannel-io/flannel/security/advisories/new\n\nPlease include as much detail as possible in your report:\n\n- A description of the vulnerability and its potential impact\n- Steps to reproduce the issue\n- Any suggested mitigations or patches\n\n## Disclosure Policy\n\nWe follow a **coordinated disclosure** process:\n\n1. You report the vulnerability privately via GitHub's private vulnerability reporting.\n2. The maintainers will acknowledge receipt of your report within **7 days**.\n3. The maintainers will investigate and aim to produce a fix within **90 days**\n   of the initial report, depending on severity and complexity.\n4. A security advisory and patched release will be published simultaneously.\n5. You are credited in the advisory (unless you prefer to remain anonymous).\n\nIf a vulnerability is not resolved within 90 days, we encourage reporters to\ndisclose publicly while coordinating with the maintainers to minimize user risk.\n\n## Embargoed Vulnerability Announcements\n\nTo receive advance notifications of embargoed security vulnerabilities before\npublic disclosure, subscribe to the flannel distributors mailing list:\n\n**[flannel-distributors-announce@googlegroups.com](https://groups.google.com/g/flannel-distributors-announce)**\n\nThis list is intended for distributors and downstream consumers of flannel who\nneed early access to security information to prepare patches or advisories.\n\n## Security Advisories\n\nPublished security advisories for flannel can be found at:\nhttps://github.com/flannel-io/flannel/security/advisories\n\n## Scope\n\nThe following are considered in scope for vulnerability reports:\n\n- The `flanneld` daemon and its backends (VXLAN, host-gw, WireGuard, etc.)\n- The CNI plugin\n- The flannel container image (e.g. secrets exposure, privilege escalation)\n- The release workflow and supply chain (e.g. tampered artifacts)\n\nThe following are generally **out of scope**:\n\n- Vulnerabilities in upstream dependencies (please report those upstream)\n- Issues requiring physical access to the host\n- Social engineering attacks\n\n## Security-Related Configuration\n\nFlannel runs as a privileged daemonset with access to the host network. Users\nare advised to:\n\n- Follow the [principle of least privilege](https://kubernetes.io/docs/concepts/security/rbac-good-practices/) when deploying flannel\n- Keep flannel updated to the latest release\n- Review the [flannel documentation](https://github.com/flannel-io/flannel/blob/master/Documentation/) for secure deployment guidance\n"
  },
  {
    "path": "chart/README.md",
    "content": "Flannel Helm Repository\n"
  },
  {
    "path": "chart/kube-flannel/.helmignore",
    "content": "tests/\n.helmignore\n"
  },
  {
    "path": "chart/kube-flannel/Chart.yaml",
    "content": "apiVersion: v1\nappVersion: v0.28.1\ndescription: Install Flannel Network Plugin.\nkeywords:\n- Flannel\nname: flannel\nsources:\n- https://github.com/flannel-io/flannel\nversion: v0.28.1\n"
  },
  {
    "path": "chart/kube-flannel/templates/config.yaml",
    "content": "kind: ConfigMap\napiVersion: v1\nmetadata:\n  name: kube-flannel-cfg\n  namespace: {{ .Release.Namespace }}\n  labels:\n    tier: node\n    app: flannel\ndata:\n  cni-conf.json: {{ .Values.flannel.cniConf | toJson }}\n  net-conf.json: |\n    {\n{{- if .Values.podCidr }}\n      \"Network\": {{ .Values.podCidr | quote }},\n{{- else }}\n      \"EnableIPv4\": false,\n{{- end }}\n{{- if .Values.podCidrv6 }}\n      \"IPv6Network\": {{ .Values.podCidrv6 | quote }},\n      \"EnableIPv6\": true,\n{{- end }}\n{{- if .Values.flannel.enableNFTables }}\n      \"EnableNFTables\": true,\n{{- end }}\n      \"Backend\": {\n{{- if eq .Values.flannel.backend \"vxlan\" }}\n{{- if .Values.flannel.backendPort }}\n        \"Port\": {{ .Values.flannel.backendPort }},\n{{- end }}\n{{- if .Values.flannel.vni }}\n        \"VNI\": {{ .Values.flannel.vni }},\n{{- end }}\n{{- if .Values.flannel.GBP }}\n        \"GBP\": {{ .Values.flannel.GBP }},\n{{- end }}\n{{- if .Values.flannel.directRouting }}\n        \"DirectRouting\": {{ .Values.flannel.directRouting }},\n{{- end }}\n{{- if .Values.flannel.mtu }}\n        \"MTU\": {{ .Values.flannel.mtu }},\n{{- end }}\n{{- if .Values.flannel.macPrefix }}\n        \"MacPrefix\": {{ .Values.flannel.macPrefix | quote }},\n{{- end }}\n        \"Type\": {{ .Values.flannel.backend | quote }}\n{{- else if eq .Values.flannel.backend \"wireguard\" }}\n{{- if .Values.flannel.backendPort }}\n        \"ListenPort\": {{ .Values.flannel.backendPort }},\n{{- end }}\n{{- if .Values.flannel.backendPortv6 }}\n        \"ListenPortV6\": {{ .Values.flannel.backendPortv6 }},\n{{- end }}\n{{- if .Values.flannel.psk }}\n        \"PSK\": {{ .Values.flannel.psk | quote }},\n{{- end }}\n{{- if .Values.flannel.mtu }}\n        \"MTU\": {{ .Values.flannel.mtu }},\n{{- end }}\n{{- if .Values.flannel.tunnelMode }}\n        \"Mode\": {{ .Values.flannel.tunnelMode }},\n{{- end }}\n{{- if .Values.flannel.keepaliveInterval }}\n        \"PersistentKeepaliveInterval\": {{ .Values.flannel.keepaliveInterval }},\n{{- end }}\n        \"Type\": {{ .Values.flannel.backend | quote }}\n{{- else if eq .Values.flannel.backend \"udp\" }}\n{{- if .Values.flannel.backendPort }}\n        \"Port\": {{ .Values.flannel.backendPort }},\n{{- end }}\n        \"Type\": {{ .Values.flannel.backend | quote }}\n{{- else }}\n        \"Type\": {{ .Values.flannel.backend | quote }}\n{{- end }}\n      }\n    }\n"
  },
  {
    "path": "chart/kube-flannel/templates/daemonset.yaml",
    "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: kube-flannel-ds\n  namespace: {{ .Release.Namespace }}\n  labels:\n    tier: node\n    app: flannel\nspec:\n  selector:\n    matchLabels:\n      app: flannel\n  template:\n    metadata:\n      labels:\n        tier: node\n        app: flannel\n    spec:\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: kubernetes.io/os\n                operator: In\n                values:\n                - linux\n      hostNetwork: true\n      priorityClassName: system-node-critical\n      {{- with .Values.flannel.tolerations }}\n      tolerations:\n        {{- toYaml . | trim | nindent 8 }}\n      {{- end }}\n      {{- with .Values.flannel.nodeSelector }}\n      nodeSelector:\n        {{- toYaml . | nindent 8 }}\n      {{- end }}\n      serviceAccountName: flannel\n      initContainers:\n      - name: install-cni-plugin\n        image: {{ .Values.flannel.image_cni.repository }}:{{ .Values.flannel.image_cni.tag }}\n        command:\n        - cp\n        args:\n        - -f\n        - /flannel\n        - /opt/cni/bin/flannel\n        volumeMounts:\n        - name: cni-plugin\n          mountPath: /opt/cni/bin\n      {{- if not .Values.flannel.skipCNIConfigInstallation }}\n      - name: install-cni\n        image: {{ .Values.flannel.image.repository }}:{{ .Values.flannel.image.tag }}\n        command:\n        - cp\n        args:\n        - -f\n        - /etc/kube-flannel/cni-conf.json\n        - /etc/cni/net.d/10-flannel.conflist\n        volumeMounts:\n        - name: cni\n          mountPath: /etc/cni/net.d\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n      {{- end }}\n      containers:\n      - name: kube-flannel\n        image: {{ .Values.flannel.image.repository }}:{{ .Values.flannel.image.tag }}\n        command:\n        - \"/opt/bin/flanneld\"\n        {{- range .Values.flannel.args }}\n        - {{ . | quote }}\n        {{- end }}\n        {{- with .Values.flannel.resources }}\n        resources:\n          {{- toYaml . | trim | nindent 10 }}\n        {{- end }}\n        securityContext:\n          privileged: false\n          capabilities:\n            add: [\"NET_ADMIN\", \"NET_RAW\"]\n        env:\n        - name: POD_NAME\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.name\n        - name: POD_NAMESPACE\n          valueFrom:\n            fieldRef:\n              fieldPath: metadata.namespace\n        - name: EVENT_QUEUE_DEPTH\n          value: \"5000\"\n        - name: CONT_WHEN_CACHE_NOT_READY\n          value: \"false\"\n        volumeMounts:\n        - name: run\n          mountPath: /run/flannel\n        - name: flannel-cfg\n          mountPath: /etc/kube-flannel/\n        - name: xtables-lock\n          mountPath: /run/xtables.lock\n{{- if .Values.netpol.enabled }}\n      - name: kube-network-policies\n        image: {{ .Values.netpol.image.repository }}:{{ .Values.netpol.image.tag }}\n        command:\n        - \"/bin/netpol\"\n        {{- range .Values.netpol.args }}\n        - {{ . | quote }}\n        {{- end }}\n        env:\n        - name: MY_NODE_NAME\n          valueFrom:\n            fieldRef:\n              fieldPath: spec.nodeName\n        volumeMounts:\n        - name: lib-modules\n          mountPath: /lib/modules\n          readOnly: true\n        resources:\n          requests:\n            cpu: \"100m\"\n            memory: \"50Mi\"\n        securityContext:\n          privileged: true\n          capabilities:\n            add: [\"NET_ADMIN\"]\n{{- end }}\n      volumes:\n{{- if .Values.netpol.enabled }}\n      - name: lib-modules\n        hostPath:\n          path: /lib/modules\n{{- end }}\n      - name: run\n        hostPath:\n          path: /run/flannel\n      - name: cni-plugin\n        hostPath:\n          path: {{ .Values.flannel.cniBinDir }}\n      - name: cni\n        hostPath:\n          path: {{ .Values.flannel.cniConfDir }}\n      - name: flannel-cfg\n        configMap:\n          name: kube-flannel-cfg\n      - name: xtables-lock\n        hostPath:\n          path: /run/xtables.lock\n          type: FileOrCreate\n      {{- if .Values.global.imagePullSecrets }}\n      imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 6 }}\n      {{- end }}\n"
  },
  {
    "path": "chart/kube-flannel/templates/rbac.yaml",
    "content": "kind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: flannel\nrules:\n- apiGroups:\n  - \"\"\n  resources:\n  - pods\n  - nodes\n  - namespaces\n  verbs:\n  - get\n  - list\n  - watch\n- apiGroups:\n  - \"\"\n  resources:\n  - nodes/status\n  verbs:\n  - patch\n{{- if .Values.netpol.enabled }}\n- apiGroups:\n  - \"networking.k8s.io\"\n  resources:\n  - networkpolicies\n  verbs:\n  - list\n  - watch\n- apiGroups:\n  - \"policy.networking.k8s.io\"\n  resources:\n  - adminnetworkpolicies\n  - baselineadminnetworkpolicies\n  verbs:\n  - list\n  - watch\n{{- end }}\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: flannel\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: flannel\nsubjects:\n- kind: ServiceAccount\n  name: flannel\n  namespace: {{ .Release.Namespace }}\n"
  },
  {
    "path": "chart/kube-flannel/templates/serviceaccount.yaml",
    "content": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: flannel\n  namespace: {{ .Release.Namespace }}\n"
  },
  {
    "path": "chart/kube-flannel/tests/daemonset_test.yaml",
    "content": "suite: test daemonset\ntemplates:\n  - daemonset.yaml\ntests:\n  - it: should have the correct api version\n    asserts:\n      - isKind:\n          of: DaemonSet\n      - isAPIVersion:\n          of: apps/v1\n      - equal:\n          path: metadata.name\n          value: kube-flannel-ds\n\n  - it: should have the correct image\n    set:\n      flannel.image.repository: cicker.io/flannel/flannel\n      flannel.image.tag: v555.44.3\n    asserts:\n      - equal:\n          path: spec.template.spec.containers[0].image\n          value: cicker.io/flannel/flannel:v555.44.3\n      - equal:\n          path: spec.template.spec.initContainers[1].image\n          value: cicker.io/flannel/flannel:v555.44.3\n\n  - it: should have the correct cni image\n    set:\n      flannel.image_cni.repository: cicker.io/flannel/flannel-cni-plugin\n      flannel.image_cni.tag: v777.66.5\n    asserts:\n      - equal:\n          path: spec.template.spec.initContainers[0].image\n          value: cicker.io/flannel/flannel-cni-plugin:v777.66.5\n\n  - it: should have the correct args\n    set:\n      flannel.args:\n        - \"--ip-masq\"\n        - \"--kube-subnet-mgr\"\n    asserts:\n      - equal:\n          path: spec.template.spec.containers[0].command\n          value:\n            - \"/opt/bin/flanneld\"\n            - \"--ip-masq\"\n            - \"--kube-subnet-mgr\"\n\n  - it: should have the correct image pull secrets\n    set:\n      global.imagePullSecrets:\n        - name: \"a-test-secret\"\n    asserts:\n      - equal:\n          path: spec.template.spec.imagePullSecrets\n          value:\n            - name: \"a-test-secret\"\n\n  - it: should add the install-cni init container when default values are used\n    asserts:\n      - contains:\n          path: spec.template.spec.initContainers\n          content:\n            name: install-cni\n          any: true\n\n  - it: should not add the install-cni init container when skipCNIConfigInstallation is set\n    set:\n      flannel.skipCNIConfigInstallation: true\n    asserts:\n      - notContains:\n          path: spec.template.spec.initContainers\n          content:\n            name: install-cni\n          any: true\n\n  - it: should render nodeSelector when set\n    set:\n      flannel.nodeSelector:\n        eks.amazonaws.com/compute-type: hybrid\n    asserts:\n      - equal:\n          path: spec.template.spec.nodeSelector[\"eks.amazonaws.com/compute-type\"]\n          value: hybrid\n"
  },
  {
    "path": "chart/kube-flannel/values.yaml",
    "content": "---\nglobal:\n  imagePullSecrets:\n# - name: \"a-secret-name\"\n\n# The IPv4 cidr pool to create on startup if none exists. Pod IPs will be\n# chosen from this range.\npodCidr: \"10.244.0.0/16\"\npodCidrv6: \"\"\n\nflannel:\n  # kube-flannel image\n  image:\n    repository: ghcr.io/flannel-io/flannel\n    tag: v0.28.1\n  image_cni:\n    repository: ghcr.io/flannel-io/flannel-cni-plugin\n    tag: v1.9.0-flannel1\n  # cniBinDir is the directory to which the flannel CNI binary is installed.\n  cniBinDir: \"/opt/cni/bin\"\n  # cniConfDir is the directory where the CNI configuration is located.\n  cniConfDir: \"/etc/cni/net.d\"\n  # skipCNIConfigInstallation skips the installation of the flannel CNI config. This is useful when the CNI config is\n  # provided externally.\n  skipCNIConfigInstallation: false\n  # flannel command arguments\n  enableNFTables: false\n  args:\n  - \"--ip-masq\"\n  - \"--kube-subnet-mgr\"\n  # Backend for kube-flannel. Backend should not be changed\n  # at runtime. (vxlan, host-gw, wireguard, udp)\n  # Documentation at https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md\n  backend: \"vxlan\"\n  # Port used by the backend 0 means default value (VXLAN: 8472, Wireguard: 51821, UDP: 8285)\n  #backendPort: 0\n  # MTU to use for outgoing packets (VXLAN and Wiregurad) if not defined the MTU of the external interface is used.\n  # mtu: 1500\n  #\n  # VXLAN Configs:\n  #\n  # VXLAN Identifier to be used. On Linux default is 1.\n  #vni: 1\n  # Enable VXLAN Group Based Policy (Default false)\n  # GBP: false\n  # Enable direct routes (default is false)\n  # directRouting: false\n  # MAC prefix to be used on Windows. (Defaults is 0E-2A)\n  # macPrefix: \"0E-2A\"\n  #\n  # Wireguard Configs:\n  #\n  # UDP listen port used with IPv6\n  # backendPortv6: 51821\n  # Pre shared key to use\n  # psk: 0\n  # IP version to use on Wireguard\n  # tunnelMode: \"separate\"\n  # Persistent keep interval to use\n  # keepaliveInterval: 0\n  #\n  cniConf: |\n    {\n      \"name\": \"cbr0\",\n      \"cniVersion\": \"0.3.1\",\n      \"plugins\": [\n        {\n          \"type\": \"flannel\",\n          \"delegate\": {\n            \"hairpinMode\": true,\n            \"isDefaultGateway\": true\n          }\n        },\n        {\n          \"type\": \"portmap\",\n          \"capabilities\": {\n            \"portMappings\": true\n          }\n        }\n      ]\n    }\n  #\n  # General daemonset configs\n  #\n  resources:\n    requests:\n      cpu: 100m\n      memory: 50Mi\n  tolerations:\n  - effect: NoExecute\n    operator: Exists\n  - effect: NoSchedule\n    operator: Exists\n  nodeSelector: {}\n\nnetpol:\n  enabled: false\n  args:\n  - \"--hostname-override=$(MY_NODE_NAME)\"\n  - \"--v=2\"\n  image:\n    repository: registry.k8s.io/networking/kube-network-policies\n    tag: v1.0.0\n"
  },
  {
    "path": "code-of-conduct.md",
    "content": "## CoreOS Community Code of Conduct\n\n### Contributor Code of Conduct\n\nAs contributors and maintainers of this project, and in the interest of\nfostering an open and welcoming community, we pledge to respect all people who\ncontribute through reporting issues, posting feature requests, updating\ndocumentation, submitting pull requests or patches, and other activities.\n\nWe are committed to making participation in this project a harassment-free\nexperience for everyone, regardless of level of experience, gender, gender\nidentity and expression, sexual orientation, disability, personal appearance,\nbody size, race, ethnicity, age, religion, or nationality.\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery\n* Personal attacks\n* Trolling or insulting/derogatory comments\n* Public or private harassment\n* Publishing others' private information, such as physical or electronic addresses, without explicit permission\n* Other unethical or unprofessional conduct.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct. By adopting this Code of Conduct,\nproject maintainers commit themselves to fairly and consistently applying these\nprinciples to every aspect of managing this project. Project maintainers who do\nnot follow or enforce the Code of Conduct may be permanently removed from the\nproject team.\n\nThis code of conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community.\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting a project maintainer, Brandon Philips\n<brandon.philips@coreos.com>, and/or Rithu John <rithu.john@coreos.com>.\n\nThis Code of Conduct is adapted from the Contributor Covenant\n(http://contributor-covenant.org), version 1.2.0, available at\nhttp://contributor-covenant.org/version/1/2/0/\n\n### CoreOS Events Code of Conduct\n\nCoreOS events are working conferences intended for professional networking and\ncollaboration in the CoreOS community. Attendees are expected to behave\naccording to professional standards and in accordance with their employer’s\npolicies on appropriate workplace behavior.\n\nWhile at CoreOS events or related social networking opportunities, attendees\nshould not engage in discriminatory or offensive speech or actions including\nbut not limited to gender, sexuality, race, age, disability, or religion.\nSpeakers should be especially aware of these concerns.\n\nCoreOS does not condone any statements by speakers contrary to these standards.\nCoreOS reserves the right to deny entrance and/or eject from an event (without\nrefund) any individual found to be engaging in discriminatory or offensive\nspeech or actions.\n\nPlease bring any concerns to the immediate attention of designated on-site\nstaff, Brandon Philips <brandon.philips@coreos.com>, and/or Rithu John <rithu.john@coreos.com>.\n"
  },
  {
    "path": "dist/extension-hostgw",
    "content": "{\n  \"Network\": \"10.50.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"extension\",\n    \"SubnetAddCommand\": \"ip route add $SUBNET via $PUBLIC_IP\",\n    \"SubnetRemoveCommand\": \"ip route del $SUBNET via $PUBLIC_IP\"\n  }\n}\n"
  },
  {
    "path": "dist/extension-vxlan",
    "content": "{\n  \"Network\": \"10.50.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"extension\",\n    \"PreStartupCommand\": \"export VNI=1; export IF_NAME=flannel-vxlan; ip link del $IF_NAME 2>/dev/null; ip link add $IF_NAME type vxlan id $VNI dstport 8472 nolearning && ip link set mtu 1450 dev $IF_NAME && cat /sys/class/net/$IF_NAME/address\",\n    \"PostStartupCommand\": \"export IF_NAME=flannel-vxlan; export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; ip addr add $SUBNET_IP/32 dev $IF_NAME && ip link set $IF_NAME up\",\n    \"ShutdownCommand\": \"export IF_NAME=flannel-vxlan; ip link del $IF_NAME\",\n    \"SubnetAddCommand\": \"export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; export IF_NAME=flannel-vxlan; read VTEP; ip route add $SUBNET nexthop via $SUBNET_IP dev $IF_NAME onlink && ip neigh replace $SUBNET_IP dev $IF_NAME lladdr $VTEP && bridge fdb add $VTEP dev $IF_NAME self dst $PUBLIC_IP\"\n  }\n}\n"
  },
  {
    "path": "dist/extension-wireguard",
    "content": "// This is deprecated and should not be used. Please use the wireguard backend instead!\n{\n  \"Network\": \"10.50.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"extension\",\n    \"PreStartupCommand\": \"wg genkey | tee privatekey | wg pubkey\",\n    \"PostStartupCommand\": \"export SUBNET_IP=`echo $SUBNET | cut -d'/' -f 1`; ip link del flannel-wg 2>/dev/null; ip link add flannel-wg type wireguard && wg set flannel-wg listen-port 51820 private-key privatekey && ip addr add $SUBNET_IP/32 dev flannel-wg && ip link set flannel-wg up && ip route add $NETWORK dev flannel-wg\",\n    \"ShutdownCommand\": \"ip link del flannel-wg\",\n    \"SubnetAddCommand\": \"read PUBLICKEY; wg set flannel-wg peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET\",\n    \"SubnetRemoveCommand\": \"read PUBLICKEY; wg set flannel-wg peer $PUBLICKEY remove\"\n  }\n}\n"
  },
  {
    "path": "dist/fake-node.yaml",
    "content": "apiVersion: v1\nkind: Node\nmetadata:\n  name: test\n  annotations:\n    flannel.alpha.coreos.com/backend-data: '{\"VtepMAC\":\"96:59:b2:3f:04:ee\"}' \n    flannel.alpha.coreos.com/backend-type: vxlan\n    flannel.alpha.coreos.com/kube-subnet-manager: \"true\"\n    flannel.alpha.coreos.com/public-ip: 192.168.77.17\nspec:\n  podCIDR: 10.244.7.0/24\n"
  },
  {
    "path": "dist/functional-test-k8s.sh",
    "content": "#!/bin/bash\n\nARCH=\"${ARCH:-amd64}\"\nETCD_IMG=\"${ETCD_IMG:-quay.io/coreos/etcd:v3.6.2}\"\nETCD_LOCATION=\"${ETCD_LOCATION:-etcd}\"\nFLANNEL_NET=\"${FLANNEL_NET:-10.10.0.0/16}\"\nTAG=`git describe --tags --always`\nFLANNEL_DOCKER_IMAGE=\"${FLANNEL_DOCKER_IMAGE:-quay.io/coreos/flannel:$TAG}\"\nK8S_VERSION=\"${K8S_VERSION:-1.32.6}\"\nHYPERKUBE_IMG=\"docker.io/rancher/hyperkube\"\nHYPERKUBE_CMD=\"${HYPERKUBE_CMD:-\" \"}\"\nHYPERKUBE_APISERVER_CMD=\"${HYPERKUBE_APISERVER_CMD:-kube-apiserver}\"\n\ndocker_ip=$(ip -o -f inet addr show docker0 | grep -Po 'inet \\K[\\d.]+')\netcd_endpt=\"http://$docker_ip:2379\"\nk8s_endpt=\"https://$docker_ip:6443\"\n\n# Set the proper imagename according to architecture\nif [[ ${ARCH} == \"ppc64le\" ]]; then\n    ETCD_IMG+=\"-ppc64le\"\nelif [[ ${ARCH} == \"arm64\" ]]; then\n    ETCD_IMG+=\"-arm64\"\nfi\n\nsetup_suite() {\n    # Run etcd, killing any existing one that was running\n\n    # Start etcd\n    docker rm -f flannel-e2e-test-etcd >/dev/null 2>/dev/null\n    docker run --name=flannel-e2e-test-etcd -d -p 2379:2379 -e ETCD_UNSUPPORTED_ARCH=${ARCH} $ETCD_IMG etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls $etcd_endpt >/dev/null\n    sleep 1\n\n    # Start a kubernetes API server\n    docker rm -f flannel-e2e-k8s-apiserver >/dev/null 2>/dev/null\n    dir=$(mktemp -d)\n    \n    mkdir $dir/pki\n    \n    openssl genrsa -out $dir/pki/ca.key 2048\n    openssl req -new -key $dir/pki/ca.key -subj \"/CN=KUBERNETES-CA/O=Kubernetes\" -out $dir/pki/ca.csr\n    openssl x509 -req -in $dir/pki/ca.csr -signkey $dir/pki/ca.key -CAcreateserial  -out $dir/pki/ca.crt -days 1000\n    cat > $dir/openssl.cnf <<EOF\n[req]\nreq_extensions = v3_req\ndistinguished_name = req_distinguished_name\n[req_distinguished_name]\n[v3_req]\nbasicConstraints = critical, CA:FALSE\nkeyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment\nextendedKeyUsage = serverAuth\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = kubernetes\nDNS.2 = kubernetes.default\nDNS.3 = kubernetes.default.svc\nDNS.4 = kubernetes.default.svc.cluster\nDNS.5 = kubernetes.default.svc.cluster.local\nIP.1 = $docker_ip\nIP.2 = 127.0.0.1\nEOF\n    openssl genrsa -out $dir/pki/kube-apiserver.key 2048\n    openssl req -new -key $dir/pki/kube-apiserver.key \\\n    -subj \"/CN=kube-apiserver/O=Kubernetes\" -out $dir/pki/kube-apiserver.csr -config $dir/openssl.cnf\n    openssl x509 -req -in $dir/pki/kube-apiserver.csr \\\n    -CA $dir/pki/ca.crt -CAkey $dir/pki/ca.key -CAcreateserial  -out $dir/pki/kube-apiserver.crt -extensions v3_req -extfile $dir/openssl.cnf -days 1000\n\n    openssl genrsa -out $dir/pki/service-account.key 2048\n    openssl req -new -key $dir/pki/service-account.key \\\n    -subj \"/CN=service-accounts/O=Kubernetes\" -out $dir/pki/service-account.csr\n    openssl x509 -req -in $dir/pki/service-account.csr \\\n    -CA $dir/pki/ca.crt -CAkey $dir/pki/ca.key -CAcreateserial  -out $dir/pki/service-account.crt -days 100\n\n    openssl genrsa -out $dir/pki/admin.key 2048\n    openssl req -new -key $dir/pki/admin.key -subj \"/CN=admin/O=system:masters\" -out $dir/pki/admin.csr\n    openssl x509 -req -in $dir/pki/admin.csr -CA $dir/pki/ca.crt -CAkey $dir/pki/ca.key -CAcreateserial  -out $dir/pki/admin.crt -days 1000\n    \n    docker run -d --net=host -v $dir:/var/lib/kubernetes --name flannel-e2e-k8s-apiserver ${HYPERKUBE_IMG}:v$K8S_VERSION-rancher1 \\\n      ${HYPERKUBE_CMD} ${HYPERKUBE_APISERVER_CMD} --etcd-servers=$etcd_endpt --bind-address=$docker_ip \\\n      --client-ca-file=/var/lib/kubernetes/pki/ca.crt \\\n      --enable-admission-plugins=NodeRestriction,ServiceAccount \\\n      --service-account-key-file=/var/lib/kubernetes/pki/service-account.crt \\\n      --service-account-signing-key-file=/var/lib/kubernetes/pki/service-account.key \\\n      --service-account-issuer=https://kubernetes.default.svc.local \\\n      --tls-cert-file=/var/lib/kubernetes/pki/kube-apiserver.crt \\\n      --tls-private-key-file=/var/lib/kubernetes/pki/kube-apiserver.key \\\n      --service-cluster-ip-range=10.101.0.0/16 --allow-privileged >/dev/null\n    sleep 1\n\n    docker exec flannel-e2e-k8s-apiserver kubectl config set-cluster kubernetes-test-flannel \\\n    --certificate-authority=/var/lib/kubernetes/pki/ca.crt \\\n    --embed-certs=true \\\n    --server=\"https://$docker_ip:6443\" \\\n    --kubeconfig=/var/lib/kubernetes/admin.kubeconfig\n\n    docker exec flannel-e2e-k8s-apiserver kubectl config set-credentials admin \\\n    --client-certificate=/var/lib/kubernetes/pki/admin.crt \\\n    --client-key=/var/lib/kubernetes/pki/admin.key \\\n    --embed-certs=true \\\n    --kubeconfig=/var/lib/kubernetes/admin.kubeconfig\n\n    docker exec flannel-e2e-k8s-apiserver kubectl config set-context default \\\n    --cluster=kubernetes-test-flannel \\\n    --user=admin \\\n    --kubeconfig=/var/lib/kubernetes/admin.kubeconfig\n\n    docker exec flannel-e2e-k8s-apiserver kubectl config use-context default --kubeconfig=/var/lib/kubernetes/admin.kubeconfig\n\n    while ! cat <<EOF |  docker exec -i flannel-e2e-k8s-apiserver ${HYPERKUBE_CMD} kubectl --kubeconfig=/var/lib/kubernetes/admin.kubeconfig create -f - >/dev/null 2>/dev/null\napiVersion: v1\nkind: Node\nmetadata:\n  name: flannel1\n  annotations:\n    dummy: value\nspec:\n  podCIDR: 10.10.1.0/24\nEOF\ndo\n    sleep 1\ndone\n\ncat <<EOF |  docker exec -i flannel-e2e-k8s-apiserver ${HYPERKUBE_CMD} kubectl --kubeconfig=/var/lib/kubernetes/admin.kubeconfig create -f - >/dev/null 2>/dev/null\napiVersion: v1\nkind: Node\nmetadata:\n  name: flannel2\n  annotations:\n    dummy: value\nspec:\n  podCIDR: 10.10.2.0/24\nEOF\n}\n\nteardown_suite() {\n    # Teardown the etcd server\n    docker rm -f flannel-e2e-test-etcd >/dev/null\n    docker rm -f flannel-e2e-k8s-apiserver >/dev/null\n}\n\nteardown() {\n\tdocker rm -f flannel-e2e-test-flannel1 >/dev/null 2>/dev/null\n\tdocker rm -f flannel-e2e-test-flannel2 >/dev/null 2>/dev/null\n}\n\nstart_flannel() {\n    local backend=$1\n\n    flannel_conf=\"{ \\\"Network\\\": \\\"$FLANNEL_NET\\\", \\\"Backend\\\": { \\\"Type\\\": \\\"${backend}\\\" } }\"\n    dir=$(mktemp -d)\n\n    docker exec -i flannel-e2e-k8s-apiserver cat /var/lib/kubernetes/admin.kubeconfig > $dir/admin.kubeconfig\n\n    for host_num in 1 2; do\n       docker rm -f flannel-e2e-test-flannel$host_num >/dev/null 2>/dev/null\n\n       docker run -id --privileged \\\n\t-v $dir:/var/lib/kubernetes/ \\\n        -e NODE_NAME=flannel$host_num \\\n        --name flannel-e2e-test-flannel$host_num \\\n        --entrypoint \"/bin/sh\" \\\n        $FLANNEL_DOCKER_IMAGE \\\n        -c \"mkdir -p /etc/kube-flannel && \\\n            echo '$flannel_conf' > /etc/kube-flannel/net-conf.json && \\\n            /opt/bin/flanneld --kube-subnet-mgr --ip-masq --kubeconfig-file /var/lib/kubernetes/admin.kubeconfig --kube-api-url $k8s_endpt\" >/dev/null\n\n       while ! docker exec flannel-e2e-test-flannel$host_num ls /run/flannel/subnet.env >/dev/null 2>&1; do\n         status=$(docker inspect --format='{{.State.Status}}' flannel-e2e-test-flannel$host_num)\n         if [[ $status != \"running\" ]]; then\n            docker logs flannel-e2e-test-flannel$host_num\n\n            return\n         fi\n\n         sleep 0.1\n       done\n    done\n}\n\ncreate_ping_dest() {\n    # add a dummy interface with $FLANNEL_SUBNET so we have a known working IP to ping\n    for host_num in 1 2; do\n\n       # Use declare to allow the host_num variable to be part of the ping_dest variable name. -g is needed to make it global\n       declare -g ping_dest$host_num=$(docker \"exec\" --privileged flannel-e2e-test-flannel$host_num /bin/sh -c '\\\n\t\tsource /run/flannel/subnet.env && \\\n\t\tip link add name dummy0 type dummy && \\\n\t\tip addr add $FLANNEL_SUBNET dev dummy0 && ip link set dummy0 up && \\\n\t\techo $FLANNEL_SUBNET | cut -f 1 -d \"/\" ')\n    done\n}\n\ntest_public-ip-overwrite(){\n  docker exec flannel-e2e-k8s-apiserver kubectl --kubeconfig=/var/lib/kubernetes/admin.kubeconfig annotate node flannel1 \\\n    flannel.alpha.coreos.com/public-ip-overwrite=172.18.0.2 >/dev/null 2>&1\n  start_flannel vxlan\n  assert_equals \"172.18.0.2\" \\\n    \"$(docker exec flannel-e2e-k8s-apiserver kubectl --kubeconfig=/var/lib/kubernetes/admin.kubeconfig get node/flannel1 -o \\\n    jsonpath='{.metadata.annotations.flannel\\.alpha\\.coreos\\.com/public-ip}' 2>/dev/null)\" \\\n    \"Overwriting public IP via annotation does not work\"\n  # Remove annotation to not break all other tests\n  docker exec flannel-e2e-k8s-apiserver kubectl --kubeconfig=/var/lib/kubernetes/admin.kubeconfig annotate node flannel1 \\\n    flannel.alpha.coreos.com/public-ip-overwrite- >/dev/null 2>&1\n}\n\npings() {\n    # ping in both directions\n\tassert \"docker exec --privileged flannel-e2e-test-flannel1 /bin/ping -c 5 $ping_dest2\" \"Host 1 cannot ping host 2\"\n\tassert \"docker exec --privileged flannel-e2e-test-flannel2 /bin/ping -c 5 $ping_dest1\" \"Host 2 cannot ping host 1\"\n}\n\ncheck_iptables() {\n  read -r -d '' POSTROUTING_RULES_FLANNEL1 << EOM\n-P POSTROUTING ACCEPT\n-A POSTROUTING -m comment --comment \"flanneld masq\" -j FLANNEL-POSTRTG\n-N FLANNEL-POSTRTG\n-A FLANNEL-POSTRTG -m mark --mark 0x4000/0x4000 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.10.1.0/24 -d 10.10.0.0/16 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.10.0.0/16 -d 10.10.1.0/24 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG ! -s 10.10.0.0/16 -d 10.10.1.0/24 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.10.0.0/16 ! -d 224.0.0.0/4 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\n-A FLANNEL-POSTRTG ! -s 10.10.0.0/16 -d 10.10.0.0/16 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\nEOM\n  read -r -d '' POSTROUTING_RULES_FLANNEL2 << EOM\n-P POSTROUTING ACCEPT\n-A POSTROUTING -m comment --comment \"flanneld masq\" -j FLANNEL-POSTRTG\n-N FLANNEL-POSTRTG\n-A FLANNEL-POSTRTG -m mark --mark 0x4000/0x4000 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.10.2.0/24 -d 10.10.0.0/16 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.10.0.0/16 -d 10.10.2.0/24 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG ! -s 10.10.0.0/16 -d 10.10.2.0/24 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.10.0.0/16 ! -d 224.0.0.0/4 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\n-A FLANNEL-POSTRTG ! -s 10.10.0.0/16 -d 10.10.0.0/16 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\nEOM\n  read -r -d '' FORWARD_RULES << EOM\n-P FORWARD ACCEPT\n-A FORWARD -m comment --comment \"flanneld forward\" -j FLANNEL-FWD\n-N FLANNEL-FWD\n-A FLANNEL-FWD -s 10.10.0.0/16 -m comment --comment \"flanneld forward\" -j ACCEPT\n-A FLANNEL-FWD -d 10.10.0.0/16 -m comment --comment \"flanneld forward\" -j ACCEPT\nEOM\n  # check masquerade & forward rules\n  assert_equals \"$POSTROUTING_RULES_FLANNEL1\" \\\n                \"$(docker exec --privileged flannel-e2e-test-flannel1 /sbin/iptables -t nat -S POSTROUTING)\n$(docker exec --privileged flannel-e2e-test-flannel1 /sbin/iptables -t nat -S FLANNEL-POSTRTG)\" \"Host 1 has not expected postrouting rules\"\n  assert_equals \"$POSTROUTING_RULES_FLANNEL2\" \\\n                \"$(docker exec --privileged flannel-e2e-test-flannel2 /sbin/iptables -t nat -S POSTROUTING)\n$(docker exec --privileged flannel-e2e-test-flannel2 /sbin/iptables -t nat -S FLANNEL-POSTRTG)\" \"Host 2 has not expected postrouting rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged flannel-e2e-test-flannel1 /sbin/iptables -t filter -S FORWARD)\n$(docker exec --privileged flannel-e2e-test-flannel1 /sbin/iptables -t filter -S FLANNEL-FWD)\" \"Host 1 has not expected forward rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged flannel-e2e-test-flannel2 /sbin/iptables -t filter -S FORWARD)\n$(docker exec --privileged flannel-e2e-test-flannel2 /sbin/iptables -t filter -S FLANNEL-FWD)\" \"Host 2 has not expected forward rules\"\n}\n\ntest_manifest() {\n    dir=$(mktemp -d)\n\n    docker exec -i flannel-e2e-k8s-apiserver cat /var/lib/kubernetes/admin.kubeconfig > $dir/admin.kubeconfig\n    # This just tests that the API server accepts the manifest, not that it actually acts on it correctly.\n    assert \"cat ../Documentation/kube-flannel.yml |  docker run -v $dir:/var/lib/kubernetes -i --rm --net=host ${HYPERKUBE_IMG}:v$K8S_VERSION-rancher1 ${HYPERKUBE_CMD} kubectl --kubeconfig=/var/lib/kubernetes/admin.kubeconfig create -f -\"\n}\n"
  },
  {
    "path": "dist/functional-test.sh",
    "content": "#!/bin/bash\n\n#Add \"set -xe\" to get more information where the unit test fail\nset -xe\n\nARCH=\"${ARCH:-amd64}\"\nETCD_IMG=\"${ETCD_IMG:-quay.io/coreos/etcd:v3.6.2}\"\n# etcd might take a bit to come up - use a known etcd version so we know we have etcdctl available\nETCDCTL_IMG=\"quay.io/coreos/etcd:v3.6.2\"\nETCD_LOCATION=\"${ETCD_LOCATION:-etcd}\"\nFLANNEL_NET=\"${FLANNEL_NET:-10.10.0.0/16}\"\nTAG=`git describe --tags --always`\nFLANNEL_DOCKER_IMAGE=\"${FLANNEL_DOCKER_IMAGE:-quay.io/coreos/flannel:$TAG}\"\n\n# Set the proper imagename according to architecture\nif [[ ${ARCH} == \"ppc64le\" ]]; then\n    ETCD_IMG+=\"-ppc64le\"\n    ETCDCTL_IMG+=\"-ppc64le\"\nelif [[ ${ARCH} == \"arm64\" ]]; then\n    ETCD_IMG+=\"-arm64\"\n    ETCDCTL_IMG+=\"-arm64\"\nfi\n\nsetup_suite() {\n    # Run etcd, killing any existing one that was running\n    docker_ip=$(ip -o -f inet addr show docker0 | grep -Po 'inet \\K[\\d.]+')\n    etcd_endpt=\"http://$docker_ip:2379\"\n\n    # Start etcd\n    docker rm -f flannel-e2e-test-etcd >/dev/null 2>/dev/null\n    docker run --name=flannel-e2e-test-etcd -d --dns 8.8.8.8 -v \"${PWD}/test:/certs\" \\\n    -e ETCD_UNSUPPORTED_ARCH=${ARCH} -p 2379:2379 $ETCD_IMG $ETCD_LOCATION \\\n    --listen-client-urls http://0.0.0.0:2379 \\\n    --cert-file=/certs/server.pem \\\n    --key-file=/certs/server-key.pem \\\n    --client-cert-auth \\\n    --trusted-ca-file=/certs/ca.pem \\\n    --advertise-client-urls $etcd_endpt >/dev/null\n}\n\nteardown_suite() {\n    # Teardown the etcd server\n    docker rm -f flannel-e2e-test-etcd >/dev/null\n}\n\nsetup() {\n    # rm any old flannel container that maybe running, ignore error as it might not exist\n    docker rm -f flannel-e2e-test-flannel1 >/dev/null 2>/dev/null\n    assert \"docker run -v ${PWD}/test:/certs --name=flannel-e2e-test-flannel1 -d --privileged $FLANNEL_DOCKER_IMAGE --etcd-cafile=/certs/ca.pem --etcd-certfile=/certs/client.pem --etcd-keyfile=/certs/client-key.pem --etcd-endpoints=$etcd_endpt -v 10\"\n\n    # rm any old flannel container that maybe running, ignore error as it might not exist\n    docker rm -f flannel-e2e-test-flannel2 >/dev/null 2>/dev/null\n    assert \"docker run -v ${PWD}/test:/certs --name=flannel-e2e-test-flannel2 -d --privileged $FLANNEL_DOCKER_IMAGE --etcd-cafile=/certs/ca.pem --etcd-certfile=/certs/client.pem --etcd-keyfile=/certs/client-key.pem --etcd-endpoints=$etcd_endpt -v 10\"\n}\n\nteardown() {\n    echo \"dumping subnets in etcd\"\n    docker run --rm -e ETCDCTL_API=3 -v \"${PWD}/test:/certs\" $ETCDCTL_IMG etcdctl --endpoints=$etcd_endpt --cacert=/certs/ca.pem --cert=/certs/client.pem --key=/certs/client-key.pem get --prefix /coreos.com/network/subnets 2>&1\n    echo \"########## logs for flannel-e2e-test-flannel1 container ##########\" 2>&1\n    docker logs flannel-e2e-test-flannel1\n    echo \"########## logs for flannel-e2e-test-flannel2 container ##########\" 2>&1\n    docker logs flannel-e2e-test-flannel2\n    docker rm -f flannel-e2e-test-flannel1 flannel-e2e-test-flannel2 flannel-e2e-test-flannel1-iperf flannel-host1 flannel-host2 > /dev/null 2>&1\n    docker run --rm -e ETCDCTL_API=3 -v \"${PWD}/test:/certs\" $ETCDCTL_IMG etcdctl --endpoints=$etcd_endpt --cacert=/certs/ca.pem --cert=/certs/client.pem --key=/certs/client-key.pem del /coreos.com/network/config > /dev/null 2>&1\n}\n\nwrite_config_etcd() {\n    local backend=$1\n    if [ -e \"$backend\" ]; then\n        echo \"Reading custom conf from $backend\"\n        flannel_conf=`cat \"$backend\"`\n    else\n        flannel_conf=\"{ \\\"Network\\\": \\\"$FLANNEL_NET\\\", \\\"Backend\\\": { \\\"Type\\\": \\\"${backend}\\\" } }\"\n    fi\n\n    while ! docker run --rm -e ETCDCTL_API=3 -v \"${PWD}/test:/certs\" $ETCDCTL_IMG etcdctl --endpoints=$etcd_endpt --cacert=/certs/ca.pem --cert=/certs/client.pem --key=/certs/client-key.pem put /coreos.com/network/config \"$flannel_conf\" >/dev/null\n    do\n        sleep 0.1\n    done\n}\n\ncreate_ping_dest() {\n    # add a dummy interface with $FLANNEL_SUBNET so we have a known working IP to ping\n    for host_num in 1 2; do\n       while ! docker exec flannel-e2e-test-flannel$host_num ls /run/flannel/subnet.env >/dev/null 2>&1; do\n         sleep 0.1\n       done\n\n       # Use declare to allow the host_num variable to be part of the ping_dest variable name. -g is needed to make it global\n       declare -g ping_dest$host_num=$(docker \"exec\" --privileged flannel-e2e-test-flannel$host_num /bin/sh -c '\\\n        source /run/flannel/subnet.env && \\\n        ip link add name dummy0 type dummy && \\\n        ip addr add $FLANNEL_SUBNET dev dummy0 && ip link set dummy0 up && \\\n        echo $FLANNEL_SUBNET | cut -f 1 -d \"/\" ')\n    done\n}\n\n#test_wireguard_ping() {\n#    write_config_etcd extension-wireguard\n#    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n#    pings\n#}\n\ntest_vxlan_ping() {\n    write_config_etcd vxlan\n    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n    pings\n}\n\nif [[ ${ARCH} == \"amd64\" ]]; then\ntest_udp_ping() {\n    write_config_etcd udp\n    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n    pings\n}\nfi\n\ntest_hostgw_ping() {\n    write_config_etcd host-gw\n    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n    pings\n}\n\ntest_ipip_ping() {\n    write_config_etcd ipip\n    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n    pings\n}\n\ntest_ipsec_ping() {\n    write_config_etcd ipsec\n    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n    pings\n}\n\ntest_wireguard_ping() {\n    write_config_etcd wireguard\n    create_ping_dest # creates ping_dest1 and ping_dest2 variables\n    pings\n}\n\npings() {\n    # ping in both directions\n    assert \"docker exec --privileged flannel-e2e-test-flannel1 /bin/ping -I $ping_dest1 -c 3 $ping_dest2\" \"Host 1 cannot ping host 2\"\n    assert \"docker exec --privileged flannel-e2e-test-flannel2 /bin/ping -I $ping_dest2 -c 3 $ping_dest1\" \"Host 2 cannot ping host 1\"\n}\n\ntest_ipsec_perf() {\n    write_config_etcd ipsec\n    create_ping_dest\n    perf\n}\n\nperf() {\n    # Perf test - run iperf server on flannel1 and client on flannel2\n    docker rm -f flannel-e2e-test-flannel1-iperf 2>/dev/null\n    docker run -d --name flannel-e2e-test-flannel1-iperf --net=container:flannel-e2e-test-flannel1 iperf3:latest >/dev/null\n    wait_for flannel-e2e-test-flannel1-iperf\n    docker run --rm --net=container:flannel-e2e-test-flannel2 iperf3:latest -c $ping_dest1 -B $ping_dest2\n}\n\nwait_for() {\n  while ! docker inspect --format='{{json .State.Status}}' $1 >/dev/null\n  do\n    sleep 1\n  done\n}\n\n\ntest_multi() {\n    flannel_conf_vxlan='{\"Network\": \"10.11.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}'\n    flannel_conf_host_gw='{\"Network\": \"10.12.0.0/16\", \"Backend\": {\"Type\": \"host-gw\"}}'\n\n    while ! docker run --rm -e ETCDCTL_API=3 -v \"${PWD}/test:/certs\" $ETCD_IMG etcdctl --endpoints=$etcd_endpt --cacert=/certs/ca.pem --cert=/certs/client.pem --key=/certs/client-key.pem put /vxlan/network/config \"$flannel_conf_vxlan\" >/dev/null\n    do\n        sleep 0.1\n    done\n\n    while ! docker run --rm -e ETCDCTL_API=3 -v \"${PWD}/test:/certs\" $ETCD_IMG etcdctl --endpoints=$etcd_endpt --cacert=/certs/ca.pem --cert=/certs/client.pem --key=/certs/client-key.pem put /hostgw/network/config \"$flannel_conf_host_gw\" >/dev/null\n    do\n        sleep 0.1\n    done\n\n    for host in 1 2; do\n        # rm any old flannel container, ignore error as it might not exist\n        docker rm -f flannel-host$host 2>/dev/null >/dev/null\n\n        # Start the hosts\n        docker run -v \"${PWD}/test:/certs\" --name=flannel-host$host -id --privileged --entrypoint /bin/sh $FLANNEL_DOCKER_IMAGE   >/dev/null\n\n        # Start two flanneld instances\n        docker exec -d flannel-host$host sh -c \"/opt/bin/flanneld -v 10 -subnet-file /vxlan.env -etcd-prefix=/vxlan/network --etcd-cafile=/certs/ca.pem --etcd-certfile=/certs/client.pem --etcd-keyfile=/certs/client-key.pem --etcd-endpoints=$etcd_endpt 2>vxlan.log\"\n        docker exec -d flannel-host$host sh -c \"/opt/bin/flanneld -v 10 -subnet-file /hostgw.env -etcd-prefix=/hostgw/network --etcd-cafile=/certs/ca.pem --etcd-certfile=/certs/client.pem --etcd-keyfile=/certs/client-key.pem --etcd-endpoints=$etcd_endpt 2>hostgw.log\"\n    done\n\n    for host in 1 2; do\n        for backend_type in vxlan hostgw; do\n            while ! docker exec flannel-host$host ls /$backend_type.env  >/dev/null 2>&1; do\n              sleep 0.1\n            done\n        done\n    done\n\n    # add dummy interface on host1 only so we have a known working IP to ping then ping it from host2\n    vxlan_ping_dest=$(docker exec flannel-host1 /bin/sh -c '\\\n        source /vxlan.env &&\n        ip link add name dummy_vxlan type dummy && \\\n        ip addr add $FLANNEL_SUBNET dev dummy_vxlan && \\\n               ip link set dummy_vxlan up && \\\n        echo $FLANNEL_SUBNET | cut -f 1 -d \"/\" ')\n\n    hostgw_ping_dest=$(docker exec flannel-host1 /bin/sh -c '\\\n        source /hostgw.env &&\n        ip link add name dummy_hostgw type dummy && \\\n        ip addr add $FLANNEL_SUBNET dev dummy_hostgw && \\\n               ip link set dummy_hostgw up && \\\n        echo $FLANNEL_SUBNET | cut -f 1 -d \"/\" ')\n\n    # Send some pings from host2. Make sure we can send traffic over vxlan or directly.\n    # If a particular (wrong) interface is forced then pings should fail\n    assert \"docker exec flannel-host2 ping -c 3 $hostgw_ping_dest\"\n    assert \"docker exec flannel-host2 ping -c 3 $vxlan_ping_dest\"\n    assert_fails \"docker exec flannel-host2 ping -W 1 -c 1 -I flannel.1 $hostgw_ping_dest\"\n    assert_fails \"docker exec flannel-host2 ping -W 1 -c 1 -I eth0 $vxlan_ping_dest\"\n}\n"
  },
  {
    "path": "dist/ipsec",
    "content": "{\n  \"Network\": \"10.50.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"ipsec\",\n    \"PSK\":\"4bc1e570ff249cce3cc8cef5e2f8625bac76c7b02532f8bde9747196eb15a742480e265bbc0c60c265a8fe4eb6380cd1\"\n  }\n}\n"
  },
  {
    "path": "dist/license-check.sh",
    "content": "#!/usr/bin/env bash\n\nlicRes=$(for file in $(find . -type f -iname '*.go' ! -path './vendor/*'); do\n\t\thead -n4 \"${file}\" | grep -Eq \"(Copyright|generated|GENERATED)\" || echo -e \"  ${file}\"\n\tdone;)\nif [ -n \"${licRes}\" ]; then\n\techo -e \"license header checking failed:\\n${licRes}\"\n\texit 255\nfi\n"
  },
  {
    "path": "dist/mk-docker-opts.sh",
    "content": "#!/bin/sh\n\nusage() {\n\techo \"$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY]\n\nGenerate Docker daemon options based on flannel env file\nOPTIONS:\n\t-f\tPath to flannel env file. Defaults to /run/flannel/subnet.env\n\t-d\tPath to Docker env file to write to. Defaults to /run/docker_opts.env\n\t-i\tOutput each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500\n\t-c\tOutput combined Docker options into DOCKER_OPTS var\n\t-k\tSet the combined options key to this value (default DOCKER_OPTS=)\n\t-m\tDo not output --ip-masq (useful for older Docker version)\n\" >&2\n\n\texit 1\n}\n\nflannel_env=\"/run/flannel/subnet.env\"\ndocker_env=\"/run/docker_opts.env\"\ncombined_opts_key=\"DOCKER_OPTS\"\nindiv_opts=false\ncombined_opts=false\nipmasq=true\n\nwhile getopts \"f:d:icmk:?h\" opt; do\n\tcase $opt in\n\t\tf)\n\t\t\tflannel_env=$OPTARG\n\t\t\t;;\n\t\td)\n\t\t\tdocker_env=$OPTARG\n\t\t\t;;\n\t\ti)\n\t\t\tindiv_opts=true\n\t\t\t;;\n\t\tc)\n\t\t\tcombined_opts=true\n\t\t\t;;\n\t\tm)\n\t\t\tipmasq=false\n\t\t\t;;\n\t\tk)\n\t\t\tcombined_opts_key=$OPTARG\n\t\t\t;;\n\t\t[\\?h])\n\t\t\tusage\n\t\t\t;;\n\tesac\ndone\n\nif [ $indiv_opts = false ] && [ $combined_opts = false ]; then\n\tindiv_opts=true\n\tcombined_opts=true\nfi\n\nif [ -f \"$flannel_env\" ]; then\n\t. $flannel_env\nfi\n\nif [ -n \"$FLANNEL_SUBNET\" ]; then\n\tDOCKER_OPT_BIP=\"--bip=$FLANNEL_SUBNET\"\nfi\n\nif [ -n \"$FLANNEL_MTU\" ]; then\n\tDOCKER_OPT_MTU=\"--mtu=$FLANNEL_MTU\"\nfi\n\nif [ -n \"$FLANNEL_IPMASQ\" ] && [ $ipmasq = true ] ; then\n\tif [ \"$FLANNEL_IPMASQ\" = true ] ; then\n\t\tDOCKER_OPT_IPMASQ=\"--ip-masq=false\"\n\telif [ \"$FLANNEL_IPMASQ\" = false ] ; then\n\t\tDOCKER_OPT_IPMASQ=\"--ip-masq=true\"\n\telse\n\t\techo \"Invalid value of FLANNEL_IPMASQ: $FLANNEL_IPMASQ\" >&2\n\t\texit 1\n\tfi\nfi\n\neval docker_opts=\"\\$${combined_opts_key}\"\n\nif [ \"$docker_opts\" ]; then\n\tdocker_opts=\"$docker_opts \";\nfi\n\necho -n \"\" >$docker_env\n\nfor opt in $(set | grep \"DOCKER_OPT_\"); do\n\n\tOPT_NAME=$(echo $opt | awk -F \"=\" '{print $1;}');\n\tOPT_VALUE=$(eval echo \"\\$$OPT_NAME\");\n\n\tif [ \"$indiv_opts\" = true ]; then\n\t\techo \"$OPT_NAME=\\\"$OPT_VALUE\\\"\" >>$docker_env;\n\tfi\n\n\tdocker_opts=\"$docker_opts $OPT_VALUE\";\n\ndone\n\nif [ \"$combined_opts\" = true ]; then\n\techo \"${combined_opts_key}=\\\"${docker_opts}\\\"\" >>$docker_env\nfi\n\nif [ -n \"${FLANNEL_SUBNET}\" ];then\n    dot_four=`echo ${FLANNEL_SUBNET}|cut -d. -f4|cut -d/ -f1`\n    dot_four=$((${dot_four}-1))\n    subnets_pre=`echo ${FLANNEL_SUBNET}|cut -d. -f 1,2,3`\n    subnets_lat=`echo ${FLANNEL_SUBNET}|cut -d/ -f2`\n    subnets=${subnets_pre}.${dot_four}/${subnets_lat}\n    route_invalid=`ip route show|grep ${subnets}|grep via`\n    if [ -n \"${route_invalid}\" ];then\n        echo \"Your routing table already contains the subnet: ${route_invalid}\" >&2\n\texit 1\n    fi\nfi\n"
  },
  {
    "path": "dist/mk-docker-opts_tests.sh",
    "content": "#!/bin/bash\nset -e\n\necho \"### Dry run with input & output files set\"\necho \"$ ./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt\"\n! read -d '' EXPECTED <<EOF \nDOCKER_OPT_BIP=\"--bip=10.1.74.1/24\"\nDOCKER_OPT_IPMASQ=\"--ip-masq=true\"\nDOCKER_OPT_MTU=\"--mtu=1472\"\nDOCKER_OPTS=\" --bip=10.1.74.1/24 --ip-masq=true --mtu=1472\"\nEOF\n./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt\ndiff -B -b here.txt <(echo -e \"${EXPECTED}\")\necho\n\n\necho \"### Individual vars only (Note DOCKER_OPTS= is missing)\"\necho \"$ ./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -i\"\n! read -d '' EXPECTED <<EOF \nDOCKER_OPT_BIP=\"--bip=10.1.74.1/24\"\nDOCKER_OPT_IPMASQ=\"--ip-masq=true\"\nDOCKER_OPT_MTU=\"--mtu=1472\"\nEOF\n./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -i\ndiff -B -b here.txt <(echo -e \"${EXPECTED}\")\necho\n\n\necho \"### Combined vars only (Note DOCKER_OPT_* vars are missing)\"\necho \"$ ./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -c\"\n! read -d '' EXPECTED <<EOF \nDOCKER_OPTS=\" --bip=10.1.74.1/24 --ip-masq=true --mtu=1472\"\nEOF\n./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -c\ndiff -B -b here.txt <(echo -e \"${EXPECTED}\")\necho\n\n\necho \"### Custom key test (Note DOCKER_OPTS= is substituted by CUSTOM_KEY=)\"\necho \"$ ./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -k CUSTOM_KEY\"\n! read -d '' EXPECTED <<EOF \nDOCKER_OPT_BIP=\"--bip=10.1.74.1/24\"\nDOCKER_OPT_IPMASQ=\"--ip-masq=true\"\nDOCKER_OPT_MTU=\"--mtu=1472\"\nCUSTOM_KEY=\" --bip=10.1.74.1/24 --ip-masq=true --mtu=1472\"\nEOF\n./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -k CUSTOM_KEY\ndiff -B -b here.txt <(echo -e \"${EXPECTED}\")\necho\n\n\necho \"### Ip-masq stripping test (Note DOCKER_OPT_IPMASQ and --ip-masq=true are missing)\"\necho \"$ ./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -m\"\n! read -d '' EXPECTED <<EOF \nDOCKER_OPT_BIP=\"--bip=10.1.74.1/24\"\nDOCKER_OPT_MTU=\"--mtu=1472\"\nDOCKER_OPTS=\" --bip=10.1.74.1/24 --mtu=1472\"\nEOF\n./mk-docker-opts.sh -f ./sample_subnet.env -d here.txt -m\ndiff -B -b here.txt <(echo -e \"${EXPECTED}\")\n\n"
  },
  {
    "path": "dist/sample_subnet.env",
    "content": "FLANNEL_NETWORK=10.1.0.0/16\nFLANNEL_SUBNET=10.1.74.1/24\nFLANNEL_MTU=1472\nFLANNEL_IPMASQ=false\n"
  },
  {
    "path": "dist/snap/README.md",
    "content": "<h1 align=\"center\">\n  <img src=\"https://user-images.githubusercontent.com/45159366/61273381-36394c80-a75e-11e9-9a4a-7f3659ea32c8.png\">\n  <br />\nFlannel\n</h1>\n\n<p align=\"center\"><b>This is the snap for Flannel, a network fabric for containers, designed for Kubernetes. It works on Ubuntu, Fedora, Debian, and other major Linux distributions. </p>\n\n<!-- Uncomment and modify this when you are provided a build status badge\n<p align=\"center\">\n<a href=\"https://build.snapcraft.io/user/snapcrafters/fork-and-rename-me\"><img src=\"https://build.snapcraft.io/badge/snapcrafters/fork-and-rename-me.svg\" alt=\"Snap Status\"></a>\n</p>\n-->\n\n\n<p align=\"center\">Published for <img src=\"https://raw.githubusercontent.com/anythingcodes/slack-emoji-for-techies/gh-pages/emoji/tux.png\" align=\"top\" width=\"24\" /> with 💝 by Snapcrafters</p>\n\n## Install\n\n    sudo snap install flannel --classic\n\n([Don't have snapd installed?](https://snapcraft.io/docs/core/install))\n"
  },
  {
    "path": "dist/snap/snapcraft.yaml",
    "content": "name: flannel\nsummary:  A network fabric for containers.\ndescription:|  A simple and easy way to configure a layer 3 network fabric designed for Kubernetes.\n  \nadopt-info: flannel\n\ngrade: stable\nconfinement: classic\n\narchitectures:\n  - build-on: i386\n  - build-on: amd64\n  - build-on: armhf\n  - build-on: arm64\n\napps:\n  flannel:\n    command: bin/flannel\n    plugs:\n      - home\n      - network\n      - docker\n      - removable-media\n\nparts:\n  flannel:\n  plugin: nil \n    source: https://github.com/flannel-io/flannel.git\n    source-type: git\n    override-pull: |\n      git clone https://github.com/flannel-io/flannel.git src/github.com/flannel-io/flannel\n      cd src/github.com/flannel-io/flannel\n      last_committed_tag=\"$(git describe --tags --abbrev=0)\"\n      last_committed_tag_ver=\"$(echo ${last_committed_tag} | sed 's/v//')\"\n      last_released_tag=\"$(snap info $SNAPCRAFT_PROJECT_NAME | awk '$1 == \"beta:\" { print $2 }')\"\n      # If the latest tag from the upstream project has not been released to\n      # beta, build that tag instead of master.\n      if [ \"${last_committed_tag_ver}\" != \"${last_released_tag}\" ]; then\n        git fetch\n        git checkout \"${last_committed_tag}\"\n      fi\n      snapcraftctl set-version \"$(git describe --tags | sed 's/v//')\"\n      override-build: |\n      export GOPATH=$PWD\n      cd src/github.com/flannel-io/flannel\n      env CGO_ENABLED=0 GOOS=linux \\\n      go build --ldflags \"-s -w \\\n        -X 'github.com/flannel-io/flannel/version.GitCommit=$(git rev-list -1 HEAD)' \\\n        -X 'github.com/flannel-io/flannel/version.Version=$(git describe --tags --abbrev=0)'\" \\\n        -a -installsuffix cgo -o $SNAPCRAFT_PART_INSTALL/bin/flannel\n    build-snaps:\n      - go\n    build-packages:\n      - git\n      - sed\n"
  },
  {
    "path": "dist/test/ca-config.json",
    "content": "{\n    \"signing\": {\n        \"default\": {\n            \"expiry\": \"43800h\"\n        },\n        \"profiles\": {\n            \"server\": {\n                \"expiry\": \"43800h\",\n                \"usages\": [\n                    \"signing\",\n                    \"key encipherment\",\n                    \"server auth\"\n                ]\n            },\n            \"client\": {\n                \"expiry\": \"43800h\",\n                \"usages\": [\n                    \"signing\",\n                    \"key encipherment\",\n                    \"client auth\"\n                ]\n            },\n            \"peer\": {\n                \"expiry\": \"43800h\",\n                \"usages\": [\n                    \"signing\",\n                    \"key encipherment\",\n                    \"server auth\",\n                    \"client auth\"\n                ]\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "dist/test/ca-csr.json",
    "content": "{\n    \"CN\": \"My own CA\",\n    \"key\": {\n        \"algo\": \"rsa\",\n        \"size\": 2048\n    },\n    \"names\": [\n        {\n            \"C\": \"US\",\n            \"L\": \"CA\",\n            \"O\": \"My Company Name\",\n            \"ST\": \"San Francisco\",\n            \"OU\": \"Org Unit 1\",\n            \"OU\": \"Org Unit 2\"\n        }\n    ]\n}\n"
  },
  {
    "path": "dist/test/ca-key.pem",
    "content": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAu5kCsujw2F5xuFSJ6GXpwa7WFBbcYiJD7bw+9irHL0BIUs9N\npNi8B6dev0DM7ReIPJCgrnJyT+5dhwmGsQ3u0MHr0wMQwnHDytvV445VQlYafooz\nMb2Wdc3rLfkATEudvCn9gcs0/6N4OfY8bGLVRFplWHfhD1a9SsSFvGFQvkjRKysI\n1UaoEDj13LS8ZA63mS3xI1ovcCGq/nvNUTB30H3viGSxLc7jS3lqFJ56hMFIWqbo\nz7g6NHUqbOgbOnKCVI1Dk/pmpczQoynbmY6ZpA31alhH9p5tluc+Vdg0z4LTTsWO\nEs8GtXPWh0epgG+7rGyThbjTRVKl5WcIRIz7AQIDAQABAoIBAAn7+sjK3QanACZv\nWWelBOvqAjrPfKs8Z6Efg7pWTIOXSEIgBmHfpyJBJinHqSB1QCr5B5RBQxQ9+3xU\nZXbG5w71QzfX0eyHYYRKbvfNe3wsWPyjHlZnYLqkWv/3YKyu2ZZKJdPinM9+Q8fR\n8yVRnUgmB37N4oyOcUSpcPS1uTZnUmPYANk9MCbFdLGDc3dKw2imJQj0L2SO2ynj\ne2jpyGQaa6CqOHJDx3gjW5JFZZBgTsafajTKQN91s/Bvue9Ggqw4K5MQ09058FIU\n499dmeU8DBTZjJZNUe+MmLxPJo8bl0S6xeC3Xj9lZ3jEEUu1POyx+l+gvRKREoqS\nCz5wKIECgYEA4jv5++0Bn7DXYDJPQEPTk886AA9qstKq6PcCPA/tu7hLWijZJsmX\nAX6NSjj5BJaDi892RMFX+2EOWmyfT2p3MzG5JAKgiQMW4VYnxFIZfi1dzL9e/zlK\nStVA+puCkOGGaqa+LVVDSmkQIUtVC/aHKiAkE4pRcIaqZG/SKLu035sCgYEA1Eet\nLt8E7j9DYqif1cJqgE8fj60X+Z/w6CPB/GHCo5bRIqa/t0X2lJh1Pya7MpI2wLt4\nNolZ9U44w86xuRjBn/aVUBBw1+goBeeiiOAChPGOAdmIcdn1lhGcQxMA4R8mN0/O\nO35VyxFn5rCIMFgbmszzbj/faF+sk/sd10drj5MCgYBy+d5LbaFkokBjUE48r/vo\nY+nrO+qTJUPdECQfmEzPGZOaJ6Zs8wj+pm1yKlBMR55lQLOvr79iL2pXBFtWxhn8\nd6nLJlamK17GeL3PJZZ1LOM9+ohyF5CtRYI3my8ZKLTioQmICowfVhPvh9SaNtls\nzFbpY2OOV7JjRv083GDJPwKBgQChNZW1pLRv4idgg0Ju6WhL7MrBJ+ivL+GZyZ4F\n9pAEAALu18d8fWtXSbiwrs83BocCMtyGPiTNAAxn105sjPpuaqrV1MZ9kNbkNRbf\n7466O19m1Dakj6vLva/32DSwwiEZnEe4MqcgwiUvshya6i58juzHa0ZUU2QSNYBh\n/uEWdQKBgGWTbt1fl89585SFldW73RxIjw7Jb4gSVq4k8fG3KUhOqhrMRlpZbtAi\nij3DrDl1M5g81NVZflOthoy3UoZjxfIx56hPUnbLPH/llU+HeIGjHEmyFWIcklmR\nhyi5y1dzuxBB4zL1aExiO0mGI+Q/XtZdiwSEvYPFQJ7UYSX8NQsG\n-----END RSA PRIVATE KEY-----\n"
  },
  {
    "path": "dist/test/ca.csr",
    "content": "-----BEGIN CERTIFICATE REQUEST-----\nMIICujCCAaICAQAwdTELMAkGA1UEBhMCVVMxFjAUBgNVBAgTDVNhbiBGcmFuY2lz\nY28xCzAJBgNVBAcTAkNBMRgwFgYDVQQKEw9NeSBDb21wYW55IE5hbWUxEzARBgNV\nBAsTCk9yZyBVbml0IDIxEjAQBgNVBAMTCU15IG93biBDQTCCASIwDQYJKoZIhvcN\nAQEBBQADggEPADCCAQoCggEBALuZArLo8NhecbhUiehl6cGu1hQW3GIiQ+28PvYq\nxy9ASFLPTaTYvAenXr9AzO0XiDyQoK5yck/uXYcJhrEN7tDB69MDEMJxw8rb1eOO\nVUJWGn6KMzG9lnXN6y35AExLnbwp/YHLNP+jeDn2PGxi1URaZVh34Q9WvUrEhbxh\nUL5I0SsrCNVGqBA49dy0vGQOt5kt8SNaL3Ahqv57zVEwd9B974hksS3O40t5ahSe\neoTBSFqm6M+4OjR1KmzoGzpyglSNQ5P6ZqXM0KMp25mOmaQN9WpYR/aebZbnPlXY\nNM+C007FjhLPBrVz1odHqYBvu6xsk4W400VSpeVnCESM+wECAwEAAaAAMA0GCSqG\nSIb3DQEBCwUAA4IBAQCBL1hc4uT1BhYZVVFqEXkFWxWpb7R+Ia9Z1ZBpz9isTE1F\nRUqG7shpHrQDqY6uQsEyYC1zHVJDacCXkfQqAiKO7mnB09b4bIkLAxb7glDm83g+\n5ur3lHTkJptWncvisV6B0I6fvPpNnUd9UlOt6EVYLSdMqIY15iISWXIvWPe/cxur\nGf0gc21oK+MHZZa8BdhQlnVGzU5tw1eqXoOsNynSoCdws+M/9kZXL6Nq1859RGMT\nMgZZRDZAD1TBrDTw7LH4b0UE1uqyc2EsJ8FtRuVwQiTePyPPMR8jFh9iHixs52S8\nz/71hCoj/gD0CElA+pnxgGn6hdbTQyjHJhNuGqmg\n-----END CERTIFICATE REQUEST-----\n"
  },
  {
    "path": "dist/test/ca.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDujCCAqKgAwIBAgIURksNLgjamSSvFUWdYjuUq9ML3EAwDQYJKoZIhvcNAQEL\nBQAwdTELMAkGA1UEBhMCVVMxFjAUBgNVBAgTDVNhbiBGcmFuY2lzY28xCzAJBgNV\nBAcTAkNBMRgwFgYDVQQKEw9NeSBDb21wYW55IE5hbWUxEzARBgNVBAsTCk9yZyBV\nbml0IDIxEjAQBgNVBAMTCU15IG93biBDQTAeFw0yMjA4MTkwODUzMDBaFw0yNzA4\nMTgwODUzMDBaMHUxCzAJBgNVBAYTAlVTMRYwFAYDVQQIEw1TYW4gRnJhbmNpc2Nv\nMQswCQYDVQQHEwJDQTEYMBYGA1UEChMPTXkgQ29tcGFueSBOYW1lMRMwEQYDVQQL\nEwpPcmcgVW5pdCAyMRIwEAYDVQQDEwlNeSBvd24gQ0EwggEiMA0GCSqGSIb3DQEB\nAQUAA4IBDwAwggEKAoIBAQC7mQKy6PDYXnG4VInoZenBrtYUFtxiIkPtvD72Kscv\nQEhSz02k2LwHp16/QMztF4g8kKCucnJP7l2HCYaxDe7QwevTAxDCccPK29XjjlVC\nVhp+ijMxvZZ1zest+QBMS528Kf2ByzT/o3g59jxsYtVEWmVYd+EPVr1KxIW8YVC+\nSNErKwjVRqgQOPXctLxkDreZLfEjWi9wIar+e81RMHfQfe+IZLEtzuNLeWoUnnqE\nwUhapujPuDo0dSps6Bs6coJUjUOT+malzNCjKduZjpmkDfVqWEf2nm2W5z5V2DTP\ngtNOxY4Szwa1c9aHR6mAb7usbJOFuNNFUqXlZwhEjPsBAgMBAAGjQjBAMA4GA1Ud\nDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQubXe9u2GsXiEt\nY7fQM4iU7aRthjANBgkqhkiG9w0BAQsFAAOCAQEAVlffrW69znOxFhK6ieZinY75\nz6mm6syfMTLI5QEOCNBaiHG5uFORwFjcA3hafQF59R8Honp2zLArycXnTBkl0v5M\nQlttVmyywbo1k6AuQC3rJgSlGp6/zd0ElNhXRyQw3Y2WSlxmAFzmRlqxyQ+kGZ4e\nFnNm7Fg2SgGrYMfRV1oZ0RNOan95FZFnQ0ZjX046eSTzSJ+k195cHaoLSwuix5ku\n/veiVq13A3/0MBJm3lc3bKhhg/aM8DshDuNQEBSjEb9ULkoAdnyU3XQTiBSP4kLZ\n2to7TbvPhXwpygeC+1vZH05a8F9WoJeGf3mLWG7TB+KeOOCo6pHfmYsDePKWog==\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "dist/test/client-key.pem",
    "content": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIKih7JlJdm3jguVqkAFFRbaPrMNx+szw/3WWc4IJNJcBoAoGCCqGSM49\nAwEHoUQDQgAEriOtzABDnRTaa7Nbp1ahmt7tK9nWWBcYC+THl76ThgeAqy+96524\ney+7DJ/d35MJPQOTpH+zAVlLuFxiZmqrBg==\n-----END EC PRIVATE KEY-----\n"
  },
  {
    "path": "dist/test/client.csr",
    "content": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBHTCBwwIBADBDMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcT\nDVNhbiBGcmFuY2lzY28xDzANBgNVBAMTBmNsaWVudDBZMBMGByqGSM49AgEGCCqG\nSM49AwEHA0IABK4jrcwAQ50U2muzW6dWoZre7SvZ1lgXGAvkx5e+k4YHgKsvveud\nuHsvuwyf3d+TCT0Dk6R/swFZS7hcYmZqqwagHjAcBgkqhkiG9w0BCQ4xDzANMAsG\nA1UdEQQEMAKCADAKBggqhkjOPQQDAgNJADBGAiEAsCnAZ35WWeMi6/pebNoi0Cmg\nI9lwPxoTaE/oAYkWn6YCIQCuuvJ74dqdhFpzNfntujjIr74PNibwWS7CD6g+RCuN\ntw==\n-----END CERTIFICATE REQUEST-----\n"
  },
  {
    "path": "dist/test/client.json",
    "content": "{\n    \"CN\": \"client\",\n    \"hosts\": [\n        \"\"\n    ],\n    \"key\": {\n        \"algo\": \"ecdsa\",\n        \"size\": 256\n    },\n    \"names\": [\n        {\n            \"C\": \"US\",\n            \"ST\": \"CA\",\n            \"L\": \"San Francisco\"\n        }\n    ]\n}\n"
  },
  {
    "path": "dist/test/client.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIC/zCCAeegAwIBAgIUPL5Fd8zSYWlVplN9l26fa834S+cwDQYJKoZIhvcNAQEL\nBQAwdTELMAkGA1UEBhMCVVMxFjAUBgNVBAgTDVNhbiBGcmFuY2lzY28xCzAJBgNV\nBAcTAkNBMRgwFgYDVQQKEw9NeSBDb21wYW55IE5hbWUxEzARBgNVBAsTCk9yZyBV\nbml0IDIxEjAQBgNVBAMTCU15IG93biBDQTAeFw0yMjA4MTkwODU1MDBaFw0yNzA4\nMTgwODU1MDBaMEMxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMN\nU2FuIEZyYW5jaXNjbzEPMA0GA1UEAxMGY2xpZW50MFkwEwYHKoZIzj0CAQYIKoZI\nzj0DAQcDQgAEriOtzABDnRTaa7Nbp1ahmt7tK9nWWBcYC+THl76ThgeAqy+96524\ney+7DJ/d35MJPQOTpH+zAVlLuFxiZmqrBqOBgzCBgDAOBgNVHQ8BAf8EBAMCBaAw\nEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUu5kC\ne6LAJHelXkPkCqFTVkRyzLcwHwYDVR0jBBgwFoAULm13vbthrF4hLWO30DOIlO2k\nbYYwCwYDVR0RBAQwAoIAMA0GCSqGSIb3DQEBCwUAA4IBAQBz1LzcmNmK4VOJktJp\n4UdIuysMIK+sAYd8wATaHl0/p9zKeWKresqPHR/5A+RuqZXTScnQf3HIllYvM2ak\nxH4XdEWRtGSb1yl5XO8N0FnS2shiEDHI8kZ6sNu7xR+0iF4flGQSgsoDXk8QyPFV\ns9cKQ3qytfm0cAIkmLmUxxlFHaDPK1x2B5BsVVdJv+ZzSDYH+dI3wVHSgZ8NxHWQ\ns8wVzoOSBOp0pZe775HJjOTS5Jq51jJAWq8yAyErWOKwIF71qM2iYZ29J2o/Ibid\nMXEXtiMUGD8Q2xOduV9L6/BRj7knxoHTtNyAdEa5y+qYswZwQogHpXd6BfXWld2w\nrIw7\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "dist/test/member1.json",
    "content": "{\n    \"CN\": \"member1\",\n    \"hosts\": [\n        \"127.0.0.1\",\n        \"172.17.0.1\"\n    ],\n    \"key\": {\n        \"algo\": \"ecdsa\",\n        \"size\": 256\n    },\n    \"names\": [\n        {\n            \"C\": \"US\",\n            \"ST\": \"CA\",\n            \"L\": \"San Francisco\"\n        }\n    ]\n}\n"
  },
  {
    "path": "dist/test/server-key.pem",
    "content": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIP8wt7txIaYwVQFNC5Wjr8MRmnCMUtrQirRwvLzTTPNyoAoGCCqGSM49\nAwEHoUQDQgAE9t31xUASqx7TNXaczllMrzW0UyFGx6ypUiHXgm8pZt7D6Rxfjqfx\n9Hfw044/2M3f0DPFiW0MTGM9CLYj4G9pbg==\n-----END EC PRIVATE KEY-----\n"
  },
  {
    "path": "dist/test/server.csr",
    "content": "-----BEGIN CERTIFICATE REQUEST-----\nMIIBJTCBywIBADBBMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcT\nDVNhbiBGcmFuY2lzY28xDTALBgNVBAMTBGV0Y2QwWTATBgcqhkjOPQIBBggqhkjO\nPQMBBwNCAAT23fXFQBKrHtM1dpzOWUyvNbRTIUbHrKlSIdeCbylm3sPpHF+Op/H0\nd/DTjj/Yzd/QM8WJbQxMYz0ItiPgb2luoCgwJgYJKoZIhvcNAQkOMRkwFzAVBgNV\nHREEDjAMhwR/AAABhwSsEQABMAoGCCqGSM49BAMCA0kAMEYCIQCfNFr41VeK7brc\narHQsQMOCjZs9xuK2ZfJHu3iJL31fgIhAPDKgdTTyxfCIsWv0PSDUCkL2kpSMBGI\n1LYOkLeB2uxI\n-----END CERTIFICATE REQUEST-----\n"
  },
  {
    "path": "dist/test/server.json",
    "content": "{\n    \"CN\": \"etcd\",\n    \"hosts\": [\n        \"127.0.0.1\",\n        \"172.17.0.1\" \n    ],\n    \"key\": {\n        \"algo\": \"ecdsa\",\n        \"size\": 256\n    },\n    \"names\": [\n        {\n            \"C\": \"US\",\n            \"ST\": \"CA\",\n            \"L\": \"San Francisco\"\n        }\n    ]\n}\n\n"
  },
  {
    "path": "dist/test/server.pem",
    "content": "-----BEGIN CERTIFICATE-----\nMIIDBzCCAe+gAwIBAgIUIJ5xOJhQ5bqKWcVLHWixDOMWrDkwDQYJKoZIhvcNAQEL\nBQAwdTELMAkGA1UEBhMCVVMxFjAUBgNVBAgTDVNhbiBGcmFuY2lzY28xCzAJBgNV\nBAcTAkNBMRgwFgYDVQQKEw9NeSBDb21wYW55IE5hbWUxEzARBgNVBAsTCk9yZyBV\nbml0IDIxEjAQBgNVBAMTCU15IG93biBDQTAeFw0yMjA4MTkwODU0MDBaFw0yNzA4\nMTgwODU0MDBaMEExCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMN\nU2FuIEZyYW5jaXNjbzENMAsGA1UEAxMEZXRjZDBZMBMGByqGSM49AgEGCCqGSM49\nAwEHA0IABPbd9cVAEqse0zV2nM5ZTK81tFMhRsesqVIh14JvKWbew+kcX46n8fR3\n8NOOP9jN39AzxYltDExjPQi2I+BvaW6jgY0wgYowDgYDVR0PAQH/BAQDAgWgMBMG\nA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFOjrjTWJ\nVmCV9EXfBQ7GJ/KHKoICMB8GA1UdIwQYMBaAFC5td727YaxeIS1jt9AziJTtpG2G\nMBUGA1UdEQQOMAyHBH8AAAGHBKwRAAEwDQYJKoZIhvcNAQELBQADggEBAAc+Q117\nh4IdK8WWudly/B54/MHP0urkIzx9Rx+X16GAttXyAVUiZB6rDk7kwArjlqkLDWeG\nqHmHyvnV/qRzLFPCmt11adwotoMs+ND5ReT7fEZnvWvdAEw/m+Bb6ffiUXqeVzKl\nWrWwHcL1lHs+50rNtx8BxoEkZrewS4Tig5iSGR0zF1Sjxn+Hv0gmRn2+Z/pJf4cy\nfZGVjzbe1ryH6IW/WkeBAvdmvXAt95k06fJWRhmWeeR3i2DHXeHdYF8CLmFTyfSw\niDpyj27+zQlDtOZ3Cgy7nz5rPYvfIZHwzzZRsHwLgw6rJf3DcuqukWFMBGS91L7o\nU9auptmH2ovKXEU=\n-----END CERTIFICATE-----\n"
  },
  {
    "path": "dist/wireguard",
    "content": "{\n  \"Network\": \"10.10.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"wireguard\",\n    \"ListenPort\": 51820,\n    \"PSK\": \"H0Ad2yKBMgfGM6/Yt2dRxd8u+juyViCz3KdJfEdteJ8=\",\n    \"PersistentKeepaliveInterval\": 300\n  }\n}\n"
  },
  {
    "path": "e2e/Dockerfile",
    "content": "# Install the all-in-one binary so we can copy our run-time images into the image\n# which helps avoid pulling them when running e2e tests.\nARG SLES=\"registry.suse.com/suse/sle15:15.7\"\nFROM ${SLES} AS k3s\nARG ARCH\nARG K3S_VERSION=\"v1.32.6+k3s1\"\nARG CNI_VERSION=\"v1.8.0\"\nRUN set -x \\\n && zypper -n in \\\n    ca-certificates \\\n    curl \\\n    tar gzip\\\n    iptables \\\n    nftables \\\n    iproute2 \\\n    iputils \\\n && if [ \"${ARCH?required}\" != \"amd64\" ]; then \\\n        K3S_SUFFIX=\"-${ARCH}\"; \\\n    fi \\\n && curl -fsSL \"https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s${K3S_SUFFIX}\" > /bin/k3s \\\n && chmod +x /bin/k3s \\\n && ln -s /bin/k3s /bin/kubectl \\\n && ln -s /bin/k3s /bin/ctr \\\n && k3s --version\n#install cni plugins manually since we will only install flannel later\nRUN mkdir -p /opt/cni/bin \\\n && curl -L \"https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz\" | tar -C /opt/cni/bin -xz\n  \nCOPY scratch/*.tar /var/lib/rancher/k3s/agent/images/\nRUN ls -l /var/lib/rancher/k3s/agent/images/\n#ADD https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s-airgap-images-${ARCH}.tar /var/lib/rancher/k3s/agent/images/\n\nVOLUME /var/lib/cni\nVOLUME /var/lib/kubelet\nVOLUME /var/lib/rancher/k3s\nVOLUME /var/log\n\nENTRYPOINT [\"k3s\"]\nCMD [\"server\"]\n"
  },
  {
    "path": "e2e/docker-compose.yml",
    "content": "version: \"3.7\"\n\nvolumes:\n  kubeconfig:\n    name: e2e-local-kubeconfig\n\nservices:\n\n  leader:\n    container_name: local-leader\n    image: \"e2e/cluster/local/k3s\"\n    build:\n      context: .\n      args:\n        - ARCH=amd64\n    command: server --disable=traefik,metrics-server --flannel-backend=none --disable-network-policy\n    environment:\n      - K3S_TOKEN=e2e\n    hostname: local-leader\n    privileged: true\n    ports:\n      - \"6443:6443\" # k3s\n    volumes:\n      - source: kubeconfig\n        target: /etc/rancher/k3s\n        type: volume\n\n  worker:\n    depends_on:\n      - leader\n    container_name: local-worker\n    image: \"e2e/cluster/local/k3s\"\n    build:\n      context: .\n      args:\n        - ARCH=amd64\n    command: agent --server https://local-leader:6443\n    environment:\n      - K3S_TOKEN=e2e\n    hostname: local-worker\n    privileged: true\n"
  },
  {
    "path": "e2e/download-kubectl.sh",
    "content": "#!/usr/bin/env bash\n\nset -e -o pipefail\n\nsource $(dirname $0)/version.sh\n\nRELEASE=\"$(curl -sSL https://dl.k8s.io/release/stable.txt)\"\npushd /usr/local/bin\nsudo curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/${ARCH:-amd64}/kubectl\nsudo chmod +x kubectl\npopd\n"
  },
  {
    "path": "e2e/e2e-functions.sh",
    "content": "#!/usr/bin/env bash\n\n# ---\n\ne2e-wait-for-kubeconfig() {\n    set -e -o pipefail\n    # the `--insecure-skip-tls-verify` seems to be only needed here when run in dapper\n    while ! kubectl --kubeconfig=\"${HOME}/.kube/config\" --insecure-skip-tls-verify get pods -A >/dev/null 2>&1 ; do\n        echo 'Waiting for kubeconfig to become available...' >&2\n        sleep 5\n        cluster-get-kubeconfig >/dev/null\n    done\n}\nexport -f e2e-wait-for-kubeconfig\n\ne2e-count-ready-nodes() {\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" get nodes -o json \\\n        | jq '.items[].status.conditions[] | select(.type == \"Ready\" and .status == \"True\") | .type' \\\n        | wc -l \\\n        | tr -d '[:space:]'\n}\nexport -f e2e-count-ready-nodes\n\ne2e-wait-for-nodes() {\n    while [[ $(e2e-count-ready-nodes) -lt 2 ]]; do\n        echo 'Waiting for nodes to be ready...' >&2\n        echo \"*** nodes:\"\n        kubectl --kubeconfig=\"${HOME}/.kube/config\" get nodes\n        # echo \"*** events:\"\n        # kubectl --kubeconfig=\"${HOME}/.kube/config\" get events --sort-by='.lastTimestamp' -A\n        sleep 5\n    done\n    echo \"*** nodes are ready:\"\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" get nodes\n}\nexport -f e2e-wait-for-nodes\n\ne2e-pod-ready() {\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" get pods -A -o json \\\n        | jq \".items[].status.containerStatuses[] | select(.name == \\\"$1\\\") | .ready\" 2>/dev/null\n}\nexport -f e2e-pod-ready\n\ne2e-wait-for-services() {\n    for svc in ${WAIT_FOR_SERVICES:=\"coredns local-path-provisioner\"}; do\n        while [[ \"$(e2e-pod-ready $svc)\" != 'true' ]]; do\n            echo \"Waiting for service '$svc' to be ready...\" >&2\n            sleep 5\n        done\n        echo \"Service '$svc' is ready\"\n    done\n}\nexport -f e2e-wait-for-services\n\ne2e-get-flannel-pod() {\n    local node_name=$1\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" get pods --field-selector \"spec.nodeName=${node_name}\" -n kube-flannel --no-headers\n}\nexport -f e2e-get-flannel-pod\n\ne2e-wait-for-test-pods() {\n    for pod in ${WAIT_FOR_PODS:=\"multitool1 multitool2\"}; do\n        while [[ \"$(e2e-pod-ready $pod)\" != 'true' ]]; do\n            echo \"Waiting for pod '$pod' to be ready...\" >&2\n            sleep 5\n        done\n        echo \"Pod '$pod' is ready\"\n    done\n}\nexport -f e2e-wait-for-test-pods\n\ne2e-wait-for-ping() {\n    pod=$1\n    ip=$2\n\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" exec ${pod} -- ping -c 1 ${ip}\n    result=$?\n    while [ $result -ne 0 ]; do\n        echo \"Waiting for ${ip} to reply to ping from ${pod}...\" >&2\n        sleep 2\n        kubectl --kubeconfig=\"${HOME}/.kube/config\" exec ${pod} -- ping -c 1 ${ip}\n        result=$?\n    done\n    echo \"IP ${ip} is ready\"\n    return 0\n}\nexport -f e2e-wait-for-ping\n\n# ---\n\n"
  },
  {
    "path": "e2e/get-kubeconfig.sh",
    "content": "#!/usr/bin/env bash\n\nset -e -o pipefail\n\nexport KUBECONFIG=\"${HOME}/.kube/config\"\n\nmkdir -vp \"$(dirname $KUBECONFIG)\"\nwhile ! kubectl --insecure-skip-tls-verify get pods -A >/dev/null 2>&1 ; do\n    echo 'Waiting for kubeconfig to become available...' >&2\n    sleep 5\n    docker exec local-leader kubectl config view --raw | sed -e \"s/127.0.0.1/${KUBEHOST:=127.0.0.1}/g\" > \"${KUBECONFIG}\"\ndone\n"
  },
  {
    "path": "e2e/run-e2e-tests.sh",
    "content": "#!/bin/bash\n\nset -e -o pipefail\n\nsource $(dirname $0)/version.sh\nsource $(dirname $0)/e2e-functions.sh\n\nFLANNEL_NET=\"${FLANNEL_NET:-10.42.0.0/16}\"\nFLANNEL_IP6NET=\"${FLANNEL_IP6NET:-2001:cafe:42:0::/56}\"\n# needs to be exported for yq\nexport FLANNEL_IMAGE=\"quay.io/coreos/flannel:${TAG}-${ARCH}\"\n\n\nsetup_suite() {\n    # copy flannel image built by `make image` to docker compose context folder\n    rm -rf $(dirname $0)/scratch\n    mkdir -p $(dirname $0)/scratch\n    cp $(dirname $0)/../dist/${FLANNEL_IMAGE_FILE}.docker $(dirname $0)/scratch/${FLANNEL_IMAGE_FILE}.tar\n\n    $(dirname $0)/download-kubectl.sh\n}\n\ncreate_test_pod() {\n    local pod_name=$1\n    local worker_node=$2\n    cat <<EOF | kubectl --kubeconfig=\"${HOME}/.kube/config\" apply -f -\napiVersion: v1\nkind: Pod\nmetadata:\n  name: ${pod_name}\nspec:\n  containers:\n  - name: ${pod_name}\n    image: wbitt/network-multitool:alpine-extra\n  nodeName: ${worker_node}\nEOF\n}\n\nwrite-flannel-conf(){\n    local backend=$1\n    local enable_nftables=$2\n    cp ../Documentation/kube-flannel.yml ./kube-flannel.yml\n    yq -i 'select(.kind == \"DaemonSet\").spec.template.spec.containers[0].image |= strenv(FLANNEL_IMAGE)' ./kube-flannel.yml\n    yq -i 'select(.kind == \"DaemonSet\").spec.template.spec.initContainers[1].image |= strenv(FLANNEL_IMAGE)' ./kube-flannel.yml\n\n    export flannel_conf=\"{ \\\"Network\\\": \\\"$FLANNEL_NET\\\", \\\"Backend\\\": { \\\"Type\\\": \\\"${backend}\\\" }, \\\"EnableNFTables\\\": ${enable_nftables} }\"\n\n    yq -i 'select(.metadata.name == \"kube-flannel-cfg\").data.\"net-conf.json\" |= strenv(flannel_conf)' ./kube-flannel.yml\n\n    # udp backend needs to run in \"privileged\" mode to access /dev/net/tun\n    if [ \"$backend\" = \"udp\" ]; then\n        yq -i 'select(.kind == \"DaemonSet\").spec.template.spec.containers[0].securityContext.privileged |= true'  kube-flannel.yml\n    fi\n}\n\n# This is not used at the moment since github runners don't support dual-stack networking\nwrite-flannel-conf-dual-stack(){\n    local backend=$1\n    local enable_nftables=$2\n    cp ../Documentation/kube-flannel.yml ./kube-flannel.yml\n    yq -i 'select(.kind == \"DaemonSet\").spec.template.spec.containers[0].image |= strenv(FLANNEL_IMAGE)' ./kube-flannel.yml\n\n    export flannel_conf=\"{ \\\"EnableIPv6\\\": true, \\\"Network\\\": \\\"$FLANNEL_NET\\\", \\\"IPv6Network\\\":\\\"${FLANNEL_IP6NET}\\\", \\\"Backend\\\": { \\\"Type\\\": \\\"${backend}\\\" }, \\\"EnableNFTables\\\": ${enable_nftables} }\"\n\n    yq -i 'select(.metadata.name == \"kube-flannel-cfg\").data.\"net-conf.json\" |= strenv(flannel_conf)' ./kube-flannel.yml\n}\n\ninstall-flannel() {\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" apply -f ./kube-flannel.yml\n}\n\ndelete-flannel() {\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" delete -f ./kube-flannel.yml\n}\n\nget_pod_ip() {\n    local pod_name=$1\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" get pod ${pod_name} --template '{{.status.podIP}}'\n}\n\nget_pod_cidr() {\n    local node_name=$1\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" get node ${node_name} --template '{{.spec.podCIDR}}'\n}\n\nget_pod_logs() {\n    local pod_name=$1\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" logs $pod_name -n kube-flannel\n}\n\npings() {\n    create_test_pod multitool1 local-worker\n    create_test_pod multitool2 local-leader\n\n    # wait for test-pods to be ready\n    echo \"wait for test-pods to be ready...\"\n    timeout --foreground 1m bash -c \"e2e-wait-for-test-pods\"\n    retVal=$?\n    if [ $retVal -ne 0 ]; then\n        echo \"test pods not ready in time. Checking their status...\"\n        kubectl --kubeconfig=\"${HOME}/.kube/config\" get events --sort-by='.lastTimestamp' -A\n        echo \"*****************************\"\n        echo \"flannel pod log:\"\n        flannel_pod=$(e2e-get-flannel-pod local-worker)\n        echo \"$(get_pod_logs $flannel_pod)\"\n        exit $retVal\n    fi\n    \n    ip_1=$(get_pod_ip multitool1)\n    ip_2=$(get_pod_ip multitool2)\n\n    echo \"multitool1 IP is: ${ip_1}\"\n    echo \"multitool2 IP is: ${ip_2}\"\n\n    timeout --foreground 1m bash -c \"e2e-wait-for-ping multitool1 ${ip_2}\"\n\n    assert \"kubectl --kubeconfig=\"${HOME}/.kube/config\" exec multitool1 -- ping -c 5 ${ip_2}\"\n    assert \"kubectl --kubeconfig=\"${HOME}/.kube/config\" exec multitool2 -- ping -c 5 ${ip_1}\"\n}\n\nperf() {\n    create_test_pod multitool1 local-worker\n    create_test_pod multitool2 local-leader\n\n    # wait for test-pods to be ready\n    echo \"wait for test-pods to be ready...\"\n    timeout --foreground 1m bash -c \"e2e-wait-for-test-pods\"\n\n    ip_1=$(get_pod_ip multitool1)\n    ip_2=$(get_pod_ip multitool2)\n\n    echo \"starting iperf3 server...\" >&2\n    kubectl --kubeconfig=\"${HOME}/.kube/config\" exec multitool1 -- iperf3 -s &\n    sleep 5\n    echo \"starting iperf3 client...\" >&2\n    assert \"kubectl --kubeconfig=\"${HOME}/.kube/config\" exec multitool2 -- iperf3 -c ${ip_1} -t 10\"\n}\n\nprepare_test() {\n    local backend=$1\n    local enable_nftables=${2:-false}\n    # install flannel version to test\n    write-flannel-conf ${backend} ${enable_nftables}\n    \n    install-flannel\n    \n    # wait for nodes to be ready\n    timeout --foreground 5m bash -c \"e2e-wait-for-nodes\"\n    retVal=$?\n    if [ $retVal -ne 0 ]; then\n        echo \"test nodes not ready in time. Checking their status...\"\n        kubectl --kubeconfig=\"${HOME}/.kube/config\" get events --sort-by='.lastTimestamp' -A\n        echo \"*****************************\"\n        echo \"flannel pod log:\"\n        flannel_pod=$(e2e-get-flannel-pod local-worker)\n        echo \"$(get_pod_logs $flannel_pod)\"\n        exit $retVal\n    fi\n    # wait for services to be ready\n    echo \"wait for services to be ready...\"\n    timeout --foreground 5m bash -c \"e2e-wait-for-services\"\n}\n\nsetup() {\n    # start k3s cluster\n    echo \"starting k3s cluster...\"\n    docker compose up -d\n    # get kubeconfig for the k3s cluster\n    echo \"waiting for kubeconfig to be available...\"\n    ./get-kubeconfig.sh\n    echo \"kubeconfig is at \"${HOME}/.kube/config\":\"\n}\n\ntest_vxlan() {\n    prepare_test vxlan\n    pings\n    check_iptables\n    delete-flannel\n}\n\ntest_vxlan_nft() {\n    prepare_test vxlan true\n    pings\n    check_nftables\n    delete-flannel\n}\n\ntest_wireguard() {\n    prepare_test wireguard\n    pings\n    check_iptables\n    delete-flannel\n}\n\ntest_host-gw() {\n    prepare_test host-gw\n    pings\n    check_iptables\n    delete-flannel\n}\n\nif [[ ${ARCH} == \"amd64\" ]]; then\ntest_udp() {\n    prepare_test udp\n    pings\n    check_iptables\n    delete-flannel\n}\nfi\n\ntest_ipip() {\n    prepare_test ipip\n    pings\n    check_iptables\n    delete-flannel\n}\n\ntest_perf_vxlan() {\n    prepare_test vxlan\n    perf\n}\n\ntest_perf_wireguard() {\n    prepare_test wireguard\n    perf\n}\n\ntest_perf_host-gw() {\n    prepare_test host-gw\n    perf\n}\n\ntest_perf_ipip() {\n    prepare_test ipip\n    perf\n}\n\nif [[ ${ARCH} == \"amd64\" ]]; then\n    test_perf_udp() {\n        prepare_test udp\n        perf\n    }\nfi\n\nteardown() {\n    docker compose down   \\\n        --remove-orphans \\\n        --rmi all \\\n        --volumes\n}\n\ncheck_iptables() {\n  local worker_podcidr=$(get_pod_cidr local-worker)\n  local leader_pod_cidr=$(get_pod_cidr local-leader)\n  read -r -d '' POSTROUTING_RULES_WORKER << EOM\n-A POSTROUTING -m comment --comment \"flanneld masq\" -j FLANNEL-POSTRTG\n-N FLANNEL-POSTRTG\n-A FLANNEL-POSTRTG -m mark --mark 0x4000/0x4000 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s ${worker_podcidr} -d 10.42.0.0/16 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.42.0.0/16 -d ${worker_podcidr} -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG ! -s 10.42.0.0/16 -d ${worker_podcidr} -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.42.0.0/16 ! -d 224.0.0.0/4 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\n-A FLANNEL-POSTRTG ! -s 10.42.0.0/16 -d 10.42.0.0/16 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\nEOM\n  read -r -d '' POSTROUTING_RULES_LEADER << EOM\n-A POSTROUTING -m comment --comment \"flanneld masq\" -j FLANNEL-POSTRTG\n-N FLANNEL-POSTRTG\n-A FLANNEL-POSTRTG -m mark --mark 0x4000/0x4000 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s ${leader_pod_cidr} -d 10.42.0.0/16 -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.42.0.0/16 -d ${leader_pod_cidr} -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG ! -s 10.42.0.0/16 -d ${leader_pod_cidr} -m comment --comment \"flanneld masq\" -j RETURN\n-A FLANNEL-POSTRTG -s 10.42.0.0/16 ! -d 224.0.0.0/4 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\n-A FLANNEL-POSTRTG ! -s 10.42.0.0/16 -d 10.42.0.0/16 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\nEOM\n  read -r -d '' FORWARD_RULES << EOM\n-P FORWARD ACCEPT\n-A FORWARD -m conntrack --ctstate NEW -m comment --comment \"kubernetes load balancer firewall\" -j KUBE-PROXY-FIREWALL\n-A FORWARD -m comment --comment \"kubernetes forwarding rules\" -j KUBE-FORWARD\n-A FORWARD -m conntrack --ctstate NEW -m comment --comment \"kubernetes service portals\" -j KUBE-SERVICES\n-A FORWARD -m conntrack --ctstate NEW -m comment --comment \"kubernetes externally-visible service portals\" -j KUBE-EXTERNAL-SERVICES\n-A FORWARD -m comment --comment \"flanneld forward\" -j FLANNEL-FWD\n-N FLANNEL-FWD\n-A FLANNEL-FWD -s 10.42.0.0/16 -m comment --comment \"flanneld forward\" -j ACCEPT\n-A FLANNEL-FWD -d 10.42.0.0/16 -m comment --comment \"flanneld forward\" -j ACCEPT\nEOM\n  # check masquerade & forward rules\n  assert_equals \"$POSTROUTING_RULES_WORKER\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/iptables -t nat -S POSTROUTING | grep FLANNEL)\n$(docker exec --privileged local-worker /usr/sbin/iptables -t nat -S FLANNEL-POSTRTG)\" \"Host 1 has not expected postrouting rules\"\n  assert_equals \"$POSTROUTING_RULES_LEADER\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/iptables -t nat -S POSTROUTING | grep FLANNEL)\n$(docker exec --privileged local-leader /usr/sbin/iptables -t nat -S FLANNEL-POSTRTG)\" \"Host 2 has not expected postrouting rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FORWARD)\n$(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FLANNEL-FWD -w 5)\" \"Host 1 has not expected forward rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FORWARD)\n$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FLANNEL-FWD)\" \"Host 2 has not expected forward rules\"\n}\n\ncheck_iptables_removed() {\n  local worker_podcidr=$(get_pod_cidr local-worker)\n  local leader_pod_cidr=$(get_pod_cidr local-leader)\n  read -r -d '' POSTROUTING_RULES_WORKER << EOM\n-N FLANNEL-POSTRTG\nEOM\n  read -r -d '' POSTROUTING_RULES_LEADER << EOM\n-N FLANNEL-POSTRTG\nEOM\n  read -r -d '' FORWARD_RULES << EOM\n-P FORWARD ACCEPT\n-A FORWARD -m conntrack --ctstate NEW -m comment --comment \"kubernetes load balancer firewall\" -j KUBE-PROXY-FIREWALL\n-A FORWARD -m comment --comment \"kubernetes forwarding rules\" -j KUBE-FORWARD\n-A FORWARD -m conntrack --ctstate NEW -m comment --comment \"kubernetes service portals\" -j KUBE-SERVICES\n-A FORWARD -m conntrack --ctstate NEW -m comment --comment \"kubernetes externally-visible service portals\" -j KUBE-EXTERNAL-SERVICES\n-N FLANNEL-FWD\nEOM\n# check that masquerade & forward rules have been removed\n  assert_equals \"$POSTROUTING_RULES_WORKER\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/iptables -t nat -S POSTROUTING | grep FLANNEL)$(docker exec --privileged local-worker /usr/sbin/iptables -t nat -S FLANNEL-POSTRTG)\" \"Host 1 has not expected postrouting rules\"\n  assert_equals \"$POSTROUTING_RULES_LEADER\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/iptables -t nat -S POSTROUTING | grep FLANNEL)$(docker exec --privileged local-leader /usr/sbin/iptables -t nat -S FLANNEL-POSTRTG)\" \"Host 2 has not expected postrouting rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FORWARD)\n$(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FLANNEL-FWD -w 5)\" \"Host 1 has not expected forward rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FORWARD)\n$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FLANNEL-FWD)\" \"Host 2 has not expected forward rules\"\n}\n\n###nftables\ncheck_nftables() {\n  local worker_podcidr=$(get_pod_cidr local-worker)\n  local leader_podcidr=$(get_pod_cidr local-leader)\n  read -d '' POSTROUTING_RULES_WORKER << EOM\ntable ip flannel-ipv4 {\n\tchain postrtg {\n\t\tcomment \"chain to manage traffic masquerading by flannel\"\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n\t\tmeta mark 0x00004000 return\n\t\tip saddr ${worker_podcidr} ip daddr 10.42.0.0/16 return\n\t\tip saddr 10.42.0.0/16 ip daddr ${worker_podcidr} return\n\t\tip saddr != ${worker_podcidr} ip daddr 10.42.0.0/16 return\n\t\tip saddr 10.42.0.0/16 ip daddr != 224.0.0.0/4 masquerade fully-random\n\t\tip saddr != 10.42.0.0/16 ip daddr 10.42.0.0/16 masquerade fully-random\n\t}\n}\nEOM\n  read -r -d '' POSTROUTING_RULES_LEADER << EOM\ntable ip flannel-ipv4 {\n\tchain postrtg {\n\t\tcomment \"chain to manage traffic masquerading by flannel\"\n\t\ttype nat hook postrouting priority srcnat; policy accept;\n\t\tmeta mark 0x00004000 return\n\t\tip saddr ${leader_podcidr} ip daddr 10.42.0.0/16 return\n\t\tip saddr 10.42.0.0/16 ip daddr ${leader_podcidr} return\n\t\tip saddr != ${leader_podcidr} ip daddr 10.42.0.0/16 return\n\t\tip saddr 10.42.0.0/16 ip daddr != 224.0.0.0/4 masquerade fully-random\n\t\tip saddr != 10.42.0.0/16 ip daddr 10.42.0.0/16 masquerade fully-random\n\t}\n}\nEOM\n  read -r -d '' FORWARD_RULES << EOM\ntable ip flannel-ipv4 {\n\tchain forward {\n\t\tcomment \"chain to accept flannel traffic\"\n\t\ttype filter hook forward priority filter; policy accept;\n\t\tip saddr 10.42.0.0/16 accept\n\t\tip daddr 10.42.0.0/16 accept\n\t}\n}\nEOM\n  # check masquerade & forward rules\n  assert_equals \"$POSTROUTING_RULES_WORKER\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 postrtg)\" \"Node worker does not have expected postrouting rules\"\n  assert_equals \"$POSTROUTING_RULES_LEADER\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 postrtg)\" \"Node leader does not have expected postrouting rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 forward)\" \"Node worker does not have expected forward rules\"\n  assert_equals \"$FORWARD_RULES\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 forward)\" \"Node leader does not have expected forward rules\"\n}\n\ncheck_nftables_removed() {\n  # check masquerade & forward rules\n  assert_equals \"\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 postrtg)\" \"Node worker has unexpected postrouting rules\"\n  assert_equals \"\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 postrtg)\" \"Node leader has unexpected postrouting rules\"\n  assert_equals \"\" \\\n                \"$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 forward)\" \"Node worker has unexpected forward rules\"\n  assert_equals \"\" \\\n                \"$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 forward)\" \"Node leader has unexpected forward rules\"\n}"
  },
  {
    "path": "e2e/version.sh",
    "content": "#!/bin/bash\n\nset -e -o pipefail\n\nexport TAG=$(git describe --tags --always)\nexport ARCH=amd64\nexport FLANNEL_IMAGE_FILE=flanneld-${TAG}-${ARCH}\n"
  },
  {
    "path": "go.mod",
    "content": "module github.com/flannel-io/flannel\n\ngo 1.24.7\n\n// replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.2.0\n\nrequire (\n\tgithub.com/Microsoft/hcsshim v0.13.0\n\tgithub.com/bronze1man/goStrongswanVici v0.0.0-20231128135937-211cef3b0b20\n\tgithub.com/containernetworking/plugins v1.9.0\n\tgithub.com/coreos/go-iptables v0.8.0\n\tgithub.com/coreos/go-systemd/v22 v22.7.0\n\tgithub.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb\n\tgithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect\n\tgithub.com/joho/godotenv v1.5.1\n\tgithub.com/jonboulle/clockwork v0.5.0\n\tgithub.com/pkg/errors v0.9.1\n\tgithub.com/vishvananda/netlink v1.3.1\n\tgithub.com/vishvananda/netns v0.0.5\n\tgo.uber.org/zap v1.27.0 // indirect\n\tgolang.org/x/net v0.48.0 // indirect\n\tgolang.org/x/oauth2 v0.34.0 // indirect\n\tgolang.org/x/time v0.9.0 // indirect\n\tgolang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect\n\tgolang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6\n\tk8s.io/api v0.32.10\n\tk8s.io/apimachinery v0.32.10\n\tk8s.io/client-go v0.32.10\n\tk8s.io/klog/v2 v2.130.1\n)\n\nrequire (\n\tgithub.com/avast/retry-go/v4 v4.7.0\n\tgithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.3.51\n\tgithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc v1.3.49\n\tgolang.org/x/sync v0.19.0\n\tsigs.k8s.io/knftables v0.0.18\n)\n\nrequire (\n\tgithub.com/VividCortex/ewma v1.2.0 // indirect\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/bgentry/speakeasy v0.2.0 // indirect\n\tgithub.com/cenkalti/backoff/v4 v4.3.0 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.3.0 // indirect\n\tgithub.com/cheggaaa/pb/v3 v3.1.6 // indirect\n\tgithub.com/containerd/cgroups/v3 v3.0.3 // indirect\n\tgithub.com/containerd/errdefs v0.3.0 // indirect\n\tgithub.com/containerd/errdefs/pkg v0.3.0 // indirect\n\tgithub.com/containerd/typeurl/v2 v2.2.0 // indirect\n\tgithub.com/dustin/go-humanize v1.0.1 // indirect\n\tgithub.com/emicklei/go-restful/v3 v3.11.0 // indirect\n\tgithub.com/fatih/color v1.18.0 // indirect\n\tgithub.com/fxamacker/cbor/v2 v2.7.0 // indirect\n\tgithub.com/go-logr/stdr v1.2.2 // indirect\n\tgithub.com/go-openapi/jsonpointer v0.21.0 // indirect\n\tgithub.com/go-openapi/jsonreference v0.20.2 // indirect\n\tgithub.com/go-openapi/swag v0.23.0 // indirect\n\tgithub.com/golang-jwt/jwt/v5 v5.2.2 // indirect\n\tgithub.com/google/btree v1.1.3 // indirect\n\tgithub.com/google/gnostic-models v0.6.8 // indirect\n\tgithub.com/google/uuid v1.6.0 // indirect\n\tgithub.com/gorilla/websocket v1.5.0 // indirect\n\tgithub.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect\n\tgithub.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect\n\tgithub.com/inconshreveable/mousetrap v1.1.0 // indirect\n\tgithub.com/josharian/intern v1.0.0 // indirect\n\tgithub.com/klauspost/compress v1.17.9 // indirect\n\tgithub.com/mailru/easyjson v0.7.7 // indirect\n\tgithub.com/mattn/go-colorable v0.1.14 // indirect\n\tgithub.com/mattn/go-isatty v0.0.20 // indirect\n\tgithub.com/mattn/go-runewidth v0.0.16 // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/olekukonko/tablewriter v0.0.5 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect\n\tgithub.com/prometheus/client_golang v1.20.5 // indirect\n\tgithub.com/prometheus/client_model v0.6.1 // indirect\n\tgithub.com/prometheus/common v0.62.0 // indirect\n\tgithub.com/prometheus/procfs v0.15.1 // indirect\n\tgithub.com/rivo/uniseg v0.4.7 // indirect\n\tgithub.com/soheilhy/cmux v0.1.5 // indirect\n\tgithub.com/spf13/cobra v1.9.1 // indirect\n\tgithub.com/stretchr/testify v1.11.1 // indirect\n\tgithub.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect\n\tgithub.com/x448/float16 v0.8.4 // indirect\n\tgithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect\n\tgo.etcd.io/bbolt v1.4.3 // indirect\n\tgo.etcd.io/etcd/etcdctl/v3 v3.6.8 // indirect\n\tgo.etcd.io/etcd/pkg/v3 v3.6.8 // indirect\n\tgo.etcd.io/etcd/server/v3 v3.6.8 // indirect\n\tgo.etcd.io/gofail v0.2.0 // indirect\n\tgo.etcd.io/raft/v3 v3.6.0 // indirect\n\tgo.opentelemetry.io/auto/sdk v1.2.1 // indirect\n\tgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect\n\tgo.opentelemetry.io/otel v1.40.0 // indirect\n\tgo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect\n\tgo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect\n\tgo.opentelemetry.io/otel/metric v1.40.0 // indirect\n\tgo.opentelemetry.io/otel/sdk v1.40.0 // indirect\n\tgo.opentelemetry.io/otel/trace v1.40.0 // indirect\n\tgo.opentelemetry.io/proto/otlp v1.5.0 // indirect\n\tgoogle.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect\n\tgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect\n\tgopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect\n\tgopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tsigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect\n)\n\nrequire (\n\tgithub.com/Microsoft/go-winio v0.6.2 // indirect\n\tgithub.com/coreos/go-semver v0.3.1 // indirect\n\tgithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect\n\tgithub.com/go-logr/logr v1.4.3 // indirect\n\tgithub.com/gogo/protobuf v1.3.2 // indirect\n\tgithub.com/golang/protobuf v1.5.4 // indirect\n\tgithub.com/google/go-cmp v0.7.0 // indirect\n\tgithub.com/google/gofuzz v1.2.0 // indirect\n\tgithub.com/josharian/native v1.1.0 // indirect\n\tgithub.com/json-iterator/go v1.1.12 // indirect\n\tgithub.com/mdlayher/genetlink v1.3.2 // indirect\n\tgithub.com/mdlayher/netlink v1.7.2 // indirect\n\tgithub.com/mdlayher/socket v0.5.1 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.2 // indirect\n\tgithub.com/sirupsen/logrus v1.9.3 // indirect\n\tgithub.com/spf13/pflag v1.0.6 // indirect\n\tgo.etcd.io/etcd/api/v3 v3.6.8\n\tgo.etcd.io/etcd/client/pkg/v3 v3.6.8\n\tgo.etcd.io/etcd/client/v3 v3.6.8\n\tgo.etcd.io/etcd/tests/v3 v3.6.8\n\tgo.opencensus.io v0.24.0 // indirect\n\tgo.uber.org/multierr v1.11.0 // indirect\n\tgolang.org/x/crypto v0.46.0 // indirect\n\tgolang.org/x/sys v0.41.0\n\tgolang.org/x/term v0.38.0 // indirect\n\tgolang.org/x/text v0.32.0 // indirect\n\tgoogle.golang.org/grpc v1.79.3 // indirect\n\tgoogle.golang.org/protobuf v1.36.10 // indirect\n\tgopkg.in/inf.v0 v0.9.1 // indirect\n\tk8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect\n\tk8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect\n\tsigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect\n\tsigs.k8s.io/yaml v1.4.0 // indirect\n)\n"
  },
  {
    "path": "go.sum",
    "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=\ngithub.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=\ngithub.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=\ngithub.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=\ngithub.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA=\ngithub.com/Microsoft/hcsshim v0.13.0/go.mod h1:9KWJ/8DgU+QzYGupX4tzMhRQE8h6w90lH6HAaclpEok=\ngithub.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=\ngithub.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=\ngithub.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio=\ngithub.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E=\ngithub.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=\ngithub.com/bronze1man/goStrongswanVici v0.0.0-20231128135937-211cef3b0b20 h1:JMoL5xJSYxo1QVJ3c+4FutWQnks3gZX9DYkgAnvg+5g=\ngithub.com/bronze1man/goStrongswanVici v0.0.0-20231128135937-211cef3b0b20/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc=\ngithub.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=\ngithub.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=\ngithub.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cheggaaa/pb/v3 v3.1.6 h1:h0x+vd7EiUohAJ29DJtJy+SNAc55t/elW3jCD086EXk=\ngithub.com/cheggaaa/pb/v3 v3.1.6/go.mod h1:urxmfVtaxT+9aWk92DbsvXFZtNSWQSO5TRAp+MJ3l1s=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=\ngithub.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=\ngithub.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=\ngithub.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=\ngithub.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4=\ngithub.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=\ngithub.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=\ngithub.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=\ngithub.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso=\ngithub.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g=\ngithub.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo=\ngithub.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4=\ngithub.com/containernetworking/plugins v1.9.0 h1:Mg3SXBdRGkdXyFC4lcwr6u2ZB2SDeL6LC3U+QrEANuQ=\ngithub.com/containernetworking/plugins v1.9.0/go.mod h1:JG3BxoJifxxHBhG3hFyxyhid7JgRVBu/wtooGEvWf1c=\ngithub.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNsPRc=\ngithub.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=\ngithub.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=\ngithub.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=\ngithub.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=\ngithub.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=\ngithub.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE=\ngithub.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=\ngithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=\ngithub.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=\ngithub.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=\ngithub.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=\ngithub.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=\ngithub.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=\ngithub.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=\ngithub.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=\ngithub.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=\ngithub.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=\ngithub.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=\ngithub.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=\ngithub.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=\ngithub.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=\ngithub.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=\ngithub.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=\ngithub.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=\ngithub.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=\ngithub.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=\ngithub.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=\ngithub.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=\ngithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=\ngithub.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=\ngithub.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=\ngithub.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=\ngithub.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=\ngithub.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=\ngithub.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=\ngithub.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=\ngithub.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=\ngithub.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=\ngithub.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA=\ngithub.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU=\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI=\ngithub.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=\ngithub.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=\ngithub.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=\ngithub.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=\ngithub.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=\ngithub.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=\ngithub.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=\ngithub.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=\ngithub.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=\ngithub.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\ngithub.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=\ngithub.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=\ngithub.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=\ngithub.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=\ngithub.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=\ngithub.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=\ngithub.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=\ngithub.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=\ngithub.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=\ngithub.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=\ngithub.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=\ngithub.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=\ngithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=\ngithub.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=\ngithub.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=\ngithub.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=\ngithub.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=\ngithub.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=\ngithub.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=\ngithub.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=\ngithub.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=\ngithub.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=\ngithub.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=\ngithub.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=\ngithub.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=\ngithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=\ngithub.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=\ngithub.com/onsi/ginkgo/v2 v2.25.1 h1:Fwp6crTREKM+oA6Cz4MsO8RhKQzs2/gOIVOUscMAfZY=\ngithub.com/onsi/ginkgo/v2 v2.25.1/go.mod h1:ppTWQ1dh9KM/F1XgpeRqelR+zHVwV81DGRSDnFxK7Sk=\ngithub.com/onsi/gomega v1.38.1 h1:FaLA8GlcpXDwsb7m0h2A9ew2aTk3vnZMlzFgg5tz/pk=\ngithub.com/onsi/gomega v1.38.1/go.mod h1:LfcV8wZLvwcYRwPiJysphKAEsmcFnLMK/9c+PjvlX8g=\ngithub.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=\ngithub.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=\ngithub.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=\ngithub.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=\ngithub.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=\ngithub.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=\ngithub.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=\ngithub.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=\ngithub.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=\ngithub.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=\ngithub.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=\ngithub.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=\ngithub.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=\ngithub.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=\ngithub.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=\ngithub.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=\ngithub.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=\ngithub.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=\ngithub.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=\ngithub.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngithub.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=\ngithub.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.3.49/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.3.51 h1:yuvTAokQAdxbCr06NGOdPJpgO3z46IDinINBM0r9R9I=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.3.51/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc v1.3.49 h1:tKw0aSbhUrZN70lq8o32TojqAZNG2zv9Mm9jGYW1BVA=\ngithub.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc v1.3.49/go.mod h1:UuDgrNmedNzuxlA+wP59t/oFhQZCu7YyGgHoDgoi2QU=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=\ngithub.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=\ngithub.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=\ngithub.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=\ngithub.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=\ngithub.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngo.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=\ngo.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=\ngo.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM=\ngo.etcd.io/etcd/api/v3 v3.6.8/go.mod h1:qyQj1HZPUV3B5cbAL8scG62+fyz5dSxxu0w8pn28N6Q=\ngo.etcd.io/etcd/client/pkg/v3 v3.6.8 h1:Qs/5C0LNFiqXxYf2GU8MVjYUEXJ6sZaYOz0zEqQgy50=\ngo.etcd.io/etcd/client/pkg/v3 v3.6.8/go.mod h1:GsiTRUZE2318PggZkAo6sWb6l8JLVrnckTNfbG8PWtw=\ngo.etcd.io/etcd/client/v3 v3.6.8 h1:B3G76t1UykqAOrbio7s/EPatixQDkQBevN8/mwiplrY=\ngo.etcd.io/etcd/client/v3 v3.6.8/go.mod h1:MVG4BpSIuumPi+ELF7wYtySETmoTWBHVcDoHdVupwt8=\ngo.etcd.io/etcd/etcdctl/v3 v3.6.8 h1:iO3zX9xuY/X2Y+fnt6mUR7o/18iBGYpd/lhs/4FqHCc=\ngo.etcd.io/etcd/etcdctl/v3 v3.6.8/go.mod h1:8X8SvxOc5kPQ0e+jbSx3RgKzTNQ3O8rBuQEoDKuQFX0=\ngo.etcd.io/etcd/pkg/v3 v3.6.8 h1:Xe+LIL974spy8b4nEx3H0KMr1ofq3r0kh6FbU3aw4es=\ngo.etcd.io/etcd/pkg/v3 v3.6.8/go.mod h1:TRibVNe+FqJIe1abOAA1PsuQ4wqO87ZaOoprg09Tn8c=\ngo.etcd.io/etcd/server/v3 v3.6.8 h1:U2strdSEy1U8qcSzRIdkYpvOPtBy/9i/IfaaCI9flZ4=\ngo.etcd.io/etcd/server/v3 v3.6.8/go.mod h1:88dCtwUnSirkUoJbflQxxWXqtBSZa6lSG0Kuej+dois=\ngo.etcd.io/etcd/tests/v3 v3.6.8 h1:FUiUGXUhLkU1WZMebKWdwtM7KtzAGuUSQEUV8Sq2+QA=\ngo.etcd.io/etcd/tests/v3 v3.6.8/go.mod h1:U1ioDy7TXzz2UXhSQfbJ3++PsryNwiniHtdbXZPprX0=\ngo.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA=\ngo.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o=\ngo.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ=\ngo.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo=\ngo.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=\ngo.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=\ngo.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=\ngo.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4=\ngo.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=\ngo.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE=\ngo.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=\ngo.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=\ngo.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=\ngo.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=\ngo.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=\ngo.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=\ngo.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=\ngo.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=\ngo.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=\ngo.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=\ngo.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=\ngo.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=\ngo.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngo.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=\ngo.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=\ngo.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=\ngo.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=\ngo.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=\ngo.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=\ngo.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=\ngo.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=\ngolang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=\ngolang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=\ngolang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=\ngolang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=\ngolang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=\ngolang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=\ngolang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=\ngolang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=\ngolang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=\ngolang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=\ngolang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4=\ngolang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=\ngolang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE=\ngolang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6/go.mod h1:3rxYc4HtVcSG9gVaTs2GEBdehh+sYPOwKtyUWEOTb80=\ngonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=\ngonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=\ngoogle.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=\ngoogle.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=\ngoogle.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=\ngoogle.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\ngopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=\ngopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=\ngopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=\ngopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=\ngopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=\ngopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nk8s.io/api v0.32.10 h1:ocp4turNfa1V40TuBW/LuA17TeXG9g/GI2ebg0KxBNk=\nk8s.io/api v0.32.10/go.mod h1:AsMsc4b6TuampYqgMEGSv0HBFpRS4BlKTXAVCAa7oF4=\nk8s.io/apimachinery v0.32.10 h1:SAg2kUPLYRcBJQj66oniP1BnXSqw+l1GvJFsJlBmVvQ=\nk8s.io/apimachinery v0.32.10/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=\nk8s.io/client-go v0.32.10 h1:MFmIjsKtcnn7mStjrJG1ZW2WzLsKKn6ZtL9hHM/W0xU=\nk8s.io/client-go v0.32.10/go.mod h1:qJy/Ws3zSwnu/nD75D+/of1uxbwWHxrYT5P3FuobVLI=\nk8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=\nk8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=\nk8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=\nk8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4=\nk8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=\nk8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=\nsigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=\nsigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=\nsigs.k8s.io/knftables v0.0.18 h1:6Duvmu0s/HwGifKrtl6G3AyAPYlWiZqTgS8bkVMiyaE=\nsigs.k8s.io/knftables v0.0.18/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk=\nsigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=\nsigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=\nsigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=\nsigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=\n"
  },
  {
    "path": "images/Dockerfile",
    "content": "ARG XX_IMAGE=tonistiigi/xx:1.8.0\nARG GO_IMAGE=golang:alpine3.22\nARG RUN_IMAGE=alpine:3.22.2\n\nFROM --platform=$BUILDPLATFORM ${XX_IMAGE}  AS xx\n\nFROM --platform=$BUILDPLATFORM ${GO_IMAGE} AS base-builder\n# copy xx scripts to your build stage\nCOPY --from=xx / /\nRUN apk add bash file make git clang lld patch linux-headers\nARG TARGETPLATFORM\nRUN set -x && \\\n    xx-apk --no-cache add gcc musl-dev \n\nFROM --platform=$BUILDPLATFORM base-builder AS flannel-builder\nCOPY Makefile go.mod go.sum *.go /build/\nCOPY pkg /build/pkg\nWORKDIR /build\nRUN mkdir dist\nRUN go mod download\n\nARG TARGETPLATFORM\nARG TAG\nRUN xx-info env\nRUN export GOOS=$(xx-info os) &&\\\n    export GOARCH=$(xx-info arch) &&\\\n    export ARCH=$(xx-info arch) &&\\\n    make dist/flanneld\nRUN git clone https://github.com/kubernetes-sigs/iptables-wrappers.git /iptables-wrapper\nWORKDIR /iptables-wrapper\nRUN git checkout 5792812d9e5a5bb7f22d79d557bbfeece253343d\nRUN export GOOS=$(xx-info os) &&\\\n    export GOARCH=$(xx-info arch) &&\\\n    export ARCH=$(xx-info arch) &&\\\n    make build\n\nFROM ${RUN_IMAGE}\nRUN apk update && apk upgrade\nRUN apk add --no-cache iproute2 ca-certificates nftables iptables strongswan iptables-legacy && update-ca-certificates\nRUN apk add wireguard-tools --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community\nCOPY --from=flannel-builder /build/dist/flanneld /opt/bin/flanneld\nCOPY dist/mk-docker-opts.sh /opt/bin/\nCOPY --from=flannel-builder /iptables-wrapper/iptables-wrapper-installer.sh /\nCOPY --from=flannel-builder /iptables-wrapper/bin/iptables-wrapper /\n# check manually that iptables-legacy and iptables-nft are present since\n# iptables-wrapper-installer.sh sanity check doesn't work for multi-arch build\nRUN which iptables-legacy && which iptables-nft\nRUN /iptables-wrapper-installer.sh --no-sanity-check\n\n\nENTRYPOINT [\"/opt/bin/flanneld\"]\n\n"
  },
  {
    "path": "images/iperf3/Dockerfile",
    "content": "FROM BASEIMAGE\n\nRUN apk add --update \\\n    iperf3 \\\n  && rm -rf /var/cache/apk/*\n\nEXPOSE 5201\n\nENTRYPOINT [\"/usr/bin/iperf3\"]\n\nCMD [\"--server\",\"-p\",\"5201\"]\n"
  },
  {
    "path": "images/iperf3/Makefile",
    "content": "IPERF_IMG ?= iperf3:latest\n\nARCH ?= amd64\n\nTEMP_DIR := $(shell mktemp -d)\n\nifeq ($(ARCH),amd64)\n        BASEIMAGE=amd64/alpine:3.16\nendif\nifeq ($(ARCH),arm64)\n\tBASEIMAGE=arm64v8/alpine:3.16\nendif\nifeq ($(ARCH),ppc64le)\n\tBASEIMAGE=ppc64le/alpine:3.16\nendif\n\nall: container\n\ncontainer:\n\tcp ./* $(TEMP_DIR)\n\tcd $(TEMP_DIR) && sed -i 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile\n\tdocker build --pull -t $(IPERF_IMG) -f $(TEMP_DIR)/Dockerfile $(TEMP_DIR)\n"
  },
  {
    "path": "main.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"flag\"\n\t\"fmt\"\n\t\"math/big\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/coreos/pkg/flagutil\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/ipmatch\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tetcd \"github.com/flannel-io/flannel/pkg/subnet/etcd\"\n\t\"github.com/flannel-io/flannel/pkg/subnet/kube\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr/iptables\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr/nftables\"\n\t\"github.com/flannel-io/flannel/pkg/version\"\n\t\"github.com/joho/godotenv\"\n\tlog \"k8s.io/klog/v2\"\n\n\t// Backends need to be imported for their init() to get executed and them to register\n\t\"github.com/coreos/go-systemd/v22/daemon\"\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/alloc\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/extension\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/hostgw\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/ipip\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/ipsec\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/tencentvpc\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/udp\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/vxlan\"\n\t_ \"github.com/flannel-io/flannel/pkg/backend/wireguard\"\n)\n\ntype flagSlice []string\n\nfunc (t *flagSlice) String() string {\n\treturn fmt.Sprintf(\"%v\", *t)\n}\n\nfunc (t *flagSlice) Set(val string) error {\n\t*t = append(*t, val)\n\treturn nil\n}\n\ntype CmdLineOpts struct {\n\tetcdEndpoints             string\n\tetcdPrefix                string\n\tetcdKeyfile               string\n\tetcdCertfile              string\n\tetcdCAFile                string\n\tetcdUsername              string\n\tetcdPassword              string\n\tversion                   bool\n\tkubeSubnetMgr             bool\n\tkubeApiUrl                string\n\tkubeAnnotationPrefix      string\n\tkubeConfigFile            string\n\tiface                     flagSlice\n\tifaceRegex                flagSlice\n\tipMasq                    bool\n\tipMasqRandomFullyDisable  bool\n\tifaceCanReach             string\n\tsubnetFile                string\n\tpublicIP                  string\n\tpublicIPv6                string\n\tsubnetLeaseRenewMargin    int\n\thealthzIP                 string\n\thealthzPort               int\n\tiptablesResyncSeconds     int\n\tiptablesForwardRules      bool\n\tblackholeRoute            bool\n\tnetConfPath               string\n\tsetNodeNetworkUnavailable bool\n}\n\nvar (\n\topts           CmdLineOpts\n\terrInterrupted = errors.New(\"interrupted\")\n\terrCanceled    = errors.New(\"canceled\")\n\tflannelFlags   = flag.NewFlagSet(\"flannel\", flag.ExitOnError)\n)\n\nfunc init() {\n\tflannelFlags.StringVar(&opts.etcdEndpoints, \"etcd-endpoints\", \"http://127.0.0.1:4001,http://127.0.0.1:2379\", \"a comma-delimited list of etcd endpoints\")\n\tflannelFlags.StringVar(&opts.etcdPrefix, \"etcd-prefix\", \"/coreos.com/network\", \"etcd prefix\")\n\tflannelFlags.StringVar(&opts.etcdKeyfile, \"etcd-keyfile\", \"\", \"SSL key file used to secure etcd communication\")\n\tflannelFlags.StringVar(&opts.etcdCertfile, \"etcd-certfile\", \"\", \"SSL certification file used to secure etcd communication\")\n\tflannelFlags.StringVar(&opts.etcdCAFile, \"etcd-cafile\", \"\", \"SSL Certificate Authority file used to secure etcd communication\")\n\tflannelFlags.StringVar(&opts.etcdUsername, \"etcd-username\", \"\", \"username for BasicAuth to etcd\")\n\tflannelFlags.StringVar(&opts.etcdPassword, \"etcd-password\", \"\", \"password for BasicAuth to etcd\")\n\tflannelFlags.Var(&opts.iface, \"iface\", \"interface to use (IP or name) for inter-host communication. Can be specified multiple times to check each option in order. Returns the first match found.\")\n\tflannelFlags.Var(&opts.ifaceRegex, \"iface-regex\", \"regex expression to match the first interface to use (IP or name) for inter-host communication. Can be specified multiple times to check each regex in order. Returns the first match found. Regexes are checked after specific interfaces specified by the iface option have already been checked.\")\n\tflannelFlags.StringVar(&opts.ifaceCanReach, \"iface-can-reach\", \"\", \"detect interface to use (IP or name) for inter-host communication based on which will be used for provided IP. This is exactly the interface to use of command 'ip route get <ip-address>'\")\n\tflannelFlags.StringVar(&opts.subnetFile, \"subnet-file\", \"/run/flannel/subnet.env\", \"filename where env variables (subnet, MTU, ... ) will be written to\")\n\tflannelFlags.StringVar(&opts.publicIP, \"public-ip\", \"\", \"IP accessible by other nodes for inter-host communication\")\n\tflannelFlags.StringVar(&opts.publicIPv6, \"public-ipv6\", \"\", \"IPv6 accessible by other nodes for inter-host communication\")\n\tflannelFlags.IntVar(&opts.subnetLeaseRenewMargin, \"subnet-lease-renew-margin\", 60, \"subnet lease renewal margin, in minutes, ranging from 1 to 1439\")\n\tflannelFlags.BoolVar(&opts.ipMasq, \"ip-masq\", false, \"setup IP masquerade rule for traffic destined outside of overlay network\")\n\tflannelFlags.BoolVar(&opts.ipMasqRandomFullyDisable, \"ip-masq-fully-random-disable\", false, \"disable fully-random mode for MASQUERADE\")\n\tflannelFlags.BoolVar(&opts.kubeSubnetMgr, \"kube-subnet-mgr\", false, \"contact the Kubernetes API for subnet assignment instead of etcd.\")\n\tflannelFlags.StringVar(&opts.kubeApiUrl, \"kube-api-url\", \"\", \"Kubernetes API server URL. Does not need to be specified if flannel is running in a pod.\")\n\tflannelFlags.StringVar(&opts.kubeAnnotationPrefix, \"kube-annotation-prefix\", \"flannel.alpha.coreos.com\", `Kubernetes annotation prefix. Can contain single slash \"/\", otherwise it will be appended at the end.`)\n\tflannelFlags.StringVar(&opts.kubeConfigFile, \"kubeconfig-file\", \"\", \"kubeconfig file location. Does not need to be specified if flannel is running in a pod.\")\n\tflannelFlags.BoolVar(&opts.version, \"version\", false, \"print version and exit\")\n\tflannelFlags.StringVar(&opts.healthzIP, \"healthz-ip\", \"0.0.0.0\", \"the IP address for healthz server to listen\")\n\tflannelFlags.IntVar(&opts.healthzPort, \"healthz-port\", 0, \"the port for healthz server to listen(0 to disable)\")\n\tflannelFlags.IntVar(&opts.iptablesResyncSeconds, \"iptables-resync\", 5, \"resync period for iptables rules, in seconds\")\n\tflannelFlags.BoolVar(&opts.iptablesForwardRules, \"iptables-forward-rules\", true, \"add default accept rules to FORWARD chain in iptables\")\n\tflannelFlags.BoolVar(&opts.blackholeRoute, \"ip-blackhole-route\", false, \"add blackroute route ont the node for the local podCIDR\")\n\tflannelFlags.StringVar(&opts.netConfPath, \"net-config-path\", \"/etc/kube-flannel/net-conf.json\", \"path to the network configuration file\")\n\tflannelFlags.BoolVar(&opts.setNodeNetworkUnavailable, \"set-node-network-unavailable\", true, \"set NodeNetworkUnavailable after ready\")\n\n\tlog.InitFlags(nil)\n\n\t// klog will log to tmp files by default. override so all entries\n\t// can flow into journald (if running under systemd)\n\terr := flag.Set(\"logtostderr\", \"true\")\n\tif err != nil {\n\t\tlog.Error(\"Can't set the logtostderr flag\", err)\n\t\tos.Exit(1)\n\t}\n\n\t// Only copy the non file logging options from klog\n\tcopyFlag(\"v\")\n\tcopyFlag(\"vmodule\")\n\tcopyFlag(\"log_backtrace_at\")\n\n\t// Define the usage function\n\tflannelFlags.Usage = usage\n\n\t// now parse command line args\n\terr = flannelFlags.Parse(os.Args[1:])\n\tif err != nil {\n\t\tlog.Error(\"Can't parse flannel flags\", err)\n\t\tos.Exit(1)\n\t}\n}\n\nfunc copyFlag(name string) {\n\tflannelFlags.Var(flag.Lookup(name).Value, flag.Lookup(name).Name, flag.Lookup(name).Usage)\n}\n\nfunc usage() {\n\tfmt.Fprintf(os.Stderr, \"Usage: %s [OPTION]...\\n\", os.Args[0])\n\tflannelFlags.PrintDefaults()\n\tos.Exit(0)\n}\n\nfunc newSubnetManager(ctx context.Context) (subnet.Manager, error) {\n\tif opts.kubeSubnetMgr {\n\t\treturn kube.NewSubnetManager(ctx,\n\t\t\topts.kubeApiUrl,\n\t\t\topts.kubeConfigFile,\n\t\t\topts.kubeAnnotationPrefix,\n\t\t\topts.netConfPath,\n\t\t\topts.setNodeNetworkUnavailable)\n\t}\n\n\tcfg := &etcd.EtcdConfig{\n\t\tEndpoints: strings.Split(opts.etcdEndpoints, \",\"),\n\t\tKeyfile:   opts.etcdKeyfile,\n\t\tCertfile:  opts.etcdCertfile,\n\t\tCAFile:    opts.etcdCAFile,\n\t\tPrefix:    opts.etcdPrefix,\n\t\tUsername:  opts.etcdUsername,\n\t\tPassword:  opts.etcdPassword,\n\t}\n\n\t// Attempt to renew the lease for the subnet specified in the subnetFile\n\tprevSubnet := ReadCIDRFromSubnetFile(opts.subnetFile, \"FLANNEL_SUBNET\")\n\tprevIPv6Subnet := ReadIP6CIDRFromSubnetFile(opts.subnetFile, \"FLANNEL_IPV6_SUBNET\")\n\n\treturn etcd.NewLocalManager(ctx, cfg, prevSubnet, prevIPv6Subnet, opts.subnetLeaseRenewMargin)\n}\n\nfunc main() {\n\tif opts.version {\n\t\tfmt.Fprintln(os.Stderr, version.Version)\n\t\tos.Exit(0)\n\t}\n\n\terr := flagutil.SetFlagsFromEnv(flannelFlags, \"FLANNELD\")\n\tif err != nil {\n\t\tlog.Error(\"Failed to set flag FLANNELD from env\", err)\n\t}\n\n\t// Log the config set via CLI flags\n\tlog.Infof(\"CLI flags config: %+v\", opts)\n\n\t// Validate flags\n\tif opts.subnetLeaseRenewMargin >= 24*60 || opts.subnetLeaseRenewMargin <= 0 {\n\t\tlog.Error(\"Invalid subnet-lease-renew-margin option, out of acceptable range\")\n\t\tos.Exit(1)\n\t}\n\n\t// This is the main context that everything should run in.\n\t// All spawned goroutines should exit when cancel is called on this context.\n\t// Go routines spawned from main.go coordinate using a WaitGroup. This provides a mechanism to allow the shutdownHandler goroutine\n\t// to block until all the goroutines return . If those goroutines spawn other goroutines then they are responsible for\n\t// blocking and returning only when cancel() is called.\n\tctx, cancel := context.WithCancel(context.Background())\n\n\tsm, err := newSubnetManager(ctx)\n\tif err != nil {\n\t\tcontCacheNotReady := os.Getenv(\"CONT_WHEN_CACHE_NOT_READY\")\n\t\tif contCacheNotReady == \"true\" && errors.Is(err, context.DeadlineExceeded) {\n\t\t\tlog.Error(\"Timed out waiting for node controller sync. Continuing anyway.\")\n\t\t\t// Don't exit — continue with startup\n\t\t} else {\n\t\t\tlog.Error(\"Failed to create SubnetManager: \", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n\tlog.Infof(\"Created subnet manager: %s\", sm.Name())\n\n\t// Register for SIGINT and SIGTERM\n\tlog.Info(\"Installing signal handlers\")\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, os.Interrupt, syscall.SIGTERM)\n\n\twg := sync.WaitGroup{}\n\n\twg.Add(1)\n\tgo func() {\n\t\tshutdownHandler(ctx, sigs, cancel)\n\t\twg.Done()\n\t}()\n\n\tif opts.healthzPort > 0 {\n\t\tmustRunHealthz(ctx.Done(), &wg)\n\t}\n\n\t// Fetch the network config (i.e. what backend to use etc..).\n\tconfig, err := getConfig(ctx, sm)\n\tif err == errCanceled {\n\t\twg.Wait()\n\t\tos.Exit(0)\n\t}\n\n\t// Get ip family stack\n\tipStack, stackErr := ipmatch.GetIPFamily(config.EnableIPv4, config.EnableIPv6)\n\tif stackErr != nil {\n\t\tlog.Error(stackErr.Error())\n\t\tos.Exit(1)\n\t}\n\n\tif runtime.GOOS != \"windows\" && !config.EnableNFTables {\n\t\t// From Kubernetes 1.30 kubeadm doesn't check if the br_netfilter module is loaded and in case it's missing Flannel wrongly starts\n\t\tif config.EnableIPv4 {\n\t\t\tif _, err = os.Stat(\"/proc/sys/net/bridge/bridge-nf-call-iptables\"); os.IsNotExist(err) {\n\t\t\t\tlog.Error(\"Failed to check br_netfilter: \", err)\n\t\t\t\tos.Exit(1)\n\t\t\t}\n\t\t}\n\t\tif config.EnableIPv6 {\n\t\t\tif _, err = os.Stat(\"/proc/sys/net/bridge/bridge-nf-call-ip6tables\"); os.IsNotExist(err) {\n\t\t\t\tlog.Error(\"Failed to check br_netfilter: \", err)\n\t\t\t\tos.Exit(1)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Work out which interface to use\n\tvar extIface *backend.ExternalInterface\n\n\tannotatedPublicIP, annotatedPublicIPv6 := sm.GetStoredPublicIP(ctx)\n\tif annotatedPublicIP != \"\" {\n\t\topts.publicIP = annotatedPublicIP\n\t}\n\tif annotatedPublicIPv6 != \"\" {\n\t\topts.publicIPv6 = annotatedPublicIPv6\n\t}\n\n\toptsPublicIP := ipmatch.PublicIPOpts{\n\t\tPublicIP:   opts.publicIP,\n\t\tPublicIPv6: opts.publicIPv6,\n\t}\n\t// Check the default interface only if no interfaces are specified\n\n\tif len(opts.iface) == 0 && len(opts.ifaceRegex) == 0 && len(opts.ifaceCanReach) == 0 {\n\t\tif len(opts.publicIP) > 0 {\n\t\t\textIface, err = ipmatch.LookupExtIface(opts.publicIP, \"\", \"\", ipStack, optsPublicIP)\n\t\t} else {\n\t\t\textIface, err = ipmatch.LookupExtIface(opts.publicIPv6, \"\", \"\", ipStack, optsPublicIP)\n\t\t}\n\t\tif err != nil {\n\t\t\tlog.Error(\"Failed to find any valid interface to use: \", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t} else {\n\t\t// Check explicitly specified interfaces\n\t\tfor _, iface := range opts.iface {\n\t\t\textIface, err = ipmatch.LookupExtIface(iface, \"\", \"\", ipStack, optsPublicIP)\n\t\t\tif err != nil {\n\t\t\t\tlog.Infof(\"Could not find valid interface matching %s: %s\", iface, err)\n\t\t\t}\n\n\t\t\tif extIface != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// Check interfaces that match any specified regexes\n\t\tif extIface == nil {\n\t\t\tfor _, ifaceRegex := range opts.ifaceRegex {\n\t\t\t\textIface, err = ipmatch.LookupExtIface(\"\", ifaceRegex, \"\", ipStack, optsPublicIP)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Infof(\"Could not find valid interface matching %s: %s\", ifaceRegex, err)\n\t\t\t\t}\n\n\t\t\t\tif extIface != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif extIface == nil && len(opts.ifaceCanReach) > 0 {\n\t\t\textIface, err = ipmatch.LookupExtIface(\"\", \"\", opts.ifaceCanReach, ipStack, optsPublicIP)\n\t\t\tif err != nil {\n\t\t\t\tlog.Infof(\"Could not find valid interface matching ifaceCanReach: %s: %s\", opts.ifaceCanReach, err)\n\t\t\t}\n\t\t}\n\n\t\tif extIface == nil {\n\t\t\t// Exit if any of the specified interfaces do not match\n\t\t\tlog.Error(\"Failed to find interface to use that matches the interfaces and/or regexes provided\")\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n\n\t// Create a backend manager then use it to create the backend and register the network with it.\n\tbm := backend.NewManager(ctx, sm, extIface)\n\tbe, err := bm.GetBackend(config.BackendType)\n\tif err != nil {\n\t\tlog.Errorf(\"Error fetching backend: %s\", err)\n\t\tcancel()\n\t\twg.Wait()\n\t\tos.Exit(1)\n\t}\n\n\tbn, err := be.RegisterNetwork(ctx, &wg, config)\n\tif err != nil {\n\t\tlog.Errorf(\"Error registering network: %s\", err)\n\t\tcancel()\n\t\twg.Wait()\n\t\tos.Exit(1)\n\t}\n\n\t// Instanciate a TrafficManager to clean-up the rules of the backend we don't use\n\t// This is to ensure a clean state in case flannel is restarted with a different choice\n\tlog.Info(\"Cleaning-up unused traffic manager rules\")\n\tcleanupMngr := newTrafficManager(!config.EnableNFTables)\n\terr = cleanupMngr.CleanUp(ctx)\n\tif err != nil {\n\t\tlog.Error(err)\n\t\tcancel()\n\t\twg.Wait()\n\t\tos.Exit(1)\n\t}\n\t//Create TrafficManager and instantiate it based on whether we use iptables or nftables\n\ttrafficMngr := newTrafficManager(config.EnableNFTables)\n\terr = trafficMngr.Init(ctx)\n\tif err != nil {\n\t\tlog.Error(err)\n\t\tcancel()\n\t\twg.Wait()\n\t\tos.Exit(1)\n\t}\n\n\t// Set up ipMasq if needed\n\tif opts.ipMasq {\n\t\tprevNetwork := ReadCIDRFromSubnetFile(opts.subnetFile, \"FLANNEL_NETWORK\")\n\t\tprevSubnet := ReadCIDRFromSubnetFile(opts.subnetFile, \"FLANNEL_SUBNET\")\n\n\t\tprevIPv6Network := ReadIP6CIDRFromSubnetFile(opts.subnetFile, \"FLANNEL_IPV6_NETWORK\")\n\t\tprevIPv6Subnet := ReadIP6CIDRFromSubnetFile(opts.subnetFile, \"FLANNEL_IPV6_SUBNET\")\n\n\t\terr = trafficMngr.SetupAndEnsureMasqRules(ctx,\n\t\t\tconfig.Network, prevSubnet,\n\t\t\tprevNetwork,\n\t\t\tconfig.IPv6Network, prevIPv6Subnet,\n\t\t\tprevIPv6Network,\n\t\t\tbn.Lease(),\n\t\t\topts.iptablesResyncSeconds,\n\t\t\topts.ipMasqRandomFullyDisable)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to setup masq rules, %v\", err)\n\t\t\tcancel()\n\t\t\twg.Wait()\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n\n\t// Always enables forwarding rules. This is needed for Docker versions >1.13 (https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#container-communication-between-hosts)\n\t// In Docker 1.12 and earlier, the default FORWARD chain policy was ACCEPT.\n\t// In Docker 1.13 and later, Docker sets the default policy of the FORWARD chain to DROP.\n\tif opts.iptablesForwardRules {\n\t\ttrafficMngr.SetupAndEnsureForwardRules(ctx,\n\t\t\tconfig.Network,\n\t\t\tconfig.IPv6Network,\n\t\t\topts.iptablesResyncSeconds)\n\t}\n\n\t//Add blackhole route for the local CIDR in case the bridge plugin is not enable (e.g. Canal)\n\tif opts.blackholeRoute {\n\t\tif config.EnableIPv4 {\n\t\t\tgo func() {\n\t\t\t\tfor {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\tcase <-time.After(time.Duration(opts.iptablesResyncSeconds) * time.Second):\n\t\t\t\t\t\tif err := ip.AddBlackholeV4Route(bn.Lease().Subnet.ToIPNet()); err != nil {\n\t\t\t\t\t\t\tlog.Errorf(\"Failed to setup blackhole route, %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t\tif config.EnableIPv6 {\n\t\t\tgo func() {\n\t\t\t\tfor {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\t\treturn\n\t\t\t\t\tcase <-time.After(time.Duration(opts.iptablesResyncSeconds) * time.Second):\n\t\t\t\t\t\tif err := ip.AddBlackholeV6Route(bn.Lease().IPv6Subnet.ToIPNet()); err != nil {\n\t\t\t\t\t\t\tlog.Errorf(\"Failed to setup blackhole route, %v\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}\n\n\tif err := sm.HandleSubnetFile(opts.subnetFile, config, opts.ipMasq, bn.Lease().Subnet, bn.Lease().IPv6Subnet, bn.MTU()); err != nil {\n\t\t// Continue, even though it failed.\n\t\tlog.Warningf(\"Failed to write subnet file: %s\", err)\n\t} else {\n\t\tlog.Infof(\"Wrote subnet file to %s\", opts.subnetFile)\n\t}\n\n\t// Start \"Running\" the backend network. This will block until the context is done so run in another goroutine.\n\tlog.Info(\"Running backend.\")\n\twg.Add(1)\n\tgo func() {\n\t\tbn.Run(ctx)\n\t\twg.Done()\n\t}()\n\n\t_, err = daemon.SdNotify(false, \"READY=1\")\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to notify systemd the message READY=1 %v\", err)\n\t}\n\n\terr = sm.CompleteLease(ctx, bn.Lease(), &wg)\n\tif err != nil {\n\t\tlog.Errorf(\"CompleteLease execute error err: %v\", err)\n\t\tif strings.EqualFold(err.Error(), errInterrupted.Error()) {\n\t\t\t// The lease was \"revoked\" - shut everything down\n\t\t\tcancel()\n\t\t}\n\t}\n\n\tlog.Info(\"Waiting for all goroutines to exit\")\n\t// Block waiting for all the goroutines to finish.\n\twg.Wait()\n\tlog.Info(\"Exiting cleanly...\")\n\tos.Exit(0)\n}\n\nfunc shutdownHandler(ctx context.Context, sigs chan os.Signal, cancel context.CancelFunc) {\n\t// Wait for the context do be Done or for the signal to come in to shutdown.\n\tselect {\n\tcase <-ctx.Done():\n\t\tlog.Info(\"Stopping shutdownHandler...\")\n\tcase <-sigs:\n\t\t// Call cancel on the context to close everything down.\n\t\tcancel()\n\t\tlog.Info(\"shutdownHandler sent cancel signal...\")\n\t}\n\n\t// Unregister to get default OS nuke behaviour in case we don't exit cleanly\n\tsignal.Stop(sigs)\n}\n\nfunc getConfig(ctx context.Context, sm subnet.Manager) (*subnet.Config, error) {\n\t// Retry every second until it succeeds\n\tfor {\n\t\tconfig, err := sm.GetNetworkConfig(ctx)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Couldn't fetch network config: %s\", err)\n\t\t} else if config == nil {\n\t\t\tlog.Warningf(\"Couldn't find network config: %s\", err)\n\t\t} else {\n\t\t\tlog.Infof(\"Found network config - Backend type: %s\", config.BackendType)\n\t\t\treturn config, nil\n\t\t}\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn nil, errCanceled\n\t\tcase <-time.After(1 * time.Second):\n\t\t\tfmt.Println(\"timed out\")\n\t\t}\n\t}\n}\n\nfunc mustRunHealthz(stopChan <-chan struct{}, wg *sync.WaitGroup) {\n\taddress := net.JoinHostPort(opts.healthzIP, strconv.Itoa(opts.healthzPort))\n\tlog.Infof(\"Start healthz server on %s\", address)\n\n\thttp.HandleFunc(\"/healthz\", func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, err := w.Write([]byte(\"flanneld is running\"))\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Handling /healthz error. %v\", err)\n\t\t\tpanic(err)\n\t\t}\n\t})\n\n\tserver := &http.Server{Addr: address}\n\n\twg.Add(2)\n\tgo func() {\n\t\t// when Shutdown is called, ListenAndServe immediately return ErrServerClosed.\n\t\tif err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {\n\t\t\tlog.Errorf(\"Start healthz server error. %v\", err)\n\t\t\tpanic(err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\tgo func() {\n\t\t// wait to stop\n\t\t<-stopChan\n\n\t\t// create new context with timeout for http server to shutdown gracefully\n\t\tctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)\n\t\tdefer cancel()\n\t\tif err := server.Shutdown(ctx); err != nil {\n\t\t\tlog.Errorf(\"Shutdown healthz server error. %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n}\n\nfunc ReadCIDRFromSubnetFile(path string, CIDRKey string) ip.IP4Net {\n\tprevCIDRs := ReadCIDRsFromSubnetFile(path, CIDRKey)\n\tif len(prevCIDRs) == 0 {\n\t\tlog.Warningf(\"no subnet found for key: %s in file: %s\", CIDRKey, path)\n\t\treturn ip.IP4Net{IP: 0, PrefixLen: 0}\n\t} else if len(prevCIDRs) > 1 {\n\t\tlog.Errorf(\"error reading subnet: more than 1 entry found for key: %s in file %s: \", CIDRKey, path)\n\t\treturn ip.IP4Net{IP: 0, PrefixLen: 0}\n\t} else {\n\t\treturn prevCIDRs[0]\n\t}\n}\n\nfunc ReadCIDRsFromSubnetFile(path string, CIDRKey string) []ip.IP4Net {\n\tprevCIDRs := make([]ip.IP4Net, 0)\n\tif _, err := os.Stat(path); !os.IsNotExist(err) {\n\t\tprevSubnetVals, err := godotenv.Read(path)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Couldn't fetch previous %s from subnet file at %s: %s\", CIDRKey, path, err)\n\t\t} else if prevCIDRString, ok := prevSubnetVals[CIDRKey]; ok {\n\t\t\tcidrs := strings.Split(prevCIDRString, \",\")\n\t\t\tprevCIDRs = make([]ip.IP4Net, 0)\n\t\t\tfor i := range cidrs {\n\t\t\t\t_, cidr, err := net.ParseCIDR(cidrs[i])\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"Couldn't parse previous %s from subnet file at %s: %s\", CIDRKey, path, err)\n\t\t\t\t}\n\t\t\t\tprevCIDRs = append(prevCIDRs, ip.FromIPNet(cidr))\n\t\t\t}\n\n\t\t}\n\t}\n\treturn prevCIDRs\n}\n\nfunc ReadIP6CIDRFromSubnetFile(path string, CIDRKey string) ip.IP6Net {\n\tprevCIDRs := ReadIP6CIDRsFromSubnetFile(path, CIDRKey)\n\tif len(prevCIDRs) == 0 {\n\t\tlog.Warningf(\"no subnet found for key: %s in file: %s\", CIDRKey, path)\n\t\treturn ip.IP6Net{IP: (*ip.IP6)(big.NewInt(0)), PrefixLen: 0}\n\t} else if len(prevCIDRs) > 1 {\n\t\tlog.Errorf(\"error reading subnet: more than 1 entry found for key: %s in file %s: \", CIDRKey, path)\n\t\treturn ip.IP6Net{IP: (*ip.IP6)(big.NewInt(0)), PrefixLen: 0}\n\t} else {\n\t\treturn prevCIDRs[0]\n\t}\n}\n\nfunc ReadIP6CIDRsFromSubnetFile(path string, CIDRKey string) []ip.IP6Net {\n\tprevCIDRs := make([]ip.IP6Net, 0)\n\tif _, err := os.Stat(path); !os.IsNotExist(err) {\n\t\tprevSubnetVals, err := godotenv.Read(path)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Couldn't fetch previous %s from subnet file at %s: %s\", CIDRKey, path, err)\n\t\t} else if prevCIDRString, ok := prevSubnetVals[CIDRKey]; ok {\n\t\t\tcidrs := strings.Split(prevCIDRString, \",\")\n\t\t\tprevCIDRs = make([]ip.IP6Net, 0)\n\t\t\tfor i := range cidrs {\n\t\t\t\t_, cidr, err := net.ParseCIDR(cidrs[i])\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"Couldn't parse previous %s from subnet file at %s: %s\", CIDRKey, path, err)\n\t\t\t\t}\n\t\t\t\tprevCIDRs = append(prevCIDRs, ip.FromIP6Net(cidr))\n\t\t\t}\n\n\t\t}\n\t}\n\treturn prevCIDRs\n}\n\nfunc newTrafficManager(useNftables bool) trafficmngr.TrafficManager {\n\tif useNftables {\n\t\treturn &nftables.NFTablesManager{}\n\t} else {\n\t\treturn &iptables.IPTablesManager{}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/alloc/alloc.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage alloc\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\nfunc init() {\n\tbackend.Register(\"alloc\", New)\n}\n\ntype AllocBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbe := AllocBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\treturn &be, nil\n}\n\nfunc (be *AllocBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: ip.FromIP(be.extIface.ExtAddr),\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, &attrs)\n\tswitch err {\n\tcase nil:\n\t\treturn &backend.SimpleNetwork{\n\t\t\tSubnetLease: l,\n\t\t\tExtIface:    be.extIface,\n\t\t}, nil\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/common.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage backend\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\ntype ExternalInterface struct {\n\tIface       *net.Interface\n\tIfaceName   string\n\tIfaceAddr   net.IP\n\tIfaceV6Addr net.IP\n\tExtAddr     net.IP\n\tExtV6Addr   net.IP\n}\n\n// Besides the entry points in the Backend interface, the backend's New()\n// function receives static network interface information (like internal and\n// external IP addresses, MTU, etc) which it should cache for later use if\n// needed.\ntype Backend interface {\n\t// Called when the backend should create or begin managing a new network\n\tRegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (Network, error)\n}\n\ntype Network interface {\n\tLease() *lease.Lease\n\tMTU() int\n\tRun(ctx context.Context)\n}\n\ntype BackendCtor func(sm subnet.Manager, ei *ExternalInterface) (Backend, error)\n"
  },
  {
    "path": "pkg/backend/extension/extension.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage extension\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc init() {\n\tbackend.Register(\"extension\", New)\n}\n\ntype ExtensionBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n\tnetworks map[string]*network\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbe := &ExtensionBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t\tnetworks: make(map[string]*network),\n\t}\n\n\treturn be, nil\n}\n\nfunc (*ExtensionBackend) Run(ctx context.Context) {\n\t<-ctx.Done()\n}\n\nfunc (be *ExtensionBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\tn := &network{\n\t\textIface: be.extIface,\n\t\tsm:       be.sm,\n\t}\n\n\t// Parse out configuration\n\tif len(config.Backend) > 0 {\n\t\tcfg := struct {\n\t\t\tPreStartupCommand   string\n\t\t\tPostStartupCommand  string\n\t\t\tSubnetAddCommand    string\n\t\t\tSubnetRemoveCommand string\n\t\t}{}\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding backend config: %v\", err)\n\t\t}\n\t\tn.preStartupCommand = cfg.PreStartupCommand\n\t\tn.postStartupCommand = cfg.PostStartupCommand\n\t\tn.subnetAddCommand = cfg.SubnetAddCommand\n\t\tn.subnetRemoveCommand = cfg.SubnetRemoveCommand\n\t}\n\n\tdata := []byte{}\n\tif len(n.preStartupCommand) > 0 {\n\t\tcmd_output, err := runCmd([]string{}, \"\", \"sh\", \"-c\", n.preStartupCommand)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to run command: %s Err: %v Output: %s\", n.preStartupCommand, err, cmd_output)\n\t\t} else {\n\t\t\tlog.Infof(\"Ran command: %s\\n Output: %s\", n.preStartupCommand, cmd_output)\n\t\t}\n\n\t\tdata, err = json.Marshal(cmd_output)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tlog.Infof(\"No pre startup command configured - skipping\")\n\t}\n\n\tattrs := lease.LeaseAttrs{\n\t\tBackendType: \"extension\",\n\t\tBackendData: data,\n\t}\n\n\tif be.extIface.IfaceAddr != nil {\n\t\tattrs.PublicIP = ip.FromIP(be.extIface.IfaceAddr)\n\t}\n\n\tif be.extIface.IfaceV6Addr != nil {\n\t\tattrs.PublicIPv6 = ip.FromIP6(be.extIface.IfaceV6Addr)\n\t}\n\n\tlease, err := be.sm.AcquireLease(ctx, &attrs)\n\tswitch err {\n\tcase nil:\n\t\tn.lease = lease\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\tif len(n.postStartupCommand) > 0 {\n\t\tcmd_output, err := runCmd([]string{\n\t\t\tfmt.Sprintf(\"NETWORK=%s\", config.Network),\n\t\t\tfmt.Sprintf(\"SUBNET=%s\", lease.Subnet),\n\t\t\tfmt.Sprintf(\"IPV6SUBNET=%s\", lease.IPv6Subnet),\n\t\t\tfmt.Sprintf(\"PUBLIC_IP=%s\", attrs.PublicIP),\n\t\t\tfmt.Sprintf(\"PUBLIC_IPV6=%s\", attrs.PublicIPv6)},\n\t\t\t\"\", \"sh\", \"-c\", n.postStartupCommand)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to run command: %s Err: %v Output: %s\", n.postStartupCommand, err, cmd_output)\n\t\t} else {\n\t\t\tlog.Infof(\"Ran command: %s\\n Output: %s\", n.postStartupCommand, cmd_output)\n\t\t}\n\t} else {\n\t\tlog.Infof(\"No post startup command configured - skipping\")\n\t}\n\n\treturn n, nil\n}\n\n// Run a cmd, returning a combined stdout and stderr.\nfunc runCmd(env []string, stdin string, name string, arg ...string) (string, error) {\n\tcmd := exec.Command(name, arg...)\n\tcmd.Env = append(os.Environ(), env...)\n\n\tstdinpipe, err := cmd.StdinPipe()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t_, err = io.WriteString(stdinpipe, stdin)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t_, err = io.WriteString(stdinpipe, \"\\n\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\terr = stdinpipe.Close()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\toutput, err := cmd.CombinedOutput()\n\n\treturn strings.TrimSpace(string(output)), err\n}\n"
  },
  {
    "path": "pkg/backend/extension/extension_network.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage extension\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype network struct {\n\textIface            *backend.ExternalInterface\n\tlease               *lease.Lease\n\tsm                  subnet.Manager\n\tpreStartupCommand   string\n\tpostStartupCommand  string\n\tsubnetAddCommand    string\n\tsubnetRemoveCommand string\n}\n\nfunc (n *network) Lease() *lease.Lease {\n\treturn n.lease\n}\n\nfunc (n *network) MTU() int {\n\treturn n.extIface.Iface.MTU\n}\n\nfunc (n *network) Run(ctx context.Context) {\n\twg := sync.WaitGroup{}\n\n\tlog.Info(\"Watching for new subnet leases\")\n\tevts := make(chan []lease.Event)\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, n.sm, n.lease, evts)\n\t\twg.Done()\n\t}()\n\n\tdefer wg.Wait()\n\n\tfor {\n\t\tevtBatch, ok := <-evts\n\t\tif !ok {\n\t\t\tlog.Infof(\"evts chan closed\")\n\t\t\treturn\n\t\t}\n\t\tn.handleSubnetEvents(evtBatch)\n\t}\n}\n\nfunc (n *network) handleSubnetEvents(batch []lease.Event) {\n\tfor _, evt := range batch {\n\t\tswitch evt.Type {\n\t\tcase lease.EventAdded:\n\t\t\tlog.Infof(\"Subnet added: %v via %v\", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)\n\n\t\t\tif evt.Lease.Attrs.BackendType != \"extension\" {\n\t\t\t\tlog.Warningf(\"Ignoring non-extension subnet: type=%v\", evt.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif len(n.subnetAddCommand) > 0 {\n\t\t\t\tbackendData := \"\"\n\n\t\t\t\tif len(evt.Lease.Attrs.BackendData) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(evt.Lease.Attrs.BackendData, &backendData); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to unmarshal BackendData: %v\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tcmd_output, err := runCmd([]string{\n\t\t\t\t\tfmt.Sprintf(\"SUBNET=%s\", evt.Lease.Subnet),\n\t\t\t\t\tfmt.Sprintf(\"PUBLIC_IP=%s\", evt.Lease.Attrs.PublicIP)},\n\t\t\t\t\tbackendData,\n\t\t\t\t\t\"sh\", \"-c\", n.subnetAddCommand)\n\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to run command: %s Err: %v Output: %s\", n.subnetAddCommand, err, cmd_output)\n\t\t\t\t} else {\n\t\t\t\t\tlog.Infof(\"Ran command: %s\\n Output: %s\", n.subnetAddCommand, cmd_output)\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase lease.EventRemoved:\n\t\t\tlog.Info(\"Subnet removed: \", evt.Lease.Subnet)\n\n\t\t\tif evt.Lease.Attrs.BackendType != \"extension\" {\n\t\t\t\tlog.Warningf(\"Ignoring non-extension subnet: type=%v\", evt.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif len(n.subnetRemoveCommand) > 0 {\n\t\t\t\tbackendData := \"\"\n\n\t\t\t\tif len(evt.Lease.Attrs.BackendData) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(evt.Lease.Attrs.BackendData, &backendData); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to unmarshal BackendData: %v\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tcmd_output, err := runCmd([]string{\n\t\t\t\t\tfmt.Sprintf(\"SUBNET=%s\", evt.Lease.Subnet),\n\t\t\t\t\tfmt.Sprintf(\"PUBLIC_IP=%s\", evt.Lease.Attrs.PublicIP)},\n\t\t\t\t\tbackendData,\n\t\t\t\t\t\"sh\", \"-c\", n.subnetRemoveCommand)\n\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to run command: %s Err: %v Output: %s\", n.subnetRemoveCommand, err, cmd_output)\n\t\t\t\t} else {\n\t\t\t\t\tlog.Infof(\"Ran command: %s\\n Output: %s\", n.subnetRemoveCommand, cmd_output)\n\t\t\t\t}\n\t\t\t}\n\n\t\tdefault:\n\t\t\tlog.Error(\"Internal error: unknown event type: \", int(evt.Type))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/hostgw/hostgw.go",
    "content": "//go:build !windows && !windows\n// +build !windows,!windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hostgw\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/vishvananda/netlink\"\n)\n\nfunc init() {\n\tbackend.Register(\"host-gw\", New)\n}\n\ntype HostgwBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tif !extIface.ExtAddr.Equal(extIface.IfaceAddr) {\n\t\treturn nil, fmt.Errorf(\"your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend\")\n\t}\n\n\tbe := &HostgwBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\treturn be, nil\n}\n\nfunc (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\tn := &backend.RouteNetwork{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tExtIface: be.extIface,\n\t\t},\n\t\tSM:          be.sm,\n\t\tBackendType: \"host-gw\",\n\t\tMtu:         be.extIface.Iface.MTU,\n\t\tLinkIndex:   be.extIface.Iface.Index,\n\t}\n\n\tattrs := lease.LeaseAttrs{\n\t\tBackendType: \"host-gw\",\n\t}\n\n\tif config.EnableIPv4 {\n\t\tattrs.PublicIP = ip.FromIP(be.extIface.ExtAddr)\n\t\tn.GetRoute = func(lease *lease.Lease) *netlink.Route {\n\t\t\treturn &netlink.Route{\n\t\t\t\tDst:       lease.Subnet.ToIPNet(),\n\t\t\t\tGw:        lease.Attrs.PublicIP.ToIP(),\n\t\t\t\tLinkIndex: n.LinkIndex,\n\t\t\t}\n\t\t}\n\t}\n\n\tif config.EnableIPv6 {\n\t\tattrs.PublicIPv6 = ip.FromIP6(be.extIface.ExtV6Addr)\n\t\tn.GetV6Route = func(lease *lease.Lease) *netlink.Route {\n\t\t\treturn &netlink.Route{\n\t\t\t\tDst:       lease.IPv6Subnet.ToIPNet(),\n\t\t\t\tGw:        lease.Attrs.PublicIPv6.ToIP(),\n\t\t\t\tLinkIndex: n.LinkIndex,\n\t\t\t}\n\t\t}\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, &attrs)\n\tswitch err {\n\tcase nil:\n\t\tn.SubnetLease = l\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\treturn n, nil\n}\n"
  },
  {
    "path": "pkg/backend/hostgw/hostgw_windows.go",
    "content": "// Copyright 2018 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage hostgw\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/Microsoft/hcsshim\"\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/routing\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/pkg/errors\"\n\t\"k8s.io/apimachinery/pkg/util/json\"\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc init() {\n\tbackend.Register(\"host-gw\", New)\n}\n\ntype HostgwBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tif !extIface.ExtAddr.Equal(extIface.IfaceAddr) {\n\t\treturn nil, fmt.Errorf(\"your PublicIP differs from interface IP, meaning that probably you're on a NAT, which is not supported by host-gw backend\")\n\t}\n\n\tbe := &HostgwBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\n\treturn be, nil\n}\n\nfunc (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\t// 1. Parse configuration\n\tcfg := struct {\n\t\tName          string\n\t\tDNSServerList string\n\t}{}\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"error decoding windows host-gw backend config\")\n\t\t}\n\t}\n\tif len(cfg.Name) == 0 {\n\t\tcfg.Name = \"cbr0\"\n\t}\n\tlog.Infof(\"HOST-GW config: %+v\", cfg)\n\n\tn := &backend.RouteNetwork{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tExtIface: be.extIface,\n\t\t},\n\t\tSM:          be.sm,\n\t\tBackendType: \"host-gw\",\n\t\tMtu:         be.extIface.Iface.MTU,\n\t\tLinkIndex:   be.extIface.Iface.Index,\n\t}\n\tn.GetRoute = func(myLease *lease.Lease) *routing.Route {\n\t\treturn &routing.Route{\n\t\t\tDestinationSubnet: myLease.Subnet.ToIPNet(),\n\t\t\tGatewayAddress:    myLease.Attrs.PublicIP.ToIP(),\n\t\t\tInterfaceIndex:    n.LinkIndex,\n\t\t}\n\t}\n\n\t// 2. Acquire the lease form subnet manager\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP:    ip.FromIP(be.extIface.ExtAddr),\n\t\tBackendType: \"host-gw\",\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, &attrs)\n\tswitch err {\n\tcase nil:\n\t\tn.SubnetLease = l\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, errors.Wrap(err, \"failed to acquire lease\")\n\t}\n\n\t// 3. Check if the network exists and has the expected settings\n\tcreateNewNetwork := true\n\texpectedSubnet := n.SubnetLease.Subnet\n\texpectedAddressPrefix := expectedSubnet.String()\n\texpectedGatewayAddress := (expectedSubnet.IP + 1).String()\n\texpectedPodGatewayAddress := expectedSubnet.IP + 2\n\tnetworkName := cfg.Name\n\tvar waitErr, lastErr error\n\n\texistingNetwork, err := hcsshim.GetHNSNetworkByName(networkName)\n\tif err == nil {\n\t\tfor _, subnet := range existingNetwork.Subnets {\n\t\t\tif subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGatewayAddress {\n\t\t\t\tcreateNewNetwork = false\n\t\t\t\tlog.Infof(\"Found existing HNSNetwork %s\", networkName)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\t// 4. Create a new HNSNetwork\n\texpectedNetwork := existingNetwork\n\tif createNewNetwork {\n\t\tif existingNetwork != nil {\n\t\t\tif _, err := existingNetwork.Delete(); err != nil {\n\t\t\t\treturn nil, errors.Wrapf(err, \"failed to delete existing HNSNetwork %s\", networkName)\n\t\t\t}\n\t\t\tlog.Infof(\"Deleted stale HNSNetwork %s\", networkName)\n\t\t}\n\n\t\texpectedNetwork = &hcsshim.HNSNetwork{\n\t\t\tName:          networkName,\n\t\t\tType:          \"L2Bridge\",\n\t\t\tDNSServerList: cfg.DNSServerList,\n\t\t\tSubnets: []hcsshim.Subnet{\n\t\t\t\t{\n\t\t\t\t\tAddressPrefix:  expectedAddressPrefix,\n\t\t\t\t\tGatewayAddress: expectedGatewayAddress,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tjsonRequest, err := json.Marshal(expectedNetwork)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to marshal %+v\", expectedNetwork)\n\t\t}\n\n\t\tlog.Infof(\"Attempting to create HNSNetwork %s\", string(jsonRequest))\n\t\tnewNetwork, err := hcsshim.HNSNetworkRequest(\"POST\", \"\", string(jsonRequest))\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to create HNSNetwork %s\", networkName)\n\t\t}\n\n\t\t// Wait for the network to populate Management IP\n\t\tlog.Infof(\"Waiting to get ManagementIP from HNSNetwork %s\", networkName)\n\t\tvar newNetworkID = newNetwork.Id\n\t\twaitErr := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 5*time.Second, true, func(context.Context) (done bool, err error) {\n\t\t\tnewNetwork, lastErr = hcsshim.HNSNetworkRequest(\"GET\", newNetworkID, \"\")\n\t\t\treturn newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil\n\t\t})\n\t\tif waitErr != nil {\n\t\t\t// Do not swallow the root cause\n\t\t\tif lastErr != nil {\n\t\t\t\twaitErr = lastErr\n\t\t\t}\n\t\t\treturn nil, errors.Wrapf(waitErr, \"timeout, failed to get management IP from HNSNetwork %s\", networkName)\n\t\t}\n\n\t\t// Wait for the interface with the management IP\n\t\tlog.Infof(\"Waiting to get net interface for HNSNetwork %s (%s)\", networkName, newNetwork.ManagementIP)\n\t\tmanagementIP, err := ip.ParseIP4(newNetwork.ManagementIP)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"Failed to parse management ip (%s)\", newNetwork.ManagementIP)\n\t\t}\n\t\twaitErr = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 5*time.Second, true, func(context.Context) (done bool, err error) {\n\t\t\t_, lastErr = ip.GetInterfaceByIP(managementIP.ToIP())\n\t\t\treturn lastErr == nil, nil\n\t\t})\n\t\tif waitErr != nil {\n\t\t\treturn nil, errors.Wrapf(lastErr, \"timeout, failed to get net interface for HNSNetwork %s (%s)\", networkName, newNetwork.ManagementIP)\n\t\t}\n\n\t\tlog.Infof(\"Created HNSNetwork %s\", networkName)\n\t\texpectedNetwork = newNetwork\n\t}\n\n\t// 5. Ensure a 1.2 endpoint on this network in the host compartment\n\tcreateNewBridgeEndpoint := true\n\tbridgeEndpointName := networkName + \"_ep\"\n\texistingBridgeEndpoint, err := hcsshim.GetHNSEndpointByName(bridgeEndpointName)\n\tif err == nil && existingBridgeEndpoint.IPAddress.String() == expectedPodGatewayAddress.String() {\n\t\tlog.Infof(\"Found existing bridge HNSEndpoint %s\", bridgeEndpointName)\n\t\tcreateNewBridgeEndpoint = false\n\t}\n\n\t// 6. Create a bridge HNSEndpoint\n\texpectedBridgeEndpoint := existingBridgeEndpoint\n\tif createNewBridgeEndpoint {\n\t\tif existingBridgeEndpoint != nil {\n\t\t\tif _, err = existingBridgeEndpoint.Delete(); err != nil {\n\t\t\t\treturn nil, errors.Wrapf(err, \"failed to delete existing bridge HNSEndpoint %s\", bridgeEndpointName)\n\t\t\t}\n\t\t\tlog.Infof(\"Deleted stale bridge HNSEndpoint %s\", bridgeEndpointName)\n\t\t}\n\n\t\texpectedBridgeEndpoint = &hcsshim.HNSEndpoint{\n\t\t\tName:           bridgeEndpointName,\n\t\t\tIPAddress:      expectedPodGatewayAddress.ToIP(),\n\t\t\tVirtualNetwork: expectedNetwork.Id,\n\t\t}\n\n\t\tlog.Infof(\"Attempting to create bridge HNSEndpoint %+v\", expectedBridgeEndpoint)\n\t\tif expectedBridgeEndpoint, err = expectedBridgeEndpoint.Create(); err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to create bridge HNSEndpoint %s\", bridgeEndpointName)\n\t\t}\n\n\t\tlog.Infof(\"Created bridge HNSEndpoint %s\", bridgeEndpointName)\n\t}\n\n\t// Wait for the bridgeEndpoint to attach to the host\n\tlog.Infof(\"Waiting to attach bridge endpoint %s to host\", bridgeEndpointName)\n\twaitErr = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 5*time.Second, true, func(context.Context) (done bool, err error) {\n\t\tlastErr = expectedBridgeEndpoint.HostAttach(1)\n\t\tif lastErr == nil {\n\t\t\treturn true, nil\n\t\t}\n\t\t// See https://github.com/flannel-io/flannel/issues/1391 and\n\t\t// hcsshim lacks some validations to detect the error, so we judge it by error message.\n\t\tif strings.Contains(lastErr.Error(), \"This endpoint is already attached to the switch.\") {\n\t\t\treturn true, nil\n\t\t}\n\t\treturn false, nil\n\t})\n\tif waitErr != nil {\n\t\treturn nil, errors.Wrapf(lastErr, \"failed to hot attach bridge HNSEndpoint %s to host compartment\", bridgeEndpointName)\n\t}\n\tlog.Infof(\"Attached bridge endpoint %s to host successfully\", bridgeEndpointName)\n\n\t// 7. Enable forwarding on the host interface and endpoint\n\tfor _, interfaceIpAddress := range []string{expectedNetwork.ManagementIP, expectedBridgeEndpoint.IPAddress.String()} {\n\t\tipv4, err := ip.ParseIP4(interfaceIpAddress)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"Failed to parse expected net interface ip (%s)\", interfaceIpAddress)\n\t\t}\n\n\t\tnetInterface, err := ip.GetInterfaceByIP(ipv4.ToIP())\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to find interface for IP Address %s\", interfaceIpAddress)\n\t\t}\n\t\tlog.Infof(\"Found %+v interface with IP %s\", netInterface, interfaceIpAddress)\n\n\t\t// When a new hns network is created, the interface is modified, esp the name, index\n\t\tif expectedNetwork.ManagementIP == ipv4.String() {\n\t\t\tn.LinkIndex = netInterface.Index\n\t\t\tn.Name = netInterface.Name\n\t\t}\n\n\t\tif err := ip.EnableForwardingForInterface(netInterface); err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to enable forwarding on %s index %d\", netInterface.Name, netInterface.Index)\n\t\t}\n\t\tlog.Infof(\"Enabled forwarding on %s index %d\", netInterface.Name, netInterface.Index)\n\t}\n\n\treturn n, nil\n}\n"
  },
  {
    "path": "pkg/backend/ipip/ipip.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ipip\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\t\"syscall\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/vishvananda/netlink\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\tbackendType = \"ipip\"\n\ttunnelName  = \"flannel.ipip\"\n)\n\nfunc init() {\n\tbackend.Register(backendType, New)\n}\n\ntype IPIPBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbe := &IPIPBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\treturn be, nil\n}\n\nfunc (be *IPIPBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\tcfg := struct {\n\t\tDirectRouting bool\n\t}{}\n\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding IPIP backend config: %v\", err)\n\t\t}\n\t}\n\n\tlog.Infof(\"IPIP config: DirectRouting=%v\", cfg.DirectRouting)\n\n\tn := &backend.RouteNetwork{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tExtIface: be.extIface,\n\t\t},\n\t\tSM:          be.sm,\n\t\tBackendType: backendType,\n\t}\n\n\tattrs := &lease.LeaseAttrs{\n\t\tPublicIP:    ip.FromIP(be.extIface.ExtAddr),\n\t\tBackendType: backendType,\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, attrs)\n\tswitch err {\n\tcase nil:\n\t\tn.SubnetLease = l\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\tlink, err := be.configureIPIPDevice(n.SubnetLease, config.Network)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tn.Mtu = link.MTU\n\tn.LinkIndex = link.Index\n\tn.GetRoute = func(lease *lease.Lease) *netlink.Route {\n\t\troute := netlink.Route{\n\t\t\tDst:       lease.Subnet.ToIPNet(),\n\t\t\tGw:        lease.Attrs.PublicIP.ToIP(),\n\t\t\tLinkIndex: n.LinkIndex,\n\t\t\tFlags:     int(netlink.FLAG_ONLINK),\n\t\t}\n\n\t\tif cfg.DirectRouting {\n\t\t\tdr, err := ip.DirectRouting(lease.Attrs.PublicIP.ToIP())\n\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\n\t\t\tif dr {\n\t\t\t\tlog.V(2).Infof(\"configure route to %v via direct routing\", lease.Attrs.PublicIP.String())\n\t\t\t\troute.LinkIndex = n.ExtIface.Iface.Index\n\t\t\t}\n\t\t}\n\n\t\treturn &route\n\t}\n\n\treturn n, nil\n}\n\nfunc (be *IPIPBackend) configureIPIPDevice(lease *lease.Lease, flannelnet ip.IP4Net) (*netlink.Iptun, error) {\n\t// When modprobe ipip module, a tunl0 ipip device is created automatically per network namespace by ipip kernel module.\n\t// It is the namespace default IPIP device with attributes local=any and remote=any.\n\t// When receiving IPIP protocol packets, kernel will forward them to tunl0 as a fallback device\n\t// if it can't find an option whose local/remote attribute matches their src/dst ip address more precisely.\n\t// See https://github.com/torvalds/linux/blob/v4.13/net/ipv4/ip_tunnel.c#L85-L95 .\n\n\t// So we have two options of creating ipip device, either rename tunl0 to flannel.ipip or create an new ipip device\n\t// and set local attribute of flannel.ipip to distinguish these two devices.\n\t// Considering tunl0 might be used by users, so choose the later option.\n\tlink := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: tunnelName}, Local: be.extIface.IfaceAddr}\n\n\tif err := netlink.LinkAdd(link); err != nil {\n\t\tif err != syscall.EEXIST {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// The link already exists, so check existing link attributes.\n\t\texisting, err := netlink.LinkByName(tunnelName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// If there's an exists device but it's not an ipip/IpTun device then get the user to fix it (flannel shouldn't\n\t\t// delete a user's device)\n\t\tif existing.Type() != \"ipip\" {\n\t\t\treturn nil, fmt.Errorf(\"%v isn't an ipip mode device, please remove device and try again\", tunnelName)\n\t\t}\n\t\tipip, ok := existing.(*netlink.Iptun)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"%s isn't an iptun device (%#v), please remove device and try again\", tunnelName, link)\n\t\t}\n\n\t\t// local attribute may change if a user changes iface configuration, we need to recreate the device to ensure\n\t\t// local and remote attribute is expected.\n\t\t// local should be equal to the extIface.IfaceAddr and remote should be nil (or equal to 0.0.0.0)\n\t\tif ipip.Local == nil || !ipip.Local.Equal(be.extIface.IfaceAddr) || (ipip.Remote != nil && ipip.Remote.String() != \"0.0.0.0\") {\n\t\t\tlog.Warningf(\"%q already exists with incompatible attributes: local=%v remote=%v; recreating device\",\n\t\t\t\ttunnelName, ipip.Local, ipip.Remote)\n\n\t\t\tif err = netlink.LinkDel(existing); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to delete interface: %v\", err)\n\t\t\t}\n\n\t\t\tif err = netlink.LinkAdd(link); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to create ipip interface: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Due to the extra 20 byte IP header that the tunnel will add to each packet,\n\t// MTU size for both the workload and tunnel interfaces should be 20 bytes less than the selected iface (specified with the --iface option).\n\texpectMTU := be.extIface.Iface.MTU - 20\n\tif expectMTU <= 0 {\n\t\treturn nil, fmt.Errorf(\"MTU %d of iface %s is too small for ipip mode to work\", be.extIface.Iface.MTU, be.extIface.Iface.Name)\n\t}\n\n\toldMTU := link.Attrs().MTU\n\tif oldMTU > expectMTU || oldMTU == 0 {\n\t\tlog.Infof(\"current MTU of %s is %d, setting it to %d\", tunnelName, oldMTU, expectMTU)\n\t\terr := netlink.LinkSetMTU(link, expectMTU)\n\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to set %v MTU to %d: %v\", tunnelName, expectMTU, err)\n\t\t}\n\t\t// change MTU as it will be written into /run/flannel/subnet.env\n\t\tlink.Attrs().MTU = expectMTU\n\t}\n\n\t// Ensure that the device has a /32 address so that no broadcast routes are created.\n\t// This IP is just used as a source address for host to workload traffic (so\n\t// the return path for the traffic has an address on the flannel network to use as the destination)\n\tif err := ip.EnsureV4AddressOnLink(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, flannelnet, link); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to ensure address of interface %s: %s\", link.Attrs().Name, err)\n\t}\n\n\tif err := netlink.LinkSetUp(link); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to set %v UP: %v\", tunnelName, err)\n\t}\n\n\treturn link, nil\n}\n"
  },
  {
    "path": "pkg/backend/ipip/ipip_windows.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build windows\n// +build windows\n\npackage ipip\n"
  },
  {
    "path": "pkg/backend/ipsec/handle_charon.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage ipsec\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/bronze1man/goStrongswanVici\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype Uri struct {\n\tnetwork, address string\n}\n\ntype CharonIKEDaemon struct {\n\tviciUri     Uri\n\tespProposal string\n\tctx         context.Context\n}\n\nfunc NewCharonIKEDaemon(ctx context.Context, wg *sync.WaitGroup, espProposal string) (*CharonIKEDaemon, error) {\n\tcharon := &CharonIKEDaemon{ctx: ctx, espProposal: espProposal}\n\n\taddr := strings.Split(\"unix:///var/run/charon.vici\", \"://\")\n\tcharon.viciUri = Uri{addr[0], addr[1]}\n\n\texecPath, err := findExecPath()\n\tif err != nil {\n\t\tlog.Errorf(\"Charon daemon not found: %v\", err)\n\t\treturn nil, err\n\t}\n\n\tcmd, err := charon.run(execPath)\n\tif err != nil {\n\t\tlog.Errorf(\"Error starting charon daemon: %v\", err)\n\t\treturn nil, err\n\t} else {\n\t\tlog.Info(\"Charon daemon started\")\n\t}\n\twg.Add(1)\n\tgo func() {\n\t\t<-ctx.Done()\n\t\terr := cmd.Process.Signal(syscall.SIGTERM)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Error while processing the signal: %v\", err)\n\t\t}\n\t\terr = cmd.Wait()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Error while waiting for process to exit: %v\", err)\n\t\t}\n\t\tlog.Infof(\"Stopped charon daemon\")\n\t\twg.Done()\n\t}()\n\treturn charon, nil\n}\n\nfunc (charon *CharonIKEDaemon) getClient(wait bool) (client *goStrongswanVici.ClientConn, err error) {\n\tfor {\n\t\tsocket_conn, err := net.Dial(charon.viciUri.network, charon.viciUri.address)\n\t\tif err == nil {\n\t\t\treturn goStrongswanVici.NewClientConn(socket_conn), nil\n\t\t} else {\n\t\t\tif wait {\n\t\t\t\tselect {\n\t\t\t\tcase <-charon.ctx.Done():\n\t\t\t\t\tlog.Error(\"Cancel waiting for charon\")\n\t\t\t\t\treturn nil, err\n\t\t\t\tdefault:\n\t\t\t\t\tlog.Errorf(\"ClientConnection failed: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tlog.Info(\"Retrying in a second ...\")\n\t\t\t\ttime.Sleep(time.Second)\n\t\t\t} else {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (charon *CharonIKEDaemon) run(execPath string) (cmd *exec.Cmd, err error) {\n\tcmd = &exec.Cmd{\n\t\tPath: execPath,\n\t\tSysProcAttr: &syscall.SysProcAttr{\n\t\t\tPdeathsig: syscall.SIGTERM,\n\t\t},\n\t\tStdout: os.Stdout,\n\t\tStderr: os.Stderr,\n\t}\n\terr = cmd.Start()\n\treturn\n}\n\nfunc (charon *CharonIKEDaemon) LoadSharedKey(remotePublicIP, password string) error {\n\tvar err error\n\tvar client *goStrongswanVici.ClientConn\n\n\tclient, err = charon.getClient(true)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to acquire Vici client: %v\", err)\n\t\treturn err\n\t}\n\n\tdefer func() {\n\t\terr := client.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close Vici client: %v\", err)\n\t\t}\n\t}()\n\n\tsharedKey := &goStrongswanVici.Key{\n\t\tTyp:    \"IKE\",\n\t\tData:   password,\n\t\tOwners: []string{remotePublicIP},\n\t}\n\n\tfor {\n\t\terr = client.LoadShared(sharedKey)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to load my key. Retrying. %v\", err)\n\t\t\ttime.Sleep(time.Second)\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tlog.Infof(\"Loaded shared key for: %v\", remotePublicIP)\n\treturn nil\n}\n\nfunc (charon *CharonIKEDaemon) LoadConnection(localLease, remoteLease *lease.Lease,\n\treqID, encap string) error {\n\tvar err error\n\tvar client *goStrongswanVici.ClientConn\n\n\tclient, err = charon.getClient(true)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to acquire Vici client: %s\", err)\n\t\treturn err\n\t}\n\tdefer func() {\n\t\terr := client.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close Vici client: %v\", err)\n\t\t}\n\t}()\n\n\tchildConfMap := make(map[string]goStrongswanVici.ChildSAConf)\n\tchildSAConf := goStrongswanVici.ChildSAConf{\n\t\tLocal_ts:      []string{localLease.Subnet.String()},\n\t\tRemote_ts:     []string{remoteLease.Subnet.String()},\n\t\tESPProposals:  []string{charon.espProposal},\n\t\tStartAction:   \"start\",\n\t\tCloseAction:   \"trap\",\n\t\tDpdAction:     \"restart\",\n\t\tMode:          \"tunnel\",\n\t\tReqID:         reqID,\n\t\tRekeyTime:     \"1h\",\n\t\tInstallPolicy: \"no\",\n\t}\n\n\tchildSAConfName := formatChildSAConfName(localLease, remoteLease)\n\n\tchildConfMap[childSAConfName] = childSAConf\n\n\tlocalAuthConf := goStrongswanVici.AuthConf{\n\t\tAuthMethod: \"psk\",\n\t}\n\tremoteAuthConf := goStrongswanVici.AuthConf{\n\t\tAuthMethod: \"psk\",\n\t}\n\n\tikeConf := goStrongswanVici.IKEConf{\n\t\tLocalAddrs:  []string{localLease.Attrs.PublicIP.String()},\n\t\tRemoteAddrs: []string{remoteLease.Attrs.PublicIP.String()},\n\t\tProposals:   []string{\"aes256-sha256-modp4096\"},\n\t\tVersion:     \"2\",\n\t\tKeyingTries: \"0\", //continues to retry\n\t\tLocalAuth:   localAuthConf,\n\t\tRemoteAuth:  remoteAuthConf,\n\t\tChildren:    childConfMap,\n\t\tEncap:       encap,\n\t}\n\tikeConfMap := make(map[string]goStrongswanVici.IKEConf)\n\n\tconnectionName := formatConnectionName(localLease, remoteLease)\n\tikeConfMap[connectionName] = ikeConf\n\n\terr = client.LoadConn(&ikeConfMap)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tlog.Infof(\"Loaded connection: %v\", connectionName)\n\treturn nil\n}\n\nfunc (charon *CharonIKEDaemon) UnloadCharonConnection(localLease,\n\tremoteLease *lease.Lease) error {\n\tclient, err := charon.getClient(false)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to acquire Vici client: %s\", err)\n\t\treturn err\n\t}\n\tdefer func() {\n\t\terr := client.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close Vici client: %v\", err)\n\t\t}\n\t}()\n\n\tconnectionName := formatConnectionName(localLease, remoteLease)\n\tunloadConnRequest := &goStrongswanVici.UnloadConnRequest{\n\t\tName: connectionName,\n\t}\n\n\terr = client.UnloadConn(unloadConnRequest)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tlog.Infof(\"Unloaded connection: %v\", connectionName)\n\treturn nil\n}\n\nfunc formatConnectionName(localLease, remoteLease *lease.Lease) string {\n\treturn fmt.Sprintf(\"%s-%s-%s-%s\", localLease.Attrs.PublicIP,\n\t\tlocalLease.Subnet, remoteLease.Subnet, remoteLease.Attrs.PublicIP)\n}\n\nfunc formatChildSAConfName(localLease, remoteLease *lease.Lease) string {\n\treturn fmt.Sprintf(\"%s-%s\", localLease.Subnet, remoteLease.Subnet)\n}\n\nfunc findExecPath() (string, error) {\n\t// try well known charon paths\n\tpaths := []string{\n\t\t\"charon\",                         // PATH\n\t\t\"/usr/lib/strongswan/charon\",     // alpine, arch, flannel container\n\t\t\"/usr/lib/ipsec/charon\",          // debian/ubuntu\n\t\t\"/usr/libexec/strongswan/charon\", // centos/rhel\n\t\t\"/usr/libexec/ipsec/charon\",      // opensuse/sles\n\t}\n\tfor _, path := range paths {\n\t\tpath, err := exec.LookPath(path)\n\t\tif err != nil {\n\t\t\tlog.Warningf(\"no valid charon executable found at path %s: %v\", path, err)\n\t\t\tcontinue\n\t\t}\n\t\treturn path, nil\n\t}\n\n\terr := fmt.Errorf(\"no valid charon executable found at paths %v\", paths)\n\treturn \"\", err\n}\n"
  },
  {
    "path": "pkg/backend/ipsec/handle_xfrm.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage ipsec\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"syscall\"\n\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/vishvananda/netlink\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc AddXFRMPolicy(myLease, remoteLease *lease.Lease, dir netlink.Dir, reqID int) error {\n\tsrc := myLease.Subnet.ToIPNet()\n\n\tdst := remoteLease.Subnet.ToIPNet()\n\n\tpolicy := &netlink.XfrmPolicy{\n\t\tSrc: src,\n\t\tDst: dst,\n\t\tDir: dir,\n\t}\n\n\ttunnelLeft := myLease.Attrs.PublicIP.ToIP()\n\ttunnelRight := remoteLease.Attrs.PublicIP.ToIP()\n\n\ttmpl := netlink.XfrmPolicyTmpl{\n\t\tSrc:   tunnelLeft,\n\t\tDst:   tunnelRight,\n\t\tProto: netlink.XFRM_PROTO_ESP,\n\t\tMode:  netlink.XFRM_MODE_TUNNEL,\n\t\tReqid: reqID,\n\t}\n\n\tpolicy.Tmpls = append(policy.Tmpls, tmpl)\n\n\tif existingPolicy, err := netlink.XfrmPolicyGet(policy); err != nil {\n\t\tif errors.Is(err, syscall.ENOENT) {\n\t\t\tlog.Infof(\"Adding ipsec policy: %+v\", tmpl)\n\t\t\tif err := netlink.XfrmPolicyAdd(policy); err != nil {\n\t\t\t\treturn fmt.Errorf(\"error adding policy: %+v err: %v\", policy, err)\n\t\t\t}\n\t\t} else {\n\t\t\treturn fmt.Errorf(\"error getting policy: %+v err: %v\", policy, err)\n\t\t}\n\t} else {\n\t\tlog.Infof(\"Updating ipsec policy %+v with %+v\", existingPolicy, policy)\n\t\tif err := netlink.XfrmPolicyUpdate(policy); err != nil {\n\t\t\treturn fmt.Errorf(\"error updating policy: %+v err: %v\", policy, err)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc DeleteXFRMPolicy(localSubnet, remoteSubnet *net.IPNet, localPublicIP, remotePublicIP net.IP, dir netlink.Dir, reqID int) error {\n\tsrc := localSubnet\n\tdst := remoteSubnet\n\n\tpolicy := netlink.XfrmPolicy{\n\t\tSrc: src,\n\t\tDst: dst,\n\t\tDir: dir,\n\t}\n\n\ttunnelLeft := localPublicIP\n\ttunnelRight := remotePublicIP\n\n\ttmpl := netlink.XfrmPolicyTmpl{\n\t\tSrc:   tunnelLeft,\n\t\tDst:   tunnelRight,\n\t\tProto: netlink.XFRM_PROTO_ESP,\n\t\tMode:  netlink.XFRM_MODE_TUNNEL,\n\t\tReqid: reqID,\n\t}\n\n\tlog.Infof(\"Deleting ipsec policy: %+v\", tmpl)\n\n\tpolicy.Tmpls = append(policy.Tmpls, tmpl)\n\n\tif err := netlink.XfrmPolicyDel(&policy); err != nil {\n\t\treturn fmt.Errorf(\"error deleting policy: %+v err: %v\", policy, err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/backend/ipsec/ipsec.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage ipsec\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\n/*\n\tFlannel's approach to IPSec uses Strongswan to handle the key exchange (using IKEv2) and the kernel to handle the\n\tactual encryption.\n\n\tFlannel runs Strongswan's \"charon\" as a child process when the ipsec backend is selected and communicates with it\n\tusing the \"VICI\" interface. Strongswan ships a utility \"swanctl\" which also uses the VICI interface. This utility\n\tis bundled in the flannel container and can help with debugging.\n\n\tThe file \"handle_charon.go\" contains the logic for working with the charon. It supports creating a \"CharonIKEDaemon\"\n\twhich supports loading the PSK into the charon and adding and removing connections.\n\n\tThe file \"handle_xfrm.go\" contains functions for adding and removing the ipsec polcies.\n\n\tipsec_network.go ties it all together, loading the PSK for current host on startu and as new hosts are added and\n\tremoved it, adds/removes the PSK and connection details to strongswan and adds/remove the policy to the kernel.\n*/\n\nconst (\n\tdefaultESPProposal = \"aes128gcm16-sha256-prfsha256-ecp256\"\n\tminPasswordLength  = 96\n)\n\nfunc init() {\n\tbackend.Register(\"ipsec\", New)\n}\n\ntype IPSECBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (\n\tbackend.Backend, error) {\n\tbe := &IPSECBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\n\treturn be, nil\n}\n\nfunc (be *IPSECBackend) RegisterNetwork(\n\tctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\n\tcfg := struct {\n\t\tUDPEncap    bool\n\t\tESPProposal string\n\t\tPSK         string\n\t}{\n\t\tUDPEncap:    false,\n\t\tESPProposal: defaultESPProposal,\n\t}\n\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding IPSEC backend config: %v\", err)\n\t\t}\n\t}\n\n\tif len(cfg.PSK) < minPasswordLength {\n\t\treturn nil, fmt.Errorf(\"config error, password is too short\")\n\t}\n\n\tlog.Infof(\"IPSec config: UDPEncap=%v ESPProposal=%s\", cfg.UDPEncap, cfg.ESPProposal)\n\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP:    ip.FromIP(be.extIface.ExtAddr),\n\t\tBackendType: \"ipsec\",\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, &attrs)\n\n\tswitch err {\n\tcase nil:\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\tikeDaemon, err := NewCharonIKEDaemon(ctx, wg, cfg.ESPProposal)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error creating CharonIKEDaemon struct: %v\", err)\n\t}\n\n\treturn newNetwork(be.sm, be.extIface, cfg.UDPEncap, cfg.PSK, ikeDaemon, l)\n}\n"
  },
  {
    "path": "pkg/backend/ipsec/ipsec_network.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage ipsec\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strconv\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/vishvananda/netlink\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\t/*\n\t   New IP header (Tunnel Mode)   : 20\n\t   SPI (ESP Header)              : 4\n\t   Sequence (ESP Header)         : 4\n\t   ESP-AES IV                    : 16\n\t   ESP-AES Pad                   : 0-15\n\t   Pad length (ESP Trailer)      : 1\n\t   Next Header (ESP Trailer)     : 1\n\t   ESP-SHA-256 ICV               : 16\n\t*/\n\tipsecOverhead    = 77\n\tudpEncapOverhead = 8\n\n\tdefaultReqID = 11\n)\n\ntype network struct {\n\tbackend.SimpleNetwork\n\tpassword string\n\tUDPEncap bool\n\tsm       subnet.Manager\n\tiked     *CharonIKEDaemon\n}\n\nfunc newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface,\n\tUDPEncap bool, password string, ikeDaemon *CharonIKEDaemon,\n\tl *lease.Lease) (*network, error) {\n\tn := &network{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tSubnetLease: l,\n\t\t\tExtIface:    extIface,\n\t\t},\n\t\tsm:       sm,\n\t\tiked:     ikeDaemon,\n\t\tpassword: password,\n\t\tUDPEncap: UDPEncap,\n\t}\n\n\treturn n, nil\n}\n\nfunc (n *network) Run(ctx context.Context) {\n\n\terr := n.iked.LoadSharedKey(n.SimpleNetwork.SubnetLease.Attrs.PublicIP.ToIP().String(), n.password)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to load PSK: %v\", err)\n\t\treturn\n\t}\n\n\twg := sync.WaitGroup{}\n\tdefer wg.Wait()\n\n\tlog.Info(\"Watching for new subnet leases\")\n\n\tevts := make(chan []lease.Event)\n\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, n.sm, n.SubnetLease, evts)\n\t\tlog.Info(\"WatchLeases exited\")\n\t\twg.Done()\n\t}()\n\n\tfor {\n\t\tevtsBatch, ok := <-evts\n\t\tif !ok {\n\t\t\tlog.Infof(\"evts chan closed\")\n\t\t\treturn\n\t\t}\n\t\tlog.Info(\"Handling event\")\n\t\tn.handleSubnetEvents(evtsBatch)\n\t}\n}\n\nfunc (n *network) handleSubnetEvents(batch []lease.Event) {\n\tfor _, evt := range batch {\n\t\tswitch evt.Type {\n\t\tcase lease.EventAdded:\n\t\t\tlog.Info(\"Subnet added: \", evt.Lease.Subnet)\n\n\t\t\tif evt.Lease.Attrs.BackendType != \"ipsec\" {\n\t\t\t\tlog.Warningf(\"Ignoring non-ipsec event: type: %v\", evt.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif evt.Lease.Subnet.Equal(n.SubnetLease.Subnet) {\n\t\t\t\tlog.Warningf(\"Ignoring own lease add event: %+v\", evt.Lease)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif err := n.AddIPSECPolicies(&evt.Lease, defaultReqID); err != nil {\n\t\t\t\tlog.Errorf(\"error adding ipsec policy: %v\", err)\n\t\t\t}\n\n\t\t\tif err := n.iked.LoadSharedKey(evt.Lease.Attrs.PublicIP.String(), n.password); err != nil {\n\t\t\t\tlog.Errorf(\"error loading shared key into IKE daemon: %v\", err)\n\t\t\t}\n\n\t\t\tif err := n.iked.LoadConnection(n.SubnetLease, &evt.Lease, strconv.Itoa(defaultReqID),\n\t\t\t\tstrconv.FormatBool(n.UDPEncap)); err != nil {\n\t\t\t\tlog.Errorf(\"error loading connection into IKE daemon: %v\", err)\n\t\t\t}\n\n\t\tcase lease.EventRemoved:\n\t\t\tlog.Info(\"Subnet removed: \", evt.Lease.Subnet)\n\t\t\tif evt.Lease.Attrs.BackendType != \"ipsec\" {\n\t\t\t\tlog.Warningf(\"Ignoring non-ipsec event: type: %v\", evt.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif evt.Lease.Subnet.Equal(n.SubnetLease.Subnet) {\n\t\t\t\tlog.Warningf(\"Ignoring own lease remove event: %+v\", evt.Lease)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif err := n.iked.UnloadCharonConnection(n.SubnetLease, &evt.Lease); err != nil {\n\t\t\t\tlog.Errorf(\"error unloading charon connections: %v\", err)\n\t\t\t}\n\n\t\t\tif err := n.DeleteIPSECPolicies(n.SubnetLease.Subnet.ToIPNet(), evt.Lease.Subnet.ToIPNet(),\n\t\t\t\tn.SubnetLease.Attrs.PublicIP.ToIP(), evt.Lease.Attrs.PublicIP.ToIP(), defaultReqID); err != nil {\n\n\t\t\t\tlog.Errorf(\"error deleting ipsec policies: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (n *network) MTU() int {\n\tmtu := n.ExtIface.Iface.MTU - ipsecOverhead\n\tif n.UDPEncap {\n\t\tmtu -= udpEncapOverhead\n\t}\n\n\treturn mtu\n}\n\nfunc (n *network) AddIPSECPolicies(remoteLease *lease.Lease, reqID int) error {\n\terr := AddXFRMPolicy(n.SubnetLease, remoteLease, netlink.XFRM_DIR_OUT, reqID)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error adding ipsec out policy: %v\", err)\n\t}\n\n\terr = AddXFRMPolicy(remoteLease, n.SubnetLease, netlink.XFRM_DIR_IN, reqID)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error adding ipsec in policy: %v\", err)\n\t}\n\n\terr = AddXFRMPolicy(remoteLease, n.SubnetLease, netlink.XFRM_DIR_FWD, reqID)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error adding ipsec fwd policy: %v\", err)\n\t}\n\n\treturn nil\n}\n\nfunc (n *network) DeleteIPSECPolicies(localSubnet, remoteSubnet *net.IPNet, localPublicIP, remotePublicIP net.IP, reqID int) error {\n\terr := DeleteXFRMPolicy(localSubnet, remoteSubnet, localPublicIP, remotePublicIP, netlink.XFRM_DIR_OUT, reqID)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error deleting ipsec out policy: %v\", err)\n\t}\n\n\terr = DeleteXFRMPolicy(remoteSubnet, localSubnet, remotePublicIP, localPublicIP, netlink.XFRM_DIR_IN, reqID)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error deleting ipsec in policy: %v\", err)\n\t}\n\n\terr = DeleteXFRMPolicy(remoteSubnet, localSubnet, remotePublicIP, localPublicIP, netlink.XFRM_DIR_FWD, reqID)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error deleting ipsec fwd policy: %v\", err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/backend/ipsec/ipsec_windows.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ipsec\n"
  },
  {
    "path": "pkg/backend/manager.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage backend\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\nvar constructors = make(map[string]BackendCtor)\n\ntype Manager interface {\n\tGetBackend(backendType string) (Backend, error)\n}\n\ntype manager struct {\n\tctx      context.Context\n\tsm       subnet.Manager\n\textIface *ExternalInterface\n\tmux      sync.Mutex\n\tactive   map[string]Backend\n\twg       sync.WaitGroup\n}\n\nfunc NewManager(ctx context.Context, sm subnet.Manager, extIface *ExternalInterface) Manager {\n\treturn &manager{\n\t\tctx:      ctx,\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t\tactive:   make(map[string]Backend),\n\t}\n}\n\nfunc (bm *manager) GetBackend(backendType string) (Backend, error) {\n\tbm.mux.Lock()\n\tdefer bm.mux.Unlock()\n\n\tbetype := strings.ToLower(backendType)\n\t// see if one is already running\n\tif be, ok := bm.active[betype]; ok {\n\t\treturn be, nil\n\t}\n\n\t// first request, need to create and run it\n\tbefunc, ok := constructors[betype]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"unknown backend type: %v\", betype)\n\t}\n\n\tbe, err := befunc(bm.sm, bm.extIface)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbm.active[betype] = be\n\n\tbm.wg.Add(1)\n\tgo func() {\n\t\t<-bm.ctx.Done()\n\n\t\t// TODO(eyakubovich): this obviosly introduces a race.\n\t\t// GetBackend() could get called while we are here.\n\t\t// Currently though, all backends' Run exit only\n\t\t// on shutdown\n\n\t\tbm.mux.Lock()\n\t\tdelete(bm.active, betype)\n\t\tbm.mux.Unlock()\n\n\t\tbm.wg.Done()\n\t}()\n\n\treturn be, nil\n}\n\nfunc Register(name string, ctor BackendCtor) {\n\tconstructors[name] = ctor\n}\n"
  },
  {
    "path": "pkg/backend/route_network.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage backend\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"net\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/vishvananda/netlink\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\trouteCheckRetries = 10\n)\n\ntype RouteNetwork struct {\n\tSimpleNetwork\n\tBackendType string\n\troutes      []netlink.Route\n\tv6Routes    []netlink.Route\n\tSM          subnet.Manager\n\tGetRoute    func(lease *lease.Lease) *netlink.Route\n\tGetV6Route  func(lease *lease.Lease) *netlink.Route\n\tMtu         int\n\tLinkIndex   int\n}\n\nfunc (n *RouteNetwork) MTU() int {\n\treturn n.Mtu\n}\n\nfunc (n *RouteNetwork) Run(ctx context.Context) {\n\twg := sync.WaitGroup{}\n\n\tlog.Info(\"Watching for new subnet leases\")\n\tevts := make(chan []lease.Event)\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, n.SM, n.SubnetLease, evts)\n\t\twg.Done()\n\t}()\n\n\tn.routes = make([]netlink.Route, 0, 10)\n\twg.Add(1)\n\tgo func() {\n\t\tn.routeCheck(ctx)\n\t\twg.Done()\n\t}()\n\n\tdefer wg.Wait()\n\n\tfor {\n\t\tevtBatch, ok := <-evts\n\t\tif !ok {\n\t\t\tlog.Infof(\"evts chan closed\")\n\t\t\treturn\n\t\t}\n\t\tn.handleSubnetEvents(evtBatch)\n\t}\n}\n\nfunc (n *RouteNetwork) handleSubnetEvents(batch []lease.Event) {\n\tfor _, evt := range batch {\n\t\tswitch evt.Type {\n\t\tcase lease.EventAdded:\n\t\t\tif evt.Lease.Attrs.BackendType != n.BackendType {\n\t\t\t\tlog.Warningf(\"Ignoring non-%v subnet: type=%v\", n.BackendType, evt.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif evt.Lease.EnableIPv4 {\n\t\t\t\tlog.Infof(\"Subnet added: %v via %v\", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)\n\n\t\t\t\troute := n.GetRoute(&evt.Lease)\n\t\t\t\trouteAdd(route, netlink.FAMILY_V4, n.addToRouteList, n.removeFromV4RouteList)\n\t\t\t}\n\n\t\t\tif evt.Lease.EnableIPv6 {\n\t\t\t\tlog.Infof(\"Subnet added: %v via %v\", evt.Lease.IPv6Subnet, evt.Lease.Attrs.PublicIPv6)\n\n\t\t\t\troute := n.GetV6Route(&evt.Lease)\n\t\t\t\trouteAdd(route, netlink.FAMILY_V6, n.addToV6RouteList, n.removeFromV6RouteList)\n\t\t\t}\n\n\t\tcase lease.EventRemoved:\n\t\t\tif evt.Lease.Attrs.BackendType != n.BackendType {\n\t\t\t\tlog.Warningf(\"Ignoring non-%v subnet: type=%v\", n.BackendType, evt.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif evt.Lease.EnableIPv4 {\n\t\t\t\tlog.Info(\"Subnet removed: \", evt.Lease.Subnet)\n\n\t\t\t\troute := n.GetRoute(&evt.Lease)\n\t\t\t\t// Always remove the route from the route list.\n\t\t\t\tn.removeFromV4RouteList(*route)\n\n\t\t\t\tif err := netlink.RouteDel(route); err != nil {\n\t\t\t\t\tlog.Errorf(\"Error deleting route to %v: %v\", evt.Lease.Subnet, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif evt.Lease.EnableIPv6 {\n\t\t\t\tlog.Info(\"Subnet removed: \", evt.Lease.IPv6Subnet)\n\n\t\t\t\troute := n.GetV6Route(&evt.Lease)\n\t\t\t\t// Always remove the route from the route list.\n\t\t\t\tn.removeFromV6RouteList(*route)\n\n\t\t\t\tif err := netlink.RouteDel(route); err != nil {\n\t\t\t\t\tlog.Errorf(\"Error deleting route to %v: %v\", evt.Lease.IPv6Subnet, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\tdefault:\n\t\t\tlog.Error(\"Internal error: unknown event type: \", int(evt.Type))\n\t\t}\n\t}\n}\n\nfunc routeAdd(route *netlink.Route, ipFamily int, addToRouteList, removeFromRouteList func(netlink.Route)) {\n\taddToRouteList(*route)\n\t// Check if route exists before attempting to add it\n\trouteList, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)\n\tif err != nil {\n\t\tlog.Warningf(\"Unable to list routes: %v\", err)\n\t}\n\n\tif len(routeList) > 0 && !routeEqual(routeList[0], *route) {\n\t\t// Same Dst different Gw or different link index. Remove it, correct route will be added below.\n\t\tlog.Warningf(\"Replacing existing route to %v with %v\", routeList[0], route)\n\t\tif err := netlink.RouteDel(&routeList[0]); err != nil {\n\t\t\tlog.Errorf(\"Effor deleteing route to %v: %v\", routeList[0].Dst, err)\n\t\t\treturn\n\t\t}\n\t\tremoveFromRouteList(routeList[0])\n\t}\n\trouteList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)\n\tif err != nil {\n\t\tlog.Warningf(\"Unable to list routes: %v\", err)\n\t}\n\n\tif len(routeList) > 0 && routeEqual(routeList[0], *route) {\n\t\t// Same Dst and same Gw, keep it and do not attempt to add it.\n\t\tlog.Infof(\"Route to %v already exists, skipping.\", route)\n\t} else if err := netlink.RouteAdd(route); err != nil {\n\t\tlog.Errorf(\"Error adding route to %v: %s\", route, err)\n\t\treturn\n\t}\n\t_, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)\n\tif err != nil {\n\t\tlog.Warningf(\"Unable to list routes: %v\", err)\n\t}\n}\n\nfunc (n *RouteNetwork) addToRouteList(route netlink.Route) {\n\tn.routes = addToRouteList(&route, n.routes)\n}\n\nfunc (n *RouteNetwork) addToV6RouteList(route netlink.Route) {\n\tn.v6Routes = addToRouteList(&route, n.v6Routes)\n}\n\nfunc addToRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {\n\tfor _, r := range routes {\n\t\tif routeEqual(r, *route) {\n\t\t\treturn routes\n\t\t}\n\t}\n\treturn append(routes, *route)\n}\n\nfunc (n *RouteNetwork) removeFromV4RouteList(route netlink.Route) {\n\tn.routes = n.removeFromRouteList(&route, n.routes)\n}\n\nfunc (n *RouteNetwork) removeFromV6RouteList(route netlink.Route) {\n\tn.v6Routes = n.removeFromRouteList(&route, n.v6Routes)\n}\n\nfunc (n *RouteNetwork) removeFromRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {\n\tfor index, r := range routes {\n\t\tif routeEqual(r, *route) {\n\t\t\troutes = append(routes[:index], routes[index+1:]...)\n\t\t\treturn routes\n\t\t}\n\t}\n\treturn routes\n}\n\nfunc (n *RouteNetwork) routeCheck(ctx context.Context) {\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-time.After(routeCheckRetries * time.Second):\n\t\t\tn.checkSubnetExistInV4Routes()\n\t\t\tn.checkSubnetExistInV6Routes()\n\t\t}\n\t}\n}\n\nfunc (n *RouteNetwork) checkSubnetExistInV4Routes() {\n\tn.checkSubnetExistInRoutes(n.routes, netlink.FAMILY_V4)\n}\n\nfunc (n *RouteNetwork) checkSubnetExistInV6Routes() {\n\tn.checkSubnetExistInRoutes(n.v6Routes, netlink.FAMILY_V6)\n}\n\nfunc (n *RouteNetwork) checkSubnetExistInRoutes(routes []netlink.Route, ipFamily int) {\n\trouteList, err := netlink.RouteList(nil, ipFamily)\n\tif err == nil {\n\t\tfor _, route := range routes {\n\t\t\texist := false\n\t\t\tfor _, r := range routeList {\n\t\t\t\tif r.Dst == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif routeEqual(r, route) {\n\t\t\t\t\texist = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !exist {\n\t\t\t\tif err := netlink.RouteAdd(&route); err != nil {\n\t\t\t\t\tif nerr, ok := err.(net.Error); !ok {\n\t\t\t\t\t\tlog.Errorf(\"Error recovering route to %v: %v, %v\", route.Dst, route.Gw, nerr)\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\tlog.Infof(\"Route recovered %v : %v\", route.Dst, route.Gw)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlog.Errorf(\"Error fetching route list. Will automatically retry: %v\", err)\n\t}\n}\n\nfunc routeEqual(x, y netlink.Route) bool {\n\t// For ipip backend, when enabling directrouting, link index of some routes may change\n\t// For both ipip and host-gw backend, link index may also change if updating ExtIface\n\tif x.Dst.IP.Equal(y.Dst.IP) && x.Gw.Equal(y.Gw) && bytes.Equal(x.Dst.Mask, y.Dst.Mask) && x.LinkIndex == y.LinkIndex {\n\t\treturn true\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "pkg/backend/route_network_test.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage backend\n\nimport (\n\t\"net\"\n\t\"testing\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/ns\"\n\t\"github.com/vishvananda/netlink\"\n)\n\nfunc TestRouteCache(t *testing.T) {\n\tteardown := ns.SetUpNetlinkTest(t)\n\tdefer teardown()\n\n\tlo, err := netlink.LinkByName(\"lo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(\"127.0.0.1\"), Mask: net.CIDRMask(32, 32)}}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := netlink.LinkSetUp(lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tnw := RouteNetwork{\n\t\tSimpleNetwork: SimpleNetwork{\n\t\t\tExtIface: &ExternalInterface{Iface: &net.Interface{Index: lo.Attrs().Index}},\n\t\t},\n\t\tBackendType: \"host-gw\",\n\t\tLinkIndex:   lo.Attrs().Index,\n\t}\n\tnw.GetRoute = func(lease *lease.Lease) *netlink.Route {\n\t\treturn &netlink.Route{\n\t\t\tDst:       lease.Subnet.ToIPNet(),\n\t\t\tGw:        lease.Attrs.PublicIP.ToIP(),\n\t\t\tLinkIndex: nw.LinkIndex,\n\t\t}\n\t}\n\tgw1, gw2 := ip.FromIP(net.ParseIP(\"127.0.0.1\")), ip.FromIP(net.ParseIP(\"127.0.0.2\"))\n\tsubnet1 := ip.IP4Net{IP: ip.FromIP(net.ParseIP(\"192.168.0.0\")), PrefixLen: 24}\n\tnw.handleSubnetEvents([]lease.Event{\n\t\t{Type: lease.EventAdded, Lease: lease.Lease{\n\t\t\tSubnet: subnet1, EnableIPv4: true, Attrs: lease.LeaseAttrs{PublicIP: gw1, BackendType: \"host-gw\"}}},\n\t})\n\tif len(nw.routes) != 1 {\n\t\tt.Fatal(nw.routes)\n\t}\n\tif !routeEqual(nw.routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw1.ToIP(), LinkIndex: lo.Attrs().Index}) {\n\t\tt.Fatal(nw.routes[0])\n\t}\n\t// change gateway of previous route\n\tnw.handleSubnetEvents([]lease.Event{\n\t\t{Type: lease.EventAdded, Lease: lease.Lease{\n\t\t\tSubnet: subnet1, EnableIPv4: true, Attrs: lease.LeaseAttrs{PublicIP: gw2, BackendType: \"host-gw\"}}}})\n\tif len(nw.routes) != 1 {\n\t\tt.Fatal(nw.routes)\n\t}\n\tif !routeEqual(nw.routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw2.ToIP(), LinkIndex: lo.Attrs().Index}) {\n\t\tt.Fatal(nw.routes[0])\n\t}\n}\n\nfunc TestV6RouteCache(t *testing.T) {\n\tteardown := ns.SetUpNetlinkTest(t)\n\tdefer teardown()\n\n\tla := netlink.NewLinkAttrs()\n\tla.Name = \"br\"\n\tbr := &netlink.Bridge{LinkAttrs: la}\n\tif err := netlink.LinkAdd(br); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := netlink.AddrAdd(br, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(\"2001:db8:1::1\"), Mask: net.CIDRMask(64, 128)}}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := netlink.LinkSetUp(br); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tnw := RouteNetwork{\n\t\tSimpleNetwork: SimpleNetwork{\n\t\t\tExtIface: &ExternalInterface{Iface: &net.Interface{Index: br.Attrs().Index}},\n\t\t},\n\t\tBackendType: \"host-gw\",\n\t\tLinkIndex:   br.Attrs().Index,\n\t}\n\tnw.GetV6Route = func(lease *lease.Lease) *netlink.Route {\n\t\treturn &netlink.Route{\n\t\t\tDst:       lease.IPv6Subnet.ToIPNet(),\n\t\t\tGw:        lease.Attrs.PublicIPv6.ToIP(),\n\t\t\tLinkIndex: nw.LinkIndex,\n\t\t}\n\t}\n\tgw1, gw2 := ip.FromIP6(net.ParseIP(\"2001:db8:1::2\")), ip.FromIP6(net.ParseIP(\"2001:db8:1::10\"))\n\tsubnet1 := ip.IP6Net{IP: ip.FromIP6(net.ParseIP(\"2001:db8:ffff::\")), PrefixLen: 64}\n\tnw.handleSubnetEvents([]lease.Event{\n\t\t{Type: lease.EventAdded, Lease: lease.Lease{\n\t\t\tIPv6Subnet: subnet1, EnableIPv6: true, Attrs: lease.LeaseAttrs{PublicIPv6: gw1, BackendType: \"host-gw\"}}},\n\t})\n\tif len(nw.v6Routes) != 1 {\n\t\tt.Fatal(nw.v6Routes)\n\t}\n\tif !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw1.ToIP(), LinkIndex: br.Attrs().Index}) {\n\t\tt.Fatal(nw.v6Routes[0])\n\t}\n\t// change gateway of previous route\n\tnw.handleSubnetEvents([]lease.Event{\n\t\t{Type: lease.EventAdded, Lease: lease.Lease{\n\t\t\tIPv6Subnet: subnet1, EnableIPv6: true, Attrs: lease.LeaseAttrs{PublicIPv6: gw2, BackendType: \"host-gw\"}}}})\n\tlinkbr, _ := netlink.LinkByName(la.Name)\n\troutes, _ := netlink.RouteList(linkbr, netlink.FAMILY_V6)\n\tIsGw := \"\"\n\tfor _, route := range routes {\n\t\tif len(route.Gw) != 0 {\n\t\t\tIsGw = route.Gw.String()\n\t\t}\n\t}\n\n\tif IsGw != gw2.String() {\n\t\tt.Fatal(\"Expected Gateway: \", gw2, \" is not the same as the configured gateway: \", IsGw)\n\t}\n\n\tif len(nw.v6Routes) != 1 {\n\t\tt.Fatal(nw.v6Routes)\n\t}\n\tif !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw2.ToIP(), LinkIndex: br.Attrs().Index}) {\n\t\tt.Fatal(nw.v6Routes[0])\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/route_network_windows.go",
    "content": "// Copyright 2018 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage backend\n\nimport (\n\t\"context\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/routing\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\trouteCheckRetries = 10\n)\n\ntype RouteNetwork struct {\n\tSimpleNetwork\n\tName        string\n\tBackendType string\n\tSM          subnet.Manager\n\tGetRoute    func(myLease *lease.Lease) *routing.Route\n\tMtu         int\n\tLinkIndex   int\n\troutes      []routing.Route\n}\n\nfunc (n *RouteNetwork) MTU() int {\n\treturn n.Mtu\n}\n\nfunc (n *RouteNetwork) Run(ctx context.Context) {\n\twg := sync.WaitGroup{}\n\n\tlog.Info(\"Watching for new subnet leases\")\n\tevts := make(chan []lease.Event)\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, n.SM, n.SubnetLease, evts)\n\t\twg.Done()\n\t}()\n\n\tn.routes = make([]routing.Route, 0, 10)\n\twg.Add(1)\n\tgo func() {\n\t\tn.routeCheck(ctx)\n\t\twg.Done()\n\t}()\n\n\tdefer wg.Wait()\n\n\tfor {\n\t\tselect {\n\t\tcase evtBatch, ok := <-evts:\n\t\t\tif !ok {\n\t\t\t\tlog.Infof(\"evts chan closed\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tn.handleSubnetEvents(evtBatch)\n\t\t}\n\t}\n}\n\nfunc (n *RouteNetwork) handleSubnetEvents(batch []lease.Event) {\n\trouter := routing.RouterWindows{}\n\n\tfor _, evt := range batch {\n\t\tleaseSubnet := evt.Lease.Subnet\n\t\tleaseAttrs := evt.Lease.Attrs\n\t\tif !strings.EqualFold(leaseAttrs.BackendType, n.BackendType) {\n\t\t\tlog.Warningf(\"Ignoring non-%v subnet(%v): type=%v\", n.BackendType, leaseSubnet, leaseAttrs.BackendType)\n\t\t\tcontinue\n\t\t}\n\n\t\texpectedRoute := n.GetRoute(&evt.Lease)\n\n\t\tswitch evt.Type {\n\t\tcase lease.EventAdded:\n\t\t\tlog.Infof(\"Subnet added: %v via %v\", leaseSubnet, leaseAttrs.PublicIP)\n\n\t\t\texistingRoutes, _ := router.GetRoutesFromInterfaceToSubnet(expectedRoute.InterfaceIndex, expectedRoute.DestinationSubnet)\n\t\t\tif len(existingRoutes) > 0 {\n\t\t\t\texistingRoute := existingRoutes[0]\n\t\t\t\tif existingRoute.Equal(*expectedRoute) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tlog.Warningf(\"Replacing existing route %v via %v with %v via %v\", leaseSubnet, existingRoute.GatewayAddress, leaseSubnet, leaseAttrs.PublicIP)\n\t\t\t\terr := router.DeleteRoute(existingRoute.InterfaceIndex, existingRoute.DestinationSubnet, existingRoute.GatewayAddress)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"Error removing route: %v\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\n\t\t\terr := router.CreateRoute(expectedRoute.InterfaceIndex, expectedRoute.DestinationSubnet, expectedRoute.GatewayAddress)\n\t\t\tif err != nil {\n\t\t\t\tlog.Errorf(\"Error creating route: %v\", err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tn.addToRouteList(expectedRoute)\n\n\t\tcase lease.EventRemoved:\n\t\t\tlog.Infof(\"Subnet removed: %v\", leaseSubnet)\n\n\t\t\texistingRoutes, _ := router.GetRoutesFromInterfaceToSubnet(expectedRoute.InterfaceIndex, expectedRoute.DestinationSubnet)\n\t\t\tif len(existingRoutes) > 0 {\n\t\t\t\texistingRoute := existingRoutes[0]\n\t\t\t\tif existingRoute.Equal(*expectedRoute) {\n\t\t\t\t\tlog.Infof(\"Removing existing route %v via %v\", leaseSubnet, existingRoute.GatewayAddress)\n\n\t\t\t\t\terr := router.DeleteRoute(existingRoute.InterfaceIndex, existingRoute.DestinationSubnet, existingRoute.GatewayAddress)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Warningf(\"Error removing route: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tn.removeFromRouteList(expectedRoute)\n\n\t\tdefault:\n\t\t\tlog.Error(\"Internal error: unknown event type: \", int(evt.Type))\n\t\t}\n\t}\n}\n\nfunc (n *RouteNetwork) addToRouteList(newRoute *routing.Route) {\n\tfor _, route := range n.routes {\n\t\tif route.Equal(*newRoute) {\n\t\t\treturn\n\t\t}\n\t}\n\n\tn.routes = append(n.routes, *newRoute)\n}\n\nfunc (n *RouteNetwork) removeFromRouteList(oldRoute *routing.Route) {\n\tfor index, route := range n.routes {\n\t\tif route.Equal(*oldRoute) {\n\t\t\tn.routes = append(n.routes[:index], n.routes[index+1:]...)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (n *RouteNetwork) routeCheck(ctx context.Context) {\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tcase <-time.After(routeCheckRetries * time.Second):\n\t\t\tn.checkSubnetExistInRoutes()\n\t\t}\n\t}\n}\n\nfunc (n *RouteNetwork) checkSubnetExistInRoutes() {\n\trouter := routing.RouterWindows{}\n\n\texistingRoutes, err := router.GetAllRoutes()\n\tif err != nil {\n\t\tlog.Errorf(\"Error enumerating routes: %v\", err)\n\t\treturn\n\t}\n\tfor _, expectedRoute := range n.routes {\n\t\texist := false\n\t\tfor _, existingRoute := range existingRoutes {\n\t\t\tif expectedRoute.Equal(existingRoute) {\n\t\t\t\texist = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif !exist {\n\t\t\terr := router.CreateRoute(expectedRoute.InterfaceIndex, expectedRoute.DestinationSubnet, expectedRoute.GatewayAddress)\n\t\t\tif err != nil {\n\t\t\t\tlog.Warningf(\"Error recovering route to %v via %v on %v (%v).\", expectedRoute.DestinationSubnet, expectedRoute.GatewayAddress, expectedRoute.InterfaceIndex, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlog.Infof(\"Recovered route to %v via %v on %v.\", expectedRoute.DestinationSubnet, expectedRoute.GatewayAddress, expectedRoute.InterfaceIndex)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/simple_network.go",
    "content": "// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage backend\n\nimport (\n\t\"context\"\n\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n)\n\ntype SimpleNetwork struct {\n\tSubnetLease *lease.Lease\n\tExtIface    *ExternalInterface\n}\n\nfunc (n *SimpleNetwork) Lease() *lease.Lease {\n\treturn n.SubnetLease\n}\n\nfunc (n *SimpleNetwork) MTU() int {\n\treturn n.ExtIface.Iface.MTU\n}\n\nfunc (*SimpleNetwork) Run(ctx context.Context) {\n\t<-ctx.Done()\n}\n"
  },
  {
    "path": "pkg/backend/tencentvpc/tencentvpc.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//      http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage tencentvpc\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common\"\n\t\"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors\"\n\tvpc \"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc init() {\n\tbackend.Register(\"tencent-vpc\", New)\n}\n\ntype TencentVpcBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbe := TencentVpcBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\treturn &be, nil\n}\n\nfunc get_vm_metadata(url string) (string, error) {\n\tresp, err := http.Get(url)\n\n\tif err != nil || resp.StatusCode != http.StatusOK {\n\t\treturn \"\", fmt.Errorf(\"get vm region error: %v\", err)\n\t}\n\n\tdefer func() {\n\t\terr := resp.Body.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close response body: %v\", err)\n\t\t}\n\t}()\n\n\tmetadata, _ := io.ReadAll(resp.Body)\n\treturn string(metadata), nil\n}\n\nfunc get_vm_region() (string, error) {\n\turl := \"http://metadata.tencentyun.com/latest/meta-data/placement/region\"\n\treturn get_vm_metadata(url)\n}\n\nfunc get_vm_vpcid() (string, error) {\n\tmacUrl := \"http://metadata.tencentyun.com/latest/meta-data/mac\"\n\tmac, err := get_vm_metadata(macUrl)\n\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"get vm mac error: %v\", err)\n\t}\n\n\tvpcUrl := fmt.Sprintf(\"http://metadata.tencentyun.com/latest/meta-data/network/interfaces/macs/%s/vpc-id\", mac)\n\tvpcid, err := get_vm_metadata(vpcUrl)\n\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"get vm vpcid error: %v\", err)\n\t}\n\n\treturn vpcid, nil\n}\n\nfunc (be *TencentVpcBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\t// 1. Parse our configuration\n\tcfg := struct {\n\t\tAccessKeyID     string\n\t\tAccessKeySecret string\n\t}{}\n\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding VPC backend config: %v\", err)\n\t\t}\n\t}\n\tlog.Infof(\"Unmarshal Configure : %v\\n\", cfg)\n\n\t// 2. Acquire the lease form subnet manager\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: ip.FromIP(be.extIface.ExtAddr),\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, &attrs)\n\tswitch err {\n\tcase nil:\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\tif cfg.AccessKeyID == \"\" || cfg.AccessKeySecret == \"\" {\n\t\tcfg.AccessKeyID = os.Getenv(\"ACCESS_KEY_ID\")\n\t\tcfg.AccessKeySecret = os.Getenv(\"ACCESS_KEY_SECRET\")\n\n\t\tif cfg.AccessKeyID == \"\" || cfg.AccessKeySecret == \"\" {\n\t\t\treturn nil, fmt.Errorf(\"ACCESS_KEY_ID and ACCESS_KEY_SECRET must be provided! \")\n\t\t}\n\t}\n\n\tregion, err := get_vm_region()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvpcid, err := get_vm_vpcid()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tc, _ := vpc.NewClientWithSecretId(cfg.AccessKeyID, cfg.AccessKeySecret, region)\n\trequest := vpc.NewDescribeRouteTablesRequest()\n\trequest.Filters = []*vpc.Filter{\n\t\t&vpc.Filter{\n\t\t\tName:   common.StringPtr(\"vpc-id\"),\n\t\t\tValues: common.StringPtrs([]string{vpcid}),\n\t\t},\n\t}\n\n\tres, err := c.DescribeRouteTables(request)\n\tif _, ok := err.(*errors.TencentCloudSDKError); ok {\n\t\treturn nil, fmt.Errorf(\"describe route table error: %v\", ok)\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresponse := res.Response\n\n\tif len(response.RouteTableSet) <= 0 {\n\t\treturn nil, fmt.Errorf(\"no suitable routing table found\")\n\t}\n\n\trouteTable := response.RouteTableSet[0]\n\texists := false\n\tgatewayType := \"NORMAL_CVM\"\n\trouteType := \"USER\"\n\n\tfor _, route := range routeTable.RouteSet {\n\t\tif *route.DestinationCidrBlock == l.Subnet.String() &&\n\t\t\t*route.GatewayId == be.extIface.ExtAddr.String() &&\n\t\t\t*route.GatewayType == gatewayType &&\n\t\t\t*route.RouteType == routeType {\n\t\t\tif *route.Enabled {\n\t\t\t\texists = true\n\t\t\t} else {\n\t\t\t\tdelRouteRequest := vpc.NewDeleteRoutesRequest()\n\t\t\t\tdelRouteRequest.RouteTableId = routeTable.RouteTableId\n\t\t\t\tdelRouteRequest.Routes = []*vpc.Route{\n\t\t\t\t\t&vpc.Route{\n\t\t\t\t\t\tRouteId: route.RouteId,\n\t\t\t\t\t},\n\t\t\t\t}\n\n\t\t\t\t_, err := c.DeleteRoutes(delRouteRequest)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif !exists {\n\t\tcreateRouteRequest := vpc.NewCreateRoutesRequest()\n\t\tcreateRouteRequest.RouteTableId = routeTable.RouteTableId\n\t\tcreateRouteRequest.Routes = []*vpc.Route{\n\t\t\t&vpc.Route{\n\t\t\t\tDestinationCidrBlock: common.StringPtr(l.Subnet.String()),\n\t\t\t\tGatewayType:          &gatewayType,\n\t\t\t\tGatewayId:            common.StringPtr(be.extIface.ExtAddr.String()),\n\t\t\t\tEnabled:              common.BoolPtr(true),\n\t\t\t},\n\t\t}\n\n\t\t_, err := c.CreateRoutes(createRouteRequest)\n\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn &backend.SimpleNetwork{\n\t\tSubnetLease: l,\n\t\tExtIface:    be.extIface,\n\t}, nil\n}\n"
  },
  {
    "path": "pkg/backend/tencentvpc/tencentvpc_windows.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tencentvpc\n"
  },
  {
    "path": "pkg/backend/udp/cproxy_amd64.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage udp\n\n//#include \"proxy_amd64.h\"\nimport \"C\"\n\nimport (\n\t\"net\"\n\t\"os\"\n\t\"unsafe\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc runCProxy(tun *os.File, conn *net.UDPConn, ctl *os.File, tunIP ip.IP4, tunMTU int) {\n\tvar log_errors int\n\tif log.V(1).Enabled() {\n\t\tlog_errors = 1\n\t}\n\n\tc, err := conn.File()\n\tif err != nil {\n\t\tlog.Error(\"Converting UDPConn to File failed: \", err)\n\t\treturn\n\t}\n\tdefer func() {\n\t\terr := c.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close UDPConn file: %v\", err)\n\t\t}\n\t}()\n\n\tC.run_proxy(\n\t\tC.int(tun.Fd()),\n\t\tC.int(c.Fd()),\n\t\tC.int(ctl.Fd()),\n\t\tC.in_addr_t(tunIP.NetworkOrder()),\n\t\tC.size_t(tunMTU),\n\t\tC.int(log_errors),\n\t)\n}\n\nfunc writeCommand(f *os.File, cmd *C.command) {\n\tptr := (*byte)(unsafe.Pointer(cmd))\n\tsli := unsafe.Slice(ptr, unsafe.Sizeof(*cmd))\n\n\t_, err := f.Write(sli)\n\tif err != nil {\n\t\tlog.Errorf(\"Error while writing the command %v. Error: %v\", cmd, err)\n\t}\n}\n\nfunc setRoute(ctl *os.File, dst ip.IP4Net, nextHopIP ip.IP4, nextHopPort int) {\n\tcmd := C.command{\n\t\tcmd:           C.CMD_SET_ROUTE,\n\t\tdest_net:      C.in_addr_t(dst.IP.NetworkOrder()),\n\t\tdest_net_len:  C.int(dst.PrefixLen),\n\t\tnext_hop_ip:   C.in_addr_t(nextHopIP.NetworkOrder()),\n\t\tnext_hop_port: C.short(nextHopPort),\n\t}\n\n\twriteCommand(ctl, &cmd)\n}\n\nfunc removeRoute(ctl *os.File, dst ip.IP4Net) {\n\tcmd := C.command{\n\t\tcmd:          C.CMD_DEL_ROUTE,\n\t\tdest_net:     C.in_addr_t(dst.IP.NetworkOrder()),\n\t\tdest_net_len: C.int(dst.PrefixLen),\n\t}\n\n\twriteCommand(ctl, &cmd)\n}\n\nfunc stopProxy(ctl *os.File) {\n\tcmd := C.command{\n\t\tcmd: C.CMD_STOP,\n\t}\n\n\twriteCommand(ctl, &cmd)\n}\n"
  },
  {
    "path": "pkg/backend/udp/proxy_amd64.c",
    "content": "// Copyright 2015 CoreOS, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n// +build !windows\n\n#include <stdlib.h>\n#include <stdio.h>\n#include <stdarg.h>\n#include <memory.h>\n#include <assert.h>\n\n#include <errno.h>\n#include <poll.h>\n#include <unistd.h>\n#include <sys/types.h>\n#include <arpa/inet.h>\n#include <netinet/in.h>\n#include <linux/ip.h>\n#include <linux/icmp.h>\n#include <fcntl.h>\n\n#define CMD_DEFINE\n#include \"proxy_amd64.h\"\n\nstruct ip_net {\n\tin_addr_t ip;\n\tin_addr_t mask;\n};\n\nstruct route_entry {\n\tstruct ip_net      dst;\n\tstruct sockaddr_in next_hop;\n};\n\ntypedef struct icmp_pkt {\n\tstruct iphdr   iph;\n\tstruct icmphdr icmph;\n\t/* dest unreachable must include IP hdr 8 bytes of upper layer proto\n\t * of the original packet. */\n\tchar    data[sizeof(struct iphdr) + MAX_IPOPTLEN + 8];\n} __attribute__ ((aligned (4))) icmp_pkt;\n\n/* we calc hdr checksums using 32bit uints that can alias other types */\ntypedef uint32_t __attribute__((__may_alias__)) aliasing_uint32_t;\n\nstruct route_entry *routes;\nsize_t routes_alloc;\nsize_t routes_cnt;\n\nin_addr_t tun_addr;\n\nint log_enabled;\nint exit_flag;\n\nstatic inline in_addr_t netmask(int prefix_len) {\n\treturn htonl(~((uint32_t)0) << (32 - prefix_len));\n}\n\nstatic inline int contains(struct ip_net net, in_addr_t ip) {\n\treturn net.ip == (ip & net.mask);\n}\n\nstatic void log_error(const char *fmt, ...) {\n\tva_list ap;\n\n\tif( log_enabled ) {\n\t\tva_start(ap, fmt);\n\t\tvfprintf(stderr, fmt, ap);\n\t\tva_end(ap);\n\t}\n}\n\n/* fast version -- only works with mults of 4 bytes */\nstatic uint16_t cksum(aliasing_uint32_t *buf, int len) {\n\tuint32_t sum = 0;\n\tuint16_t t1, t2;\n\n\tfor( ; len > 0; len-- ) {\n\t\tuint32_t s = *buf++;\n\t\tsum += s;\n\t\tif( sum < s )\n\t\t\tsum++;\n\t}\n\n\t/* Fold down to 16 bits */\n\tt1 = sum;\n\tt2 = sum >> 16;\n\tt1 += t2;\n\tif( t1 < t2 )\n\t\tt1++;\n\n\treturn ~t1;\n}\n\nstatic void send_net_unreachable(int tun, char *offender) {\n\ticmp_pkt pkt;\n\tint off_iph_len;\n\tstruct iphdr *off_iph = (struct iphdr *)offender;\n\tsize_t pktlen, nsent;\n\n\toff_iph_len = off_iph->ihl * 4;\n\tif( off_iph_len >= sizeof(struct iphdr) + MAX_IPOPTLEN ) {\n\t\tlog_error(\"not sending net unreachable: mulformed ip pkt: iph=%d\\n\", (int)off_iph_len);\n\t\treturn; /* ip pkt mulformed */\n\t}\n\n\tif( off_iph->protocol == IPPROTO_ICMP ) {\n\t\t/* To avoid infinite loops, RFC 792 instructs not to send ICMPs\n\t\t * about ICMPs */\n\t\treturn;\n\t}\n\n\t/* Lower 3 bits (in network order) of frag_off is actually flags */\n\tif( (off_iph->frag_off & htons(0x1FFF)) != 0 ) {\n\t\t/* ICMP messages are only sent for first fragment */\n\t\treturn;\n\t}\n\n\tpktlen = sizeof(struct iphdr) + sizeof(struct icmphdr) + off_iph_len + 8;\n\n\tmemset(&pkt, 0, sizeof(pkt));\n\n\t/* Fill in the IP header */\n\tpkt.iph.ihl = sizeof(struct iphdr) / 4;\n\tpkt.iph.version = IPVERSION;\n\tpkt.iph.tot_len = htons(pktlen);\n\tpkt.iph.ttl = 8;\n\tpkt.iph.protocol = IPPROTO_ICMP;\n\tpkt.iph.saddr = tun_addr;\n\tpkt.iph.daddr = off_iph->saddr;\n\tpkt.iph.check = cksum((aliasing_uint32_t*) &pkt.iph, sizeof(struct iphdr) / sizeof(aliasing_uint32_t));\n\n\t/* Fill in the ICMP header */\n\tpkt.icmph.type = ICMP_DEST_UNREACH;\n\tpkt.icmph.code = ICMP_NET_UNREACH;\n\n\t/* Copy the offenders IP hdr + first 8 bytes of IP payload */\n\tmemcpy(pkt.data, offender, off_iph_len + 8);\n\n\t/* Compute the checksum over the ICMP header and data */\n\tpkt.icmph.checksum = cksum((aliasing_uint32_t*) &pkt.icmph,\n\t\t\t(sizeof(struct icmphdr) + off_iph_len + 8) / sizeof(aliasing_uint32_t));\n\n\t/* Kick it back */\n\tnsent = write(tun, &pkt, pktlen);\n\n\tif( nsent < 0 ) {\n\t\tlog_error(\"failed to send ICMP net unreachable: %s\\n\", strerror(errno));\n\t} else if( nsent != pktlen ) {\n\t\tlog_error(\"failed to send ICMP net unreachable: only %d out of %d byte sent\\n\", (int)nsent, (int)pktlen);\n\t}\n}\n\nstatic int set_route(struct ip_net dst, struct sockaddr_in *next_hop) {\n\tsize_t i;\n\n\tfor( i = 0; i < routes_cnt; i++ ) {\n\t\tif( dst.ip == routes[i].dst.ip && dst.mask == routes[i].dst.mask ) {\n\t\t\troutes[i].next_hop = *next_hop;\n\t\t\treturn 0;\n\t\t}\n\t}\n\n\tif( routes_alloc == routes_cnt ) {\n\t\tint new_alloc = (routes_alloc ? 2*routes_alloc : 8);\n\t\tstruct route_entry *new_routes = (struct route_entry *) realloc(routes, new_alloc*sizeof(struct route_entry));\n\t\tif( !new_routes )\n\t\t\treturn ENOMEM;\n\n\t\troutes = new_routes;\n\t\troutes_alloc = new_alloc;\n\t}\n\n\troutes[routes_cnt].dst = dst;\n\troutes[routes_cnt].next_hop = *next_hop;\n\troutes_cnt++;\n\n\treturn 0;\n}\n\nstatic int del_route(struct ip_net dst) {\n\tsize_t i;\n\n\tfor( i = 0; i < routes_cnt; i++ ) {\n\t\tif( dst.ip == routes[i].dst.ip && dst.mask == routes[i].dst.mask ) {\n\t\t\troutes[i] = routes[routes_cnt-1];\n\t\t\troutes_cnt--;\n\t\t\treturn 0;\n\t\t}\n\t}\n\n\treturn ENOENT;\n}\n\nstatic struct sockaddr_in *find_route(in_addr_t dst) {\n\tsize_t i;\n\n\tfor( i = 0; i < routes_cnt; i++ ) {\n\t\tif( contains(routes[i].dst, dst) ) {\n\t\t\t// packets for same dest tend to come in bursts. swap to front make it faster for subsequent ones\n\t\t\tif( i != 0 ) {\n\t\t\t\tstruct route_entry tmp = routes[i];\n\t\t\t\troutes[i] = routes[0];\n\t\t\t\troutes[0] = tmp;\n\t\t\t}\n\n\t\t\treturn &routes[0].next_hop;\n\t\t}\n\t}\n\n\treturn NULL;\n}\n\nstatic char *inaddr_str(in_addr_t a, char *buf, size_t len) {\n\tstruct in_addr addr;\n\taddr.s_addr = a;\n\n\tstrncpy(buf, inet_ntoa(addr), len);\n\tbuf[len-1] = '\\0';\n\n\treturn buf;\n}\n\nstatic ssize_t tun_recv_packet(int tun, char *buf, size_t buflen) {\n\tssize_t nread = read(tun, buf, buflen);\n\n\tif( nread < sizeof(struct iphdr) ) {\n\t\tif( nread < 0 ) {\n\t\t\tif( errno != EAGAIN && errno != EWOULDBLOCK )\n\t\t\t\tlog_error(\"TUN recv failed: %s\\n\", strerror(errno));\n\t\t} else {\n\t\t\tlog_error(\"TUN recv packet too small: %d bytes\\n\", (int)nread);\n\t\t}\n\t\treturn -1;\n\t}\n\n\treturn nread;\n}\n\nstatic ssize_t sock_recv_packet(int sock, char *buf, size_t buflen) {\n\tssize_t nread = recv(sock, buf, buflen, MSG_DONTWAIT);\n\n\tif( nread < sizeof(struct iphdr) ) {\n\t\tif( nread < 0 ) {\n\t\t\tif( errno != EAGAIN && errno != EWOULDBLOCK )\n\t\t\t\tlog_error(\"UDP recv failed: %s\\n\", strerror(errno));\n\t\t} else {\n\t\t\tlog_error(\"UDP recv packet too small: %d bytes\\n\", (int)nread);\n\t\t}\n\t\treturn -1;\n\t}\n\n\treturn nread;\n}\n\nstatic void sock_send_packet(int sock, char *pkt, size_t pktlen, struct sockaddr_in *dst) {\n\tssize_t nsent = sendto(sock, pkt, pktlen, 0, (struct sockaddr *)dst, sizeof(struct sockaddr_in));\n\n\tif( nsent != pktlen ) {\n\t\tif( nsent < 0 ) {\n\t\t\tlog_error(\"UDP send to %s:%hu failed: %s\\n\",\n\t\t\t\t\tinet_ntoa(dst->sin_addr), ntohs(dst->sin_port), strerror(errno));\n\t\t} else {\n\t\t\tlog_error(\"Was only able to send %d out of %d bytes to %s:%hu\\n\",\n\t\t\t\t\t(int)nsent, (int)pktlen, inet_ntoa(dst->sin_addr), ntohs(dst->sin_port));\n\t\t}\n\t}\n}\n\nstatic void tun_send_packet(int tun, char *pkt, size_t pktlen) {\n\tssize_t nsent;\n_retry:\n\tnsent = write(tun, pkt, pktlen);\n\n\tif( nsent != pktlen ) {\n\t\tif( nsent < 0 ) {\n\t\t\tif( errno == EAGAIN || errno == EWOULDBLOCK)\n\t\t\t\tgoto _retry;\n\n\t\t\tlog_error(\"TUN send failed: %s\\n\", strerror(errno));\n\t\t} else {\n\t\t\tlog_error(\"Was only able to send %d out of %d bytes to TUN\\n\", (int)nsent, (int)pktlen);\n\t\t}\n\t}\n}\n\ninline static int decrement_ttl(struct iphdr *iph) {\n\tif( --(iph->ttl) == 0 ) {\n\t\tchar saddr[32], daddr[32];\n\t\tlog_error(\"Discarding IP fragment %s -> %s due to zero TTL\\n\",\n\t\t\t\tinaddr_str(iph->saddr, saddr, sizeof(saddr)),\n\t\t\t\tinaddr_str(iph->daddr, daddr, sizeof(daddr)));\n\t\treturn 0;\n\t}\n\n\t/* patch up IP checksum (see RFC 1624) */\n\tif( iph->check >= htons(0xFFFFu - 0x100) ) {\n\t\tiph->check += htons(0x100) + 1;\n\t} else {\n\t\tiph->check += htons(0x100);\n\t}\n\n\treturn 1;\n}\n\nstatic int tun_to_udp(int tun, int sock, char *buf, size_t buflen) {\n\tstruct iphdr *iph;\n\tstruct sockaddr_in *next_hop;\n\n\tssize_t pktlen = tun_recv_packet(tun, buf, buflen);\n\tif( pktlen < 0 )\n\t\treturn 0;\n\t\n\tiph = (struct iphdr *)buf;\n\n\tnext_hop = find_route((in_addr_t) iph->daddr);\n\tif( !next_hop ) {\n\t\tsend_net_unreachable(tun, buf);\n\t\tgoto _active;\n\t}\n\n\tif( !decrement_ttl(iph) ) {\n\t\t/* TTL went to 0, discard.\n\t\t * TODO: send back ICMP Time Exceeded\n\t\t */\n\t\tgoto _active;\n\t}\n\n\tsock_send_packet(sock, buf, pktlen, next_hop);\n_active:\n\treturn 1;\n}\n\nstatic int udp_to_tun(int sock, int tun, char *buf, size_t buflen) {\n\tstruct iphdr *iph;\n\n\tssize_t pktlen = sock_recv_packet(sock, buf, buflen);\n\tif( pktlen < 0 )\n\t\treturn 0;\n\n\tiph = (struct iphdr *)buf;\n\n\tif( !decrement_ttl(iph) ) {\n\t\t/* TTL went to 0, discard.\n\t\t * TODO: send back ICMP Time Exceeded\n\t\t */\n\t\tgoto _active;\n\t}\n\n\ttun_send_packet(tun, buf, pktlen);\n_active:\n\treturn 1;\n}\n\nstatic void process_cmd(int ctl) {\n\tstruct command cmd;\n\tstruct ip_net ipn;\n\tstruct sockaddr_in sa = {\n\t\t.sin_family = AF_INET\n\t};\n\n\tssize_t nrecv = recv(ctl, (char *) &cmd, sizeof(cmd), 0);\n\tif( nrecv < 0 ) {\n\t\tlog_error(\"CTL recv failed: %s\\n\", strerror(errno));\n\t\treturn;\n\t}\n\n\tif( cmd.cmd == CMD_SET_ROUTE ) {\n\t\tipn.mask = netmask(cmd.dest_net_len);\n\t\tipn.ip = cmd.dest_net & ipn.mask;\n\n\t\tsa.sin_addr.s_addr = cmd.next_hop_ip;\n\t\tsa.sin_port = htons(cmd.next_hop_port);\n\n\t\tset_route(ipn, &sa);\n\n\t} else if( cmd.cmd == CMD_DEL_ROUTE ) {\n\t\tipn.mask = netmask(cmd.dest_net_len);\n\t\tipn.ip = cmd.dest_net & ipn.mask;\n\n\t\tdel_route(ipn);\n\n\t} else if( cmd.cmd == CMD_STOP ) {\n\t\texit_flag = 1;\n\t}\n}\n\nenum PFD {\n\tPFD_TUN = 0,\n\tPFD_SOCK,\n\tPFD_CTL,\n\tPFD_CNT\n};\n\nvoid run_proxy(int tun, int sock, int ctl, in_addr_t tun_ip, size_t tun_mtu, int log_errors) {\n\tchar *buf;\n\tstruct pollfd fds[PFD_CNT] = {\n\t\t{\n\t\t\t.fd = tun,\n\t\t\t.events = POLLIN\n\t\t},\n\t\t{\n\t\t\t.fd = sock,\n\t\t\t.events = POLLIN\n\t\t},\n\t\t{\n\t\t\t.fd = ctl,\n\t\t\t.events = POLLIN\n\t\t},\n\t};\n\n\texit_flag = 0;\n\ttun_addr = tun_ip;\n\tlog_enabled = log_errors;\n\n\tbuf = (char *) malloc(tun_mtu);\n\tif( !buf ) {\n\t\tlog_error(\"Failed to allocate %d byte buffer\\n\", tun_mtu);\n\t\texit(1);\n\t}\n\n\tfcntl(tun, F_SETFL, O_NONBLOCK);\n\n\twhile( !exit_flag ) {\n\t\tint nfds = poll(fds, PFD_CNT, -1), activity;\n\t\tif( nfds < 0 ) {\n\t\t\tif( errno == EINTR )\n\t\t\t\tcontinue;\n\n\t\t\tlog_error(\"Poll failed: %s\\n\", strerror(errno));\n\t\t\texit(1);\n\t\t}\n\n\t\tif( fds[PFD_CTL].revents & POLLIN )\n\t\t\tprocess_cmd(ctl);\n\n\t\tif( fds[PFD_TUN].revents & POLLIN || fds[PFD_SOCK].revents & POLLIN )\n\t\t\tdo {\n\t\t\t\tactivity = 0;\n\t\t\t\tactivity += tun_to_udp(tun, sock, buf, tun_mtu);\n\t\t\t\tactivity += udp_to_tun(sock, tun, buf, tun_mtu);\n\n\t\t\t\t/* As long as tun or udp is readable bypass poll().\n\t\t\t\t * We'll just occasionally get EAGAIN on an unreadable fd which\n\t\t\t\t * is cheaper than the poll() call, the rest of the time the\n\t\t\t\t * read/recvfrom call moves data which poll() never does for us.\n\t\t\t\t *\n\t\t\t\t * This is at the expense of the ctl socket, a counter could be\n\t\t\t\t * used to place an upper bound on how long we may neglect ctl.\n\t\t\t\t */\n\t\t\t} while( activity );\n\t}\n\n\tfree(buf);\n}\n\n"
  },
  {
    "path": "pkg/backend/udp/proxy_amd64.h",
    "content": "// Copyright 2015 CoreOS, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n// +build !windows\n\n#ifndef PROXY_H\n#define PROXY_H\n\n#include <netinet/in.h>\n\n#ifdef CMD_DEFINE\n#\tdefine cmdexport\n#else\n#\tdefine cmdexport static\n#endif\n\ncmdexport const int CMD_SET_ROUTE = 1;\ncmdexport const int CMD_DEL_ROUTE = 2;\ncmdexport const int CMD_STOP      = 3;\n\ntypedef struct command {\n\tint       cmd;\n\tin_addr_t dest_net;\n\tint       dest_net_len;\n\tin_addr_t next_hop_ip;\n\tshort     next_hop_port;\n} command;\n\nvoid run_proxy(int tun, int sock, int ctl, in_addr_t tun_ip, size_t tun_mtu, int log_errors);\n\n#endif\n"
  },
  {
    "path": "pkg/backend/udp/udp.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !amd64 && !windows\n// +build !amd64,!windows\n\npackage udp\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\nfunc init() {\n\tbackend.Register(\"udp\", New)\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\treturn nil, fmt.Errorf(\"UDP backend is not supported on this architecture\")\n}\n"
  },
  {
    "path": "pkg/backend/udp/udp_amd64.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage udp\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\nfunc init() {\n\tbackend.Register(\"udp\", New)\n}\n\nconst (\n\tdefaultPort = 8285\n)\n\ntype UdpBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbe := UdpBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\treturn &be, nil\n}\n\nfunc (be *UdpBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\tcfg := struct {\n\t\tPort int\n\t}{\n\t\tPort: defaultPort,\n\t}\n\n\t// Parse our configuration\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding UDP backend config: %v\", err)\n\t\t}\n\t}\n\n\t// Acquire the lease form subnet manager\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: ip.FromIP(be.extIface.ExtAddr),\n\t}\n\n\tl, err := be.sm.AcquireLease(ctx, &attrs)\n\tswitch err {\n\tcase nil:\n\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\t// Tunnel's subnet is that of the whole overlay network (e.g. /16)\n\t// and not that of the individual host (e.g. /24)\n\ttunNet := ip.IP4Net{\n\t\tIP:        l.Subnet.IP,\n\t\tPrefixLen: config.Network.PrefixLen,\n\t}\n\n\treturn newNetwork(be.sm, be.extIface, cfg.Port, tunNet, l)\n}\n"
  },
  {
    "path": "pkg/backend/udp/udp_network.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !amd64 && !windows\n// +build !amd64,!windows\n\npackage udp\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\nfunc newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface, port int, nw ip.IP4Net, l *lease.Lease) (*backend.SimpleNetwork, error) {\n\treturn nil, fmt.Errorf(\"UDP backend is not supported on this architecture\")\n}\n"
  },
  {
    "path": "pkg/backend/udp/udp_network_amd64.go",
    "content": "//go:build !windows && !windows\n// +build !windows,!windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage udp\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"sync\"\n\t\"syscall\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/vishvananda/netlink\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\tencapOverhead = 28 // 20 bytes IP hdr + 8 bytes UDP hdr\n)\n\ntype network struct {\n\tbackend.SimpleNetwork\n\tport   int\n\tctl    *os.File\n\tctl2   *os.File\n\ttun    *os.File\n\tconn   *net.UDPConn\n\ttunNet ip.IP4Net\n\tsm     subnet.Manager\n}\n\nfunc newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface, port int, nw ip.IP4Net, l *lease.Lease) (*network, error) {\n\tn := &network{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tSubnetLease: l,\n\t\t\tExtIface:    extIface,\n\t\t},\n\t\tport: port,\n\t\tsm:   sm,\n\t}\n\n\tn.tunNet = nw\n\n\tif err := n.initTun(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar err error\n\tn.conn, err = net.ListenUDP(\"udp4\", &net.UDPAddr{IP: extIface.IfaceAddr, Port: port})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to start listening on UDP socket: %v\", err)\n\t}\n\n\tn.ctl, n.ctl2, err = newCtlSockets()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create control socket: %v\", err)\n\t}\n\n\treturn n, nil\n}\n\nfunc (n *network) Run(ctx context.Context) {\n\tdefer func() {\n\t\terr := n.tun.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close tun device: %v\", err)\n\t\t}\n\t\terr = n.conn.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close UDP connection: %v\", err)\n\t\t}\n\t\terr = n.ctl.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close control socket: %v\", err)\n\t\t}\n\t\terr = n.ctl2.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close control socket: %v\", err)\n\t\t}\n\t}()\n\n\t// one for each goroutine below\n\twg := sync.WaitGroup{}\n\tdefer wg.Wait()\n\n\twg.Add(1)\n\tgo func() {\n\t\trunCProxy(n.tun, n.conn, n.ctl2, n.tunNet.IP, n.MTU())\n\t\twg.Done()\n\t}()\n\n\tlog.Info(\"Watching for new subnet leases\")\n\n\tevts := make(chan []lease.Event)\n\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, n.sm, n.SubnetLease, evts)\n\t\twg.Done()\n\t}()\n\n\tfor {\n\t\tevtBatch, ok := <-evts\n\t\tif !ok {\n\t\t\tlog.Infof(\"evts chan closed\")\n\t\t\tstopProxy(n.ctl)\n\t\t\treturn\n\t\t}\n\t\tn.processSubnetEvents(evtBatch)\n\t}\n}\n\nfunc (n *network) MTU() int {\n\treturn n.ExtIface.Iface.MTU - encapOverhead\n}\n\nfunc newCtlSockets() (*os.File, *os.File, error) {\n\tfds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_SEQPACKET, 0)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tf1 := os.NewFile(uintptr(fds[0]), \"ctl\")\n\tf2 := os.NewFile(uintptr(fds[1]), \"ctl\")\n\treturn f1, f2, nil\n}\n\nfunc (n *network) initTun() error {\n\tvar tunName string\n\tvar err error\n\n\tn.tun, tunName, err = ip.OpenTun(\"flannel%d\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to open TUN device: %v\", err)\n\t}\n\n\terr = configureIface(tunName, n.tunNet, n.MTU())\n\treturn err\n}\n\nfunc configureIface(ifname string, ipn ip.IP4Net, mtu int) error {\n\tiface, err := netlink.LinkByName(ifname)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to lookup interface %v\", ifname)\n\t}\n\n\t// Ensure that the device has a /32 address so that no broadcast routes are created.\n\t// This IP is just used as a source address for host to workload traffic (so\n\t// the return path for the traffic has an address on the flannel network to use as the destination)\n\tipnLocal := ipn\n\tipnLocal.PrefixLen = 32\n\n\terr = netlink.AddrAdd(iface, &netlink.Addr{IPNet: ipnLocal.ToIPNet(), Label: \"\"})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to add IP address %v to %v: %v\", ipnLocal.String(), ifname, err)\n\t}\n\n\terr = netlink.LinkSetMTU(iface, mtu)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to set MTU for %v: %v\", ifname, err)\n\t}\n\n\terr = netlink.LinkSetUp(iface)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to set interface %v to UP state: %v\", ifname, err)\n\t}\n\n\t// explicitly add a route since there might be a route for a subnet already\n\t// installed by Docker and then it won't get auto added\n\terr = netlink.RouteAdd(&netlink.Route{\n\t\tLinkIndex: iface.Attrs().Index,\n\t\tScope:     netlink.SCOPE_UNIVERSE,\n\t\tDst:       ipn.Network().ToIPNet(),\n\t})\n\tif err != nil && err != syscall.EEXIST {\n\t\treturn fmt.Errorf(\"failed to add route (%v -> %v): %v\", ipn.Network().String(), ifname, err)\n\t}\n\n\treturn nil\n}\n\nfunc (n *network) processSubnetEvents(batch []lease.Event) {\n\tfor _, evt := range batch {\n\t\tswitch evt.Type {\n\t\tcase lease.EventAdded:\n\t\t\tlog.Info(\"Subnet added: \", evt.Lease.Subnet)\n\n\t\t\tsetRoute(n.ctl, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, n.port)\n\n\t\tcase lease.EventRemoved:\n\t\t\tlog.Info(\"Subnet removed: \", evt.Lease.Subnet)\n\n\t\t\tremoveRoute(n.ctl, evt.Lease.Subnet)\n\n\t\tdefault:\n\t\t\tlog.Error(\"Internal error: unknown event type: \", int(evt.Type))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/udp/udp_windows.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage udp\n"
  },
  {
    "path": "pkg/backend/vxlan/device.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vxlan\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"syscall\"\n\n\t\"github.com/containernetworking/plugins/pkg/utils/sysctl\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/mac\"\n\t\"github.com/vishvananda/netlink\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype vxlanDeviceAttrs struct {\n\tvni       uint32\n\tname      string\n\tMTU       int\n\tvtepIndex int\n\tvtepAddr  net.IP\n\tvtepPort  int\n\tgbp       bool\n\tlearning  bool\n\thwAddr    net.HardwareAddr\n}\n\ntype vxlanDevice struct {\n\tlink          *netlink.Vxlan\n\tdirectRouting bool\n}\n\nfunc newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {\n\tvar err error\n\thardwareAddr := devAttrs.hwAddr\n\tif devAttrs.hwAddr == nil {\n\t\thardwareAddr, err = mac.NewHardwareAddr()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tlink := &netlink.Vxlan{\n\t\tLinkAttrs: netlink.LinkAttrs{\n\t\t\tName:         devAttrs.name,\n\t\t\tHardwareAddr: hardwareAddr,\n\t\t\tMTU:          devAttrs.MTU - 50,\n\t\t},\n\t\tVxlanId:      int(devAttrs.vni),\n\t\tVtepDevIndex: devAttrs.vtepIndex,\n\t\tSrcAddr:      devAttrs.vtepAddr,\n\t\tPort:         devAttrs.vtepPort,\n\t\tLearning:     devAttrs.learning,\n\t\tGBP:          devAttrs.gbp,\n\t}\n\n\tlink, err = ensureLink(link)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t_, _ = sysctl.Sysctl(fmt.Sprintf(\"net/ipv6/conf/%s/accept_ra\", devAttrs.name), \"0\")\n\n\treturn &vxlanDevice{\n\t\tlink: link,\n\t}, nil\n}\n\nfunc ensureLink(vxlan *netlink.Vxlan) (*netlink.Vxlan, error) {\n\terr := netlink.LinkAdd(vxlan)\n\tif err == syscall.EEXIST {\n\t\t// it's ok if the device already exists as long as config is similar\n\t\tlog.V(1).Infof(\"VXLAN device already exists\")\n\t\texisting, err := netlink.LinkByName(vxlan.Name)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tincompat := vxlanLinksIncompat(vxlan, existing)\n\t\tif incompat == \"\" {\n\t\t\tlog.V(1).Infof(\"Returning existing device\")\n\t\t\treturn existing.(*netlink.Vxlan), nil\n\t\t}\n\n\t\t// delete existing\n\t\tlog.Warningf(\"%q already exists with incompatible configuration: %v; recreating device\", vxlan.Name, incompat)\n\t\tif err = netlink.LinkDel(existing); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to delete interface: %v\", err)\n\t\t}\n\n\t\t// create new\n\t\tif err = netlink.LinkAdd(vxlan); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to create vxlan interface: %v\", err)\n\t\t}\n\t} else if err != nil {\n\t\treturn nil, err\n\t}\n\n\tifindex := vxlan.Index\n\tlink, err := netlink.LinkByIndex(vxlan.Index)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"can't locate created vxlan device with index %v\", ifindex)\n\t}\n\n\tvar ok bool\n\tif vxlan, ok = link.(*netlink.Vxlan); !ok {\n\t\treturn nil, fmt.Errorf(\"created vxlan device with index %v is not vxlan\", ifindex)\n\t}\n\n\treturn vxlan, nil\n}\n\nfunc (dev *vxlanDevice) Configure(ipa ip.IP4Net, flannelnet ip.IP4Net) error {\n\tif err := ip.EnsureV4AddressOnLink(ipa, flannelnet, dev.link); err != nil {\n\t\treturn fmt.Errorf(\"failed to ensure address of interface %s: %s\", dev.link.Attrs().Name, err)\n\t}\n\n\tif err := netlink.LinkSetUp(dev.link); err != nil {\n\t\treturn fmt.Errorf(\"failed to set interface %s to UP state: %s\", dev.link.Attrs().Name, err)\n\t}\n\n\t// ensure vxlan device hadware mac\n\t// See https://github.com/flannel-io/flannel/issues/1795\n\tnLink, err := netlink.LinkByName(dev.link.Name)\n\tif err == nil {\n\t\tif vxlan, ok := nLink.(*netlink.Vxlan); ok {\n\t\t\tif vxlan.Attrs().HardwareAddr.String() != dev.MACAddr().String() {\n\t\t\t\treturn fmt.Errorf(\"%s's mac address wanted: %s, but got: %v\", dev.link.Name, dev.MACAddr().String(), vxlan.HardwareAddr)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (dev *vxlanDevice) ConfigureIPv6(ipn ip.IP6Net, flannelnet ip.IP6Net) error {\n\tif err := ip.EnsureV6AddressOnLink(ipn, flannelnet, dev.link); err != nil {\n\t\treturn fmt.Errorf(\"failed to ensure v6 address of interface %s: %w\", dev.link.Attrs().Name, err)\n\t}\n\n\tif err := netlink.LinkSetUp(dev.link); err != nil {\n\t\treturn fmt.Errorf(\"failed to set v6 interface %s to UP state: %w\", dev.link.Attrs().Name, err)\n\t}\n\n\t// ensure vxlan device hadware mac\n\t// See https://github.com/flannel-io/flannel/issues/1795\n\tnLink, err := netlink.LinkByName(dev.link.Name)\n\tif err == nil {\n\t\tif vxlan, ok := nLink.(*netlink.Vxlan); ok {\n\t\t\tif vxlan.Attrs().HardwareAddr.String() != dev.MACAddr().String() {\n\t\t\t\treturn fmt.Errorf(\"%s's v6 mac address wanted: %s, but got: %v\", dev.link.Name, dev.MACAddr().String(), vxlan.HardwareAddr)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (dev *vxlanDevice) MACAddr() net.HardwareAddr {\n\treturn dev.link.HardwareAddr\n}\n\ntype neighbor struct {\n\tMAC net.HardwareAddr\n\tIP  ip.IP4\n\tIP6 *ip.IP6\n}\n\nfunc (dev *vxlanDevice) AddFDB(n neighbor) error {\n\tlog.V(4).Infof(\"calling AddFDB: %v, %v\", n.IP, n.MAC)\n\treturn netlink.NeighSet(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tState:        netlink.NUD_PERMANENT,\n\t\tFamily:       syscall.AF_BRIDGE,\n\t\tFlags:        netlink.NTF_SELF,\n\t\tIP:           n.IP.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) AddV6FDB(n neighbor) error {\n\tlog.V(4).Infof(\"calling AddV6FDB: %v, %v\", n.IP6, n.MAC)\n\treturn netlink.NeighSet(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tState:        netlink.NUD_PERMANENT,\n\t\tFamily:       syscall.AF_BRIDGE,\n\t\tFlags:        netlink.NTF_SELF,\n\t\tIP:           n.IP6.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) DelFDB(n neighbor) error {\n\tlog.V(4).Infof(\"calling DelFDB: %v, %v\", n.IP, n.MAC)\n\treturn netlink.NeighDel(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tFamily:       syscall.AF_BRIDGE,\n\t\tFlags:        netlink.NTF_SELF,\n\t\tIP:           n.IP.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) DelV6FDB(n neighbor) error {\n\tlog.V(4).Infof(\"calling DelV6FDB: %v, %v\", n.IP6, n.MAC)\n\treturn netlink.NeighDel(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tFamily:       syscall.AF_BRIDGE,\n\t\tFlags:        netlink.NTF_SELF,\n\t\tIP:           n.IP6.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) AddARP(n neighbor) error {\n\tlog.V(4).Infof(\"calling AddARP: %v, %v\", n.IP, n.MAC)\n\treturn netlink.NeighSet(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tState:        netlink.NUD_PERMANENT,\n\t\tType:         syscall.RTN_UNICAST,\n\t\tIP:           n.IP.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) AddV6ARP(n neighbor) error {\n\tlog.V(4).Infof(\"calling AddV6ARP: %v, %v\", n.IP6, n.MAC)\n\treturn netlink.NeighSet(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tState:        netlink.NUD_PERMANENT,\n\t\tType:         syscall.RTN_UNICAST,\n\t\tIP:           n.IP6.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) DelARP(n neighbor) error {\n\tlog.V(4).Infof(\"calling DelARP: %v, %v\", n.IP, n.MAC)\n\treturn netlink.NeighDel(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tState:        netlink.NUD_PERMANENT,\n\t\tType:         syscall.RTN_UNICAST,\n\t\tIP:           n.IP.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc (dev *vxlanDevice) DelV6ARP(n neighbor) error {\n\tlog.V(4).Infof(\"calling DelV6ARP: %v, %v\", n.IP6, n.MAC)\n\treturn netlink.NeighDel(&netlink.Neigh{\n\t\tLinkIndex:    dev.link.Index,\n\t\tState:        netlink.NUD_PERMANENT,\n\t\tType:         syscall.RTN_UNICAST,\n\t\tIP:           n.IP6.ToIP(),\n\t\tHardwareAddr: n.MAC,\n\t})\n}\n\nfunc vxlanLinksIncompat(l1, l2 netlink.Link) string {\n\tif l1.Type() != l2.Type() {\n\t\treturn fmt.Sprintf(\"link type: %v vs %v\", l1.Type(), l2.Type())\n\t}\n\n\tv1 := l1.(*netlink.Vxlan)\n\tv2 := l2.(*netlink.Vxlan)\n\n\tif v1.VxlanId != v2.VxlanId {\n\t\treturn fmt.Sprintf(\"vni: %v vs %v\", v1.VxlanId, v2.VxlanId)\n\t}\n\n\tif v1.VtepDevIndex > 0 && v2.VtepDevIndex > 0 && v1.VtepDevIndex != v2.VtepDevIndex {\n\t\treturn fmt.Sprintf(\"vtep (external) interface: %v vs %v\", v1.VtepDevIndex, v2.VtepDevIndex)\n\t}\n\n\tif len(v1.SrcAddr) > 0 && len(v2.SrcAddr) > 0 && !v1.SrcAddr.Equal(v2.SrcAddr) {\n\t\treturn fmt.Sprintf(\"vtep (external) IP: %v vs %v\", v1.SrcAddr, v2.SrcAddr)\n\t}\n\n\tif len(v1.Group) > 0 && len(v2.Group) > 0 && !v1.Group.Equal(v2.Group) {\n\t\treturn fmt.Sprintf(\"group address: %v vs %v\", v1.Group, v2.Group)\n\t}\n\n\tif v1.L2miss != v2.L2miss {\n\t\treturn fmt.Sprintf(\"l2miss: %v vs %v\", v1.L2miss, v2.L2miss)\n\t}\n\n\tif v1.Port > 0 && v2.Port > 0 && v1.Port != v2.Port {\n\t\treturn fmt.Sprintf(\"port: %v vs %v\", v1.Port, v2.Port)\n\t}\n\n\tif v1.GBP != v2.GBP {\n\t\treturn fmt.Sprintf(\"gbp: %v vs %v\", v1.GBP, v2.GBP)\n\t}\n\n\treturn \"\"\n}\n"
  },
  {
    "path": "pkg/backend/vxlan/device_windows.go",
    "content": "// Copyright 2018 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build windows\n// +build windows\n\npackage vxlan\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/Microsoft/hcsshim/hcn\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/pkg/errors\"\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype vxlanDeviceAttrs struct {\n\tvni           uint32\n\tname          string\n\tgbp           bool\n\taddressPrefix ip.IP4Net\n\tinterfaceName string\n}\n\ntype vxlanDevice struct {\n\tlink          *hcn.HostComputeNetwork\n\tmacPrefix     string\n\tdirectRouting bool\n}\n\ntype NetAdapterNameSettings struct {\n\tNetworkAdapterName string `json:\"NetworkAdapterName\"`\n}\n\nfunc newVXLANDevice(ctx context.Context, devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {\n\tsubnet := createSubnet(devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String(), \"0.0.0.0/0\")\n\tnetwork := &hcn.HostComputeNetwork{\n\t\tType: \"Overlay\",\n\t\tName: devAttrs.name,\n\t\tIpams: []hcn.Ipam{\n\t\t\t{\n\t\t\t\tType: \"Static\",\n\t\t\t\tSubnets: []hcn.Subnet{\n\t\t\t\t\t*subnet,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tFlags: hcn.EnableNonPersistent,\n\t\tSchemaVersion: hcn.SchemaVersion{\n\t\t\tMajor: 2,\n\t\t\tMinor: 0,\n\t\t},\n\t}\n\n\tvsid := &hcn.VsidPolicySetting{\n\t\tIsolationId: devAttrs.vni,\n\t}\n\tvsidJson, err := json.Marshal(vsid)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tsp := &hcn.SubnetPolicy{\n\t\tType: hcn.VSID,\n\t}\n\tsp.Settings = vsidJson\n\n\tspJson, err := json.Marshal(sp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tnetwork.Ipams[0].Subnets[0].Policies = append(network.Ipams[0].Subnets[0].Policies, spJson)\n\n\tif devAttrs.interfaceName != \"\" {\n\t\taddNetAdapterName(network, devAttrs.interfaceName)\n\t}\n\n\thnsNetwork, err := ensureNetwork(ctx, network, devAttrs.addressPrefix.String())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &vxlanDevice{\n\t\tlink: hnsNetwork,\n\t}, nil\n}\n\nfunc ensureNetwork(ctx context.Context, expectedNetwork *hcn.HostComputeNetwork, expectedAddressPrefix string) (*hcn.HostComputeNetwork, error) {\n\tcreateNetwork := true\n\tnetworkName := expectedNetwork.Name\n\n\t// 1. Check if the HostComputeNetwork exists and has the expected settings\n\texistingNetwork, err := hcn.GetNetworkByName(networkName)\n\tif err == nil {\n\t\tif existingNetwork.Type == expectedNetwork.Type {\n\t\t\tif existingNetwork.Ipams[0].Subnets[0].IpAddressPrefix == expectedAddressPrefix {\n\t\t\t\tcreateNetwork = false\n\t\t\t\tlog.Infof(\"Found existing HostComputeNetwork %s\", networkName)\n\t\t\t}\n\t\t}\n\t}\n\n\t// 2. Create a new HNSNetwork\n\tif createNetwork {\n\t\tif existingNetwork != nil {\n\t\t\tif err := existingNetwork.Delete(); err != nil {\n\t\t\t\treturn nil, errors.Wrapf(err, \"failed to delete existing HostComputeNetwork %s\", networkName)\n\t\t\t}\n\t\t\tlog.Infof(\"Deleted stale HostComputeNetwork %s\", networkName)\n\t\t}\n\n\t\tlog.Infof(\"Attempting to create HostComputeNetwork %v\", expectedNetwork)\n\t\tnewNetwork, err := expectedNetwork.Create()\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to create HostComputeNetwork %s\", networkName)\n\t\t}\n\n\t\tvar waitErr, lastErr error\n\t\t// Wait for the network to populate Management IP\n\t\tlog.Infof(\"Waiting to get ManagementIP from HostComputeNetwork %s\", networkName)\n\t\tvar newNetworkID = newNetwork.Id\n\t\twaitErr = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 5*time.Second, true, func(context.Context) (done bool, err error) {\n\t\t\tnewNetwork, lastErr = hcn.GetNetworkByID(newNetworkID)\n\t\t\treturn newNetwork != nil && len(getManagementIP(newNetwork)) != 0, nil\n\t\t})\n\t\tif waitErr != nil {\n\t\t\t// Do not swallow the root cause\n\t\t\tif lastErr != nil {\n\t\t\t\twaitErr = lastErr\n\t\t\t}\n\t\t\treturn nil, errors.Wrapf(lastErr, \"timeout, failed to get management IP from HostComputeNetwork %s\", networkName)\n\t\t}\n\n\t\terr = checkHostNetworkReady(ctx, newNetwork)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"Interface bound to %s took too long to get ready. Please check your network host configuration\", networkName)\n\t\t}\n\n\t\tlog.Infof(\"Created HostComputeNetwork %s\", networkName)\n\t\texistingNetwork = newNetwork\n\t}\n\n\taddHostRoute := true\n\tfor _, policy := range existingNetwork.Policies {\n\t\tif policy.Type == hcn.HostRoute {\n\t\t\taddHostRoute = false\n\t\t}\n\t}\n\tif addHostRoute {\n\t\thostRoutePolicy := hcn.NetworkPolicy{\n\t\t\tType:     hcn.HostRoute,\n\t\t\tSettings: []byte(\"{}\"),\n\t\t}\n\n\t\tnetworkRequest := hcn.PolicyNetworkRequest{\n\t\t\tPolicies: []hcn.NetworkPolicy{hostRoutePolicy},\n\t\t}\n\t\terr = existingNetwork.AddPolicy(networkRequest)\n\t\tif err != nil {\n\t\t\tlog.Infof(\"Could not apply HostRoute policy for local host to local pod connectivity. This policy requires Windows 18321.1000.19h1_release.190117-1502 or newer\")\n\t\t}\n\t}\n\n\treturn existingNetwork, nil\n}\n\nfunc getManagementIP(network *hcn.HostComputeNetwork) string {\n\tfor _, policy := range network.Policies {\n\t\tif policy.Type == hcn.ProviderAddress {\n\t\t\tpolicySettings := hcn.ProviderAddressEndpointPolicySetting{}\n\t\t\terr := json.Unmarshal(policy.Settings, &policySettings)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\"\n\t\t\t}\n\t\t\treturn policySettings.ProviderAddress\n\t\t}\n\t}\n\treturn \"\"\n}\n\nfunc createSubnet(AddressPrefix string, NextHop string, DestPrefix string) *hcn.Subnet {\n\treturn &hcn.Subnet{\n\t\tIpAddressPrefix: AddressPrefix,\n\t\tRoutes: []hcn.Route{\n\t\t\t{\n\t\t\t\tNextHop:           NextHop,\n\t\t\t\tDestinationPrefix: DestPrefix,\n\t\t\t},\n\t\t},\n\t}\n}\n\n// addNetAdapterName adds a policy to the network to set the name of the network adapter\nfunc addNetAdapterName(network *hcn.HostComputeNetwork, netAdapterName string) error {\n\tsettings := NetAdapterNameSettings{\n\t\tNetworkAdapterName: netAdapterName,\n\t}\n\n\tsettingsJson, err := json.Marshal(settings)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"Failed to marshal settings: %w\", err)\n\t}\n\n\tpolicySettings := hcn.NetworkPolicy{\n\t\tType:     hcn.NetAdapterName,\n\t\tSettings: settingsJson,\n\t}\n\n\tnetwork.Policies = append(network.Policies, policySettings)\n\n\treturn nil\n}\n\n// checkHostNetworkReady waits for the host network to be ready: the main interface must be up and have an IP address\nfunc checkHostNetworkReady(ctx context.Context, network *hcn.HostComputeNetwork) error {\n\tmanagementIP := getManagementIP(network)\n\t// Wait for the interface with the management IP\n\tlog.Infof(\"Waiting to get net interface for HostComputeNetwork %s (%s)\", network.Name, managementIP)\n\tmanagementIPv4, err := ip.ParseIP4(managementIP)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"Failed to parse management ip (%s)\", managementIP)\n\t}\n\n\twaitErr := wait.PollUntilContextTimeout(ctx, 5*time.Second, 45*time.Second, true, func(context.Context) (done bool, err error) {\n\t\tiface, lastErr := ip.GetInterfaceByIP(managementIPv4.ToIP())\n\t\tif lastErr == nil {\n\t\t\tlog.Infof(\"Host interface: %s bound by %s ready\", iface.Name, network.Name)\n\t\t\treturn true, nil\n\t\t}\n\t\tlog.V(2).Infof(\"Host interface bound by %s not ready\", network.Name)\n\t\treturn false, nil\n\t})\n\tif waitErr != nil {\n\t\treturn errors.Wrapf(waitErr, \"timeout, failed to get net interface for HostComputeNetwork %s (%s)\", network.Name, managementIP)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/backend/vxlan/vxlan.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage vxlan\n\n// Some design notes and history:\n// VXLAN encapsulates L2 packets (though flannel is L3 only so don't expect to be able to send L2 packets across hosts)\n// The first versions of vxlan for flannel registered the flannel daemon as a handler for both \"L2\" and \"L3\" misses\n// - When a container sends a packet to a new IP address on the flannel network (but on a different host) this generates\n//   an L2 miss (i.e. an ARP lookup)\n// - The flannel daemon knows which flannel host the packet is destined for so it can supply the VTEP MAC to use.\n//   This is stored in the ARP table (with a timeout) to avoid constantly looking it up.\n// - The packet can then be encapsulated but the host needs to know where to send it. This creates another callout from\n//   the kernel vxlan code to the flannel daemon to get the public IP that should be used for that VTEP (this gets called\n//   an L3 miss). The L2/L3 miss hooks are registered when the vxlan device is created. At the same time a device route\n//   is created to the whole flannel network so that non-local traffic is sent over the vxlan device.\n//\n// In this scheme the scaling of table entries (per host) is:\n//  - 1 route (for the configured network out the vxlan device)\n//  - One arp entry for each remote container that this host has recently contacted\n//  - One FDB entry for each remote host\n//\n// The second version of flannel vxlan removed the need for the L3MISS callout. When a new remote host is found (either\n// during startup or when it's created), flannel simply adds the required entries so that no further lookup/callout is required.\n//\n//\n// The latest version of the vxlan backend  removes the need for the L2MISS too, which means that the flannel daemon is not\n// listening for any netlink messages anymore. This improves reliability (no problems with timeouts if\n// flannel crashes or restarts) and simplifies upgrades.\n//\n// How it works:\n// Create the vxlan device but don't register for any L2MISS or L3MISS messages\n// Then, as each remote host is discovered (either on startup or when they are added), do the following\n// 1) Create routing table entry for the remote subnet. It goes via the vxlan device but also specifies a next hop (of the remote flannel host).\n// 2) Create a static ARP entry for the remote flannel host IP address (and the VTEP MAC)\n// 3) Create an FDB entry with the VTEP MAC and the public IP of the remote flannel daemon.\n//\n// In this scheme the scaling of table entries is linear to the number of remote hosts - 1 route, 1 arp entry and 1 FDB entry per host\n//\n// In this newest scheme, there is also the option of skipping the use of vxlan for hosts that are on the same subnet,\n// this is called \"directRouting\"\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc init() {\n\tbackend.Register(\"vxlan\", New)\n}\n\nconst (\n\tdefaultVNI = 1\n)\n\ntype VXLANBackend struct {\n\tsubnetMgr subnet.Manager\n\textIface  *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbackend := &VXLANBackend{\n\t\tsubnetMgr: sm,\n\t\textIface:  extIface,\n\t}\n\n\treturn backend, nil\n}\n\nfunc newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, vnid uint32, dev, v6Dev *vxlanDevice) (*lease.LeaseAttrs, error) {\n\tleaseAttrs := &lease.LeaseAttrs{\n\t\tBackendType: \"vxlan\",\n\t}\n\tif publicIP != nil && dev != nil {\n\t\tdata, err := json.Marshal(&vxlanLeaseAttrs{\n\t\t\tVNI:     vnid,\n\t\t\tVtepMAC: hardwareAddr(dev.MACAddr()),\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tleaseAttrs.PublicIP = ip.FromIP(publicIP)\n\t\tleaseAttrs.BackendData = json.RawMessage(data)\n\t}\n\n\tif publicIPv6 != nil && v6Dev != nil {\n\t\tdata, err := json.Marshal(&vxlanLeaseAttrs{\n\t\t\tVNI:     vnid,\n\t\t\tVtepMAC: hardwareAddr(v6Dev.MACAddr()),\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tleaseAttrs.PublicIPv6 = ip.FromIP6(publicIPv6)\n\t\tleaseAttrs.BackendV6Data = json.RawMessage(data)\n\t}\n\treturn leaseAttrs, nil\n}\n\nfunc (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\t// Parse our configuration\n\tcfg, err := parseVXLANConfig(config.Backend, be.extIface.Iface.MTU)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error decoding VXLAN backend config: %w\", err)\n\t}\n\tlog.Infof(\"VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v\", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)\n\n\tdev, v6Dev, err := createVXLANDevice(ctx, config, cfg, be.subnetMgr, be.extIface.Iface.Index, be.extIface.ExtAddr, be.extIface.ExtV6Addr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create vxlan device: %w\", err)\n\t}\n\n\tsubnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, uint32(cfg.VNI), dev, v6Dev)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tlease, err := be.subnetMgr.AcquireLease(ctx, subnetAttrs)\n\tswitch err {\n\tcase nil:\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\t// Ensure that the device has a /32 address so that no broadcast routes are created.\n\t// This IP is just used as a source address for host to workload traffic (so\n\t// the return path for the traffic has an address on the flannel network to use as the destination)\n\tif err := configureDeviceIPv4IPv6(dev, v6Dev, lease, config); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease, cfg.MTU)\n}\n\ntype VXLANConfig struct {\n\tVNI           int  `json:\"vni\"`\n\tPort          int  `json:\"port\"`\n\tMTU           int  `json:\"mtu\"`\n\tGBP           bool `json:\"gbp\"`\n\tLearning      bool `json:\"learning\"`\n\tDirectRouting bool `json:\"directRouting\"`\n}\n\nfunc parseVXLANConfig(config json.RawMessage, defaultMTU int) (VXLANConfig, error) {\n\tcfg := VXLANConfig{\n\t\tVNI: defaultVNI,\n\t\tMTU: defaultMTU,\n\t}\n\n\tif len(config) > 0 {\n\t\tif err := json.Unmarshal(config, &cfg); err != nil {\n\t\t\treturn VXLANConfig{}, err\n\t\t}\n\t}\n\treturn cfg, nil\n}\n\nfunc createVXLANDevice(ctx context.Context,\n\tconfig *subnet.Config,\n\tcfg VXLANConfig,\n\tsubnetMgr subnet.Manager,\n\textIfaceID int,\n\textIfaceIP net.IP,\n\textIfaceV6IP net.IP,\n) (dev, v6Dev *vxlanDevice, err error) {\n\t// When flannel is restarted, it will get the MAC address from the node annotations to set flannel.1 MAC address\n\tvar hwAddr, hwAddrv6 net.HardwareAddr\n\n\tmacStr, macStrv6 := subnetMgr.GetStoredMacAddresses(ctx)\n\tif macStr != \"\" {\n\t\thwAddr, err = net.ParseMAC(macStr)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to parse mac addr(%s): %v\", macStr, err)\n\t\t}\n\t\tlog.Infof(\"Interface flannel.%d mac address set to: %s\", cfg.VNI, macStr)\n\t}\n\n\tif config.EnableIPv4 {\n\t\tdevAttrs := vxlanDeviceAttrs{\n\t\t\tvni:       uint32(cfg.VNI),\n\t\t\tname:      fmt.Sprintf(\"flannel.%d\", cfg.VNI),\n\t\t\tMTU:       cfg.MTU,\n\t\t\tvtepIndex: extIfaceID,\n\t\t\tvtepAddr:  extIfaceIP,\n\t\t\tvtepPort:  cfg.Port,\n\t\t\tgbp:       cfg.GBP,\n\t\t\tlearning:  cfg.Learning,\n\t\t\thwAddr:    hwAddr,\n\t\t}\n\n\t\tdev, err = newVXLANDevice(&devAttrs)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tdev.directRouting = cfg.DirectRouting\n\t}\n\n\tif macStrv6 != \"\" {\n\t\thwAddrv6, err = net.ParseMAC(macStrv6)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to parse mac addr(%s): %v\", macStrv6, err)\n\t\t}\n\t\tlog.Infof(\"Interface flannel-v6.%d mac address set to: %s\", cfg.VNI, macStrv6)\n\t}\n\n\tif config.EnableIPv6 {\n\t\tv6DevAttrs := vxlanDeviceAttrs{\n\t\t\tvni:       uint32(cfg.VNI),\n\t\t\tname:      fmt.Sprintf(\"flannel-v6.%d\", cfg.VNI),\n\t\t\tMTU:       cfg.MTU,\n\t\t\tvtepIndex: extIfaceID,\n\t\t\tvtepAddr:  extIfaceV6IP,\n\t\t\tvtepPort:  cfg.Port,\n\t\t\tgbp:       cfg.GBP,\n\t\t\tlearning:  cfg.Learning,\n\t\t\thwAddr:    hwAddrv6,\n\t\t}\n\t\tv6Dev, err = newVXLANDevice(&v6DevAttrs)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tv6Dev.directRouting = cfg.DirectRouting\n\t}\n\n\treturn dev, v6Dev, nil\n}\n\nfunc configureDeviceIPv4IPv6(dev *vxlanDevice, v6Dev *vxlanDevice, lease *lease.Lease, config *subnet.Config) error {\n\t// Configure IPv4 if enabled\n\tif config.EnableIPv4 {\n\t\tif lease.Subnet.Empty() {\n\t\t\treturn fmt.Errorf(\"failed to configure interface %s: IPv4 is enabled but the lease has no IPv4\", dev.link.Attrs().Name)\n\t\t}\n\t\tif err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to configure interface %s: %w\", dev.link.Attrs().Name, err)\n\t\t}\n\t}\n\n\t// Configure IPv6 if enabled\n\tif config.EnableIPv6 {\n\t\tif lease.IPv6Subnet.Empty() {\n\t\t\treturn fmt.Errorf(\"failed to configure interface %s: IPv6 is enabled but the lease has no IPv6\", v6Dev.link.Attrs().Name)\n\t\t}\n\t\tif err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}, config.IPv6Network); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to configure interface %s: %w\", v6Dev.link.Attrs().Name, err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// So we can make it JSON (un)marshalable\ntype hardwareAddr net.HardwareAddr\n\nfunc (hw hardwareAddr) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(\"%q\", net.HardwareAddr(hw))), nil\n}\n\nfunc (hw *hardwareAddr) UnmarshalJSON(bytes []byte) error {\n\tif len(bytes) < 2 || bytes[0] != '\"' || bytes[len(bytes)-1] != '\"' {\n\t\treturn fmt.Errorf(\"error parsing hardware addr\")\n\t}\n\n\tbytes = bytes[1 : len(bytes)-1]\n\n\tmac, err := net.ParseMAC(string(bytes))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t*hw = hardwareAddr(mac)\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/backend/vxlan/vxlan_network.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage vxlan\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/retry\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"github.com/vishvananda/netlink\"\n\t\"golang.org/x/sys/unix\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype network struct {\n\tbackend.SimpleNetwork\n\tdev       *vxlanDevice\n\tv6Dev     *vxlanDevice\n\tsubnetMgr subnet.Manager\n\tmtu       int\n}\n\nconst (\n\tencapOverhead = 50\n)\n\nfunc newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, v6Dev *vxlanDevice, _ ip.IP4Net, lease *lease.Lease, mtu int) (*network, error) {\n\tnw := &network{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tSubnetLease: lease,\n\t\t\tExtIface:    extIface,\n\t\t},\n\t\tsubnetMgr: subnetMgr,\n\t\tdev:       dev,\n\t\tv6Dev:     v6Dev,\n\t\tmtu:       mtu,\n\t}\n\n\treturn nw, nil\n}\n\nfunc (nw *network) Run(ctx context.Context) {\n\tvar wg sync.WaitGroup\n\n\tlog.V(0).Info(\"watching for new subnet leases\")\n\tleaseEvents := make(chan []lease.Event)\n\tvxlanMissingChan := make(chan bool, 1) // buffered to avoid blocking\n\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, nw.subnetMgr, nw.SubnetLease, leaseEvents)\n\t\tlog.V(1).Info(\"WatchLeases exited\")\n\t\twg.Done()\n\t}()\n\n\twg.Add(1)\n\tgo func() {\n\t\tnw.watchVXLANDevice(ctx, vxlanMissingChan)\n\t\tlog.V(1).Info(\"WatchVXLANDevice exited\")\n\t\twg.Done()\n\t}()\n\n\tdefer wg.Wait()\n\n\tfor {\n\t\tselect {\n\t\tcase evtBatch, ok := <-leaseEvents:\n\t\t\tif !ok {\n\t\t\t\tlog.Infof(\"leaseEvents chan closed\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnw.handleSubnetEvents(evtBatch)\n\n\t\tcase _, ok := <-vxlanMissingChan:\n\t\t\tif !ok {\n\t\t\t\tlog.Infof(\"vxlanMissingChan closed\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tlog.Info(\"vxlan device missing, attempting to recreate...\")\n\n\t\t\t// Offload recreate so this loop doesn’t block handleSubnetEvents\n\t\t\tgo func() {\n\t\t\t\tif err := nw.reCreateVxlan(ctx); err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to recreate vxlan: %v\", err)\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}\n}\n\nfunc (nw *network) watchVXLANDevice(ctx context.Context, vxlanMissingChan chan<- bool) {\n\tlog.Info(\"starting vxlan device watcher\")\n\tif nw.dev == nil {\n\t\tlog.Error(\"vxlan device is nil, cannot watch for events\")\n\t\treturn\n\t}\n\n\tupdates := make(chan netlink.LinkUpdate)\n\tdone := make(chan struct{})\n\n\tif err := netlink.LinkSubscribe(updates, done); err != nil {\n\t\tlog.Fatalf(\"failed to subscribe to netlink: %v\", err)\n\t}\n\tdefer close(done)\n\n\tname := nw.dev.link.Attrs().Name\n\tdefer close(vxlanMissingChan)\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\tlog.Info(\"stopping vxlan device watcher\")\n\t\t\treturn\n\n\t\tcase update := <-updates:\n\t\t\tif update.Attrs() == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// Detect deletion\n\t\t\tif update.Attrs().Name == name && update.Header.Type == unix.RTM_DELLINK {\n\t\t\t\tlog.Infof(\"Interface %s deleted\", name)\n\t\t\t\tselect {\n\t\t\t\tcase vxlanMissingChan <- true:\n\t\t\t\tdefault:\n\t\t\t\t\t// Skip if signal already queued\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (nw *network) reCreateVxlan(ctx context.Context) error {\n\tbackoff := time.Second\n\tmaxBackoff := 30 * time.Second\n\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn fmt.Errorf(\"context canceled, stopping vxlan recreate\")\n\t\tdefault:\n\t\t}\n\n\t\textIface, _ := net.InterfaceByName(nw.ExtIface.IfaceName)\n\t\tif extIface == nil {\n\t\t\tlog.Infof(\"external interface %s not found, retrying in %s\", nw.ExtIface.IfaceName, backoff)\n\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\tcontinue\n\t\t}\n\n\t\tconfig, err := nw.subnetMgr.GetNetworkConfig(ctx)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"failed to get network config: %v\", err)\n\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\tcontinue\n\t\t}\n\n\t\tcfg, err := parseVXLANConfig(config.Backend, extIface.MTU)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"failed to parse vxlan config: %v\", err)\n\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\tcontinue\n\t\t}\n\n\t\tvar ifaceAddrs, ifaceAddrsV6 []net.IP\n\n\t\tif config.EnableIPv4 {\n\t\t\tifaceAddrs, err = ip.GetInterfaceIP4Addrs(extIface)\n\t\t\tif err != nil {\n\t\t\t\tlog.Errorf(\"error getting IPv4 addresses for %s: %v\", extIface.Name, err)\n\t\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(ifaceAddrs) == 0 {\n\t\t\t\tlog.Warningf(\"no IPv4 addresses found for interface %s, retrying\", extIface.Name)\n\t\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif config.EnableIPv6 {\n\t\t\tifaceAddrsV6, err = ip.GetInterfaceIP6Addrs(extIface)\n\t\t\tif err != nil {\n\t\t\t\tlog.Errorf(\"error getting IPv6 addresses for %s: %v\", extIface.Name, err)\n\t\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(ifaceAddrsV6) == 0 {\n\t\t\t\tlog.Warningf(\"no IPv6 addresses found for interface %s, retrying\", extIface.Name)\n\t\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// Create the VXLAN device\n\t\tdev, v6Dev, err := createVXLANDevice(ctx, config, cfg, nw.subnetMgr, extIface.Index, ifaceAddrs[0], ifaceAddrsV6[0])\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"failed to create vxlan device: %v\", err)\n\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\tcontinue\n\t\t}\n\n\t\tif err := configureDeviceIPv4IPv6(dev, v6Dev, nw.SubnetLease, config); err != nil {\n\t\t\tlog.Errorf(\"failed to configure vxlan device: %v\", err)\n\t\t\tretryAfterBackoff(&backoff, maxBackoff)\n\t\t\tcontinue\n\t\t}\n\n\t\tnw.dev = dev\n\t\tnw.v6Dev = v6Dev\n\t\tnw.mtu = dev.link.Attrs().MTU\n\t\tlog.Infof(\"VXLAN device %s recreated successfully\", dev.link.Attrs().Name)\n\t\treturn nil\n\t}\n}\n\nfunc retryAfterBackoff(backoff *time.Duration, maxBackoff time.Duration) {\n\ttime.Sleep(*backoff)\n\t*backoff = minDuration(*backoff*2, maxBackoff)\n}\n\n// helper to cap exponential backoff\nfunc minDuration(a, b time.Duration) time.Duration {\n\tif a < b {\n\t\treturn a\n\t}\n\treturn b\n}\n\nfunc (nw *network) MTU() int {\n\treturn nw.mtu - encapOverhead\n}\n\ntype vxlanLeaseAttrs struct {\n\tVNI     uint32\n\tVtepMAC hardwareAddr\n}\n\nfunc (nw *network) handleSubnetEvents(batch []lease.Event) {\n\tfor _, event := range batch {\n\t\tsn := event.Lease.Subnet\n\t\tv6Sn := event.Lease.IPv6Subnet\n\t\tattrs := event.Lease.Attrs\n\t\tlog.Infof(\"Received Subnet Event with VxLan: %s\", attrs.String())\n\t\tif attrs.BackendType != \"vxlan\" {\n\t\t\tlog.Warningf(\"ignoring non-vxlan v4Subnet(%s) v6Subnet(%s): type=%v\", sn, v6Sn, attrs.BackendType)\n\t\t\tcontinue\n\t\t}\n\n\t\tvar (\n\t\t\tvxlanAttrs, v6VxlanAttrs           vxlanLeaseAttrs\n\t\t\tdirectRoutingOK, v6DirectRoutingOK bool\n\t\t\tdirectRoute, v6DirectRoute         netlink.Route\n\t\t\tvxlanRoute, v6VxlanRoute           netlink.Route\n\t\t)\n\n\t\tif event.Lease.EnableIPv4 && nw.dev != nil {\n\t\t\tif err := json.Unmarshal(attrs.BackendData, &vxlanAttrs); err != nil {\n\t\t\t\tlog.Error(\"error decoding subnet lease JSON: \", err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// This route is used when traffic should be vxlan encapsulated\n\t\t\tvxlanRoute = netlink.Route{\n\t\t\t\tLinkIndex: nw.dev.link.Attrs().Index,\n\t\t\t\tScope:     netlink.SCOPE_UNIVERSE,\n\t\t\t\tDst:       sn.ToIPNet(),\n\t\t\t\tGw:        sn.IP.ToIP(),\n\t\t\t}\n\t\t\tvxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)\n\n\t\t\t// directRouting is where the remote host is on the same subnet so vxlan isn't required.\n\t\t\tdirectRoute = netlink.Route{\n\t\t\t\tDst: sn.ToIPNet(),\n\t\t\t\tGw:  attrs.PublicIP.ToIP(),\n\t\t\t}\n\t\t\tif nw.dev.directRouting {\n\t\t\t\tif dr, err := ip.DirectRouting(attrs.PublicIP.ToIP()); err != nil {\n\t\t\t\t\tlog.Error(err)\n\t\t\t\t} else {\n\t\t\t\t\tdirectRoutingOK = dr\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif event.Lease.EnableIPv6 && nw.v6Dev != nil {\n\t\t\tif err := json.Unmarshal(attrs.BackendV6Data, &v6VxlanAttrs); err != nil {\n\t\t\t\tlog.Error(\"error decoding v6 subnet lease JSON: \", err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif v6Sn.IP != nil && nw.v6Dev != nil {\n\t\t\t\tv6VxlanRoute = netlink.Route{\n\t\t\t\t\tLinkIndex: nw.v6Dev.link.Attrs().Index,\n\t\t\t\t\tScope:     netlink.SCOPE_UNIVERSE,\n\t\t\t\t\tDst:       v6Sn.ToIPNet(),\n\t\t\t\t\tGw:        v6Sn.IP.ToIP(),\n\t\t\t\t}\n\t\t\t\tv6VxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)\n\n\t\t\t\t// directRouting is where the remote host is on the same subnet so vxlan isn't required.\n\t\t\t\tv6DirectRoute = netlink.Route{\n\t\t\t\t\tDst: v6Sn.ToIPNet(),\n\t\t\t\t\tGw:  attrs.PublicIPv6.ToIP(),\n\t\t\t\t}\n\n\t\t\t\tif nw.v6Dev.directRouting {\n\t\t\t\t\tif v6Dr, err := ip.DirectRouting(attrs.PublicIPv6.ToIP()); err != nil {\n\t\t\t\t\t\tlog.Error(err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tv6DirectRoutingOK = v6Dr\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tswitch event.Type {\n\t\tcase lease.EventAdded:\n\t\t\tif event.Lease.EnableIPv4 {\n\t\t\t\tif directRoutingOK {\n\t\t\t\t\tlog.V(2).Infof(\"Adding direct route to subnet: %s PublicIP: %s\", sn, attrs.PublicIP)\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteReplace(&directRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"Error adding route to %v via %v: %v\", sn, attrs.PublicIP, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tlog.V(2).Infof(\"adding subnet: %s PublicIP: %s VtepMAC: %s\", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.dev.AddARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"AddARP failed: \", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.dev.AddFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"AddFDB failed: \", err)\n\n\t\t\t\t\t\t// Try to clean up the ARP entry then continue\n\t\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\t\treturn nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)})\n\t\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\t\tlog.Error(\"DelARP failed: \", err)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\n\t\t\t\t\t// Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure\n\t\t\t\t\t// this is done last.\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteReplace(&vxlanRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to add vxlanRoute (%s -> %s): %v\", vxlanRoute.Dst, vxlanRoute.Gw, err)\n\n\t\t\t\t\t\t// Try to clean up both the ARP and FDB entries then continue\n\t\t\t\t\t\tif err := nw.dev.DelARP(neighbor{IP: event.Lease.Subnet.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {\n\t\t\t\t\t\t\tlog.Error(\"DelARP failed: \", err)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif err := nw.dev.DelFDB(neighbor{IP: event.Lease.Attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)}); err != nil {\n\t\t\t\t\t\t\tlog.Error(\"DelFDB failed: \", err)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif event.Lease.EnableIPv6 {\n\t\t\t\tif v6DirectRoutingOK {\n\t\t\t\t\tlog.V(2).Infof(\"Adding v6 direct route to v6 subnet: %s PublicIPv6: %s\", v6Sn, attrs.PublicIPv6)\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteReplace(&v6DirectRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"Error adding v6 route to %v via %v: %v\", v6Sn, attrs.PublicIPv6, err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tlog.V(2).Infof(\"adding v6 subnet: %s PublicIPv6: %s VtepMAC: %s\", v6Sn, attrs.PublicIPv6, net.HardwareAddr(v6VxlanAttrs.VtepMAC))\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.v6Dev.AddV6ARP(neighbor{IP6: v6Sn.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"AddV6ARP failed: \", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.v6Dev.AddV6FDB(neighbor{IP6: attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"AddV6FDB failed: \", err)\n\n\t\t\t\t\t\t// Try to clean up the ARP entry then continue\n\t\t\t\t\t\tif err := nw.v6Dev.DelV6ARP(neighbor{IP6: event.Lease.IPv6Subnet.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {\n\t\t\t\t\t\t\tlog.Error(\"DelV6ARP failed: \", err)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\n\t\t\t\t\t// Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure\n\t\t\t\t\t// this is done last.\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteReplace(&v6VxlanRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to add v6 vxlanRoute (%s -> %s): %v\", v6VxlanRoute.Dst, v6VxlanRoute.Gw, err)\n\n\t\t\t\t\t\t// Try to clean up both the ARP and FDB entries then continue\n\t\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\t\treturn nw.v6Dev.DelV6ARP(neighbor{IP6: event.Lease.IPv6Subnet.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)})\n\t\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\t\tlog.Error(\"DelV6ARP failed: \", err)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\t\treturn nw.v6Dev.DelV6FDB(neighbor{IP6: event.Lease.Attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)})\n\t\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\t\tlog.Error(\"DelV6FDB failed: \", err)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase lease.EventRemoved:\n\t\t\tif event.Lease.EnableIPv4 {\n\t\t\t\tif directRoutingOK {\n\t\t\t\t\tlog.V(2).Infof(\"Removing direct route to subnet: %s PublicIP: %s\", sn, attrs.PublicIP)\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteDel(&directRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"Error deleting route to %v via %v: %v\", sn, attrs.PublicIP, err)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tlog.V(2).Infof(\"removing subnet: %s PublicIP: %s VtepMAC: %s\", sn, attrs.PublicIP, net.HardwareAddr(vxlanAttrs.VtepMAC))\n\n\t\t\t\t\t// Try to remove all entries - don't bail out if one of them fails.\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.dev.DelARP(neighbor{IP: sn.IP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"DelARP failed: \", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.dev.DelFDB(neighbor{IP: attrs.PublicIP, MAC: net.HardwareAddr(vxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"DelFDB failed: \", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteDel(&vxlanRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to delete vxlanRoute (%s -> %s): %v\", vxlanRoute.Dst, vxlanRoute.Gw, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif event.Lease.EnableIPv6 {\n\t\t\t\tif v6DirectRoutingOK {\n\t\t\t\t\tlog.V(2).Infof(\"Removing v6 direct route to subnet: %s PublicIP: %s\", sn, attrs.PublicIPv6)\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteDel(&directRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"Error deleting v6 route to %v via %v: %v\", v6Sn, attrs.PublicIPv6, err)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tlog.V(2).Infof(\"removing v6subnet: %s PublicIPv6: %s VtepMAC: %s\", v6Sn, attrs.PublicIPv6, net.HardwareAddr(v6VxlanAttrs.VtepMAC))\n\n\t\t\t\t\t// Try to remove all entries - don't bail out if one of them fails.\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.v6Dev.DelV6ARP(neighbor{IP6: v6Sn.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"DelV6ARP failed: \", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn nw.v6Dev.DelV6FDB(neighbor{IP6: attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)})\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Error(\"DelV6FDB failed: \", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := retry.Do(func() error {\n\t\t\t\t\t\treturn netlink.RouteDel(&v6VxlanRoute)\n\t\t\t\t\t}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to delete v6 vxlanRoute (%s -> %s): %v\", v6VxlanRoute.Dst, v6VxlanRoute.Gw, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tlog.Error(\"internal error: unknown event type: \", int(event.Type))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/vxlan/vxlan_network_windows.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vxlan\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"net\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/Microsoft/hcsshim/hcn\"\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype network struct {\n\tbackend.SimpleNetwork\n\tdev       *vxlanDevice\n\tsubnetMgr subnet.Manager\n}\n\ntype vxlanLeaseAttrs struct {\n\tVNI     uint16\n\tVtepMAC hardwareAddr\n}\n\nconst (\n\tencapOverhead = 50\n)\n\nfunc newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, _ ip.IP4Net, myLease *lease.Lease) (*network, error) {\n\tnw := &network{\n\t\tSimpleNetwork: backend.SimpleNetwork{\n\t\t\tSubnetLease: myLease,\n\t\t\tExtIface:    extIface,\n\t\t},\n\t\tsubnetMgr: subnetMgr,\n\t\tdev:       dev,\n\t}\n\n\treturn nw, nil\n}\n\nfunc (nw *network) Run(ctx context.Context) {\n\twg := sync.WaitGroup{}\n\n\tlog.V(0).Info(\"Watching for new subnet leases\")\n\tevents := make(chan []lease.Event)\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, nw.subnetMgr, nw.SubnetLease, events)\n\t\tlog.V(1).Info(\"WatchLeases exited\")\n\t\twg.Done()\n\t}()\n\n\tdefer wg.Wait()\n\n\tfor {\n\t\tselect {\n\t\tcase evtBatch, ok := <-events:\n\t\t\tif !ok {\n\t\t\t\tlog.Infof(\"evts chan closed\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tnw.handleSubnetEvents(evtBatch)\n\t\t}\n\t}\n}\n\nfunc (nw *network) MTU() int {\n\treturn nw.ExtIface.Iface.MTU - encapOverhead\n}\n\nfunc (nw *network) handleSubnetEvents(batch []lease.Event) {\n\tfor _, event := range batch {\n\t\tleaseSubnet := event.Lease.Subnet\n\t\tleaseAttrs := event.Lease.Attrs\n\t\tif !strings.EqualFold(leaseAttrs.BackendType, \"vxlan\") {\n\t\t\tlog.Warningf(\"ignoring non-vxlan subnet(%v): type=%v\", leaseSubnet, leaseAttrs.BackendType)\n\t\t\tcontinue\n\t\t}\n\n\t\tvar vxlanAttrs vxlanLeaseAttrs\n\t\tif err := json.Unmarshal(leaseAttrs.BackendData, &vxlanAttrs); err != nil {\n\t\t\tlog.Error(\"error decoding subnet lease JSON: \", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tif vxlanAttrs.VNI < 4096 {\n\t\t\tlog.Errorf(\"VNI is %d but it is required to be greater than or equal to 4096 on Windows.\", vxlanAttrs.VNI)\n\t\t\tcontinue\n\t\t}\n\n\t\thnsnetwork, err := hcn.GetNetworkByName(nw.dev.link.Name)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Unable to find network %v, error: %v\", nw.dev.link.Name, err)\n\t\t\tcontinue\n\t\t}\n\t\tmanagementIp := event.Lease.Attrs.PublicIP.String()\n\n\t\tnetworkPolicySettings := hcn.RemoteSubnetRoutePolicySetting{\n\t\t\tIsolationId:                 vxlanAttrs.VNI,\n\t\t\tDistributedRouterMacAddress: net.HardwareAddr(vxlanAttrs.VtepMAC).String(),\n\t\t\tProviderAddress:             managementIp,\n\t\t\tDestinationPrefix:           event.Lease.Subnet.String(),\n\t\t}\n\t\trawJSON, err := json.Marshal(networkPolicySettings)\n\t\tnetworkPolicy := hcn.NetworkPolicy{\n\t\t\tType:     hcn.RemoteSubnetRoute,\n\t\t\tSettings: rawJSON,\n\t\t}\n\n\t\tpolicyNetworkRequest := hcn.PolicyNetworkRequest{\n\t\t\tPolicies: []hcn.NetworkPolicy{networkPolicy},\n\t\t}\n\n\t\tswitch event.Type {\n\t\tcase lease.EventAdded:\n\t\t\tfor _, policy := range hnsnetwork.Policies {\n\t\t\t\tif policy.Type == hcn.RemoteSubnetRoute {\n\t\t\t\t\texistingPolicySettings := hcn.RemoteSubnetRoutePolicySetting{}\n\t\t\t\t\terr = json.Unmarshal(policy.Settings, &existingPolicySettings)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Error(\"Failed to unmarshal settings\")\n\t\t\t\t\t}\n\t\t\t\t\tif existingPolicySettings.DestinationPrefix == networkPolicySettings.DestinationPrefix {\n\t\t\t\t\t\texistingJson, err := json.Marshal(existingPolicySettings)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tlog.Error(\"Failed to marshal settings\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\texistingPolicy := hcn.NetworkPolicy{\n\t\t\t\t\t\t\tType:     hcn.RemoteSubnetRoute,\n\t\t\t\t\t\t\tSettings: existingJson,\n\t\t\t\t\t\t}\n\t\t\t\t\t\texistingPolicyNetworkRequest := hcn.PolicyNetworkRequest{\n\t\t\t\t\t\t\tPolicies: []hcn.NetworkPolicy{existingPolicy},\n\t\t\t\t\t\t}\n\t\t\t\t\t\thnsnetwork.RemovePolicy(existingPolicyNetworkRequest)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif networkPolicySettings.DistributedRouterMacAddress != \"\" {\n\t\t\t\thnsnetwork.AddPolicy(policyNetworkRequest)\n\t\t\t}\n\t\tcase lease.EventRemoved:\n\t\t\tif networkPolicySettings.DistributedRouterMacAddress != \"\" {\n\t\t\t\thnsnetwork.RemovePolicy(policyNetworkRequest)\n\t\t\t}\n\t\tdefault:\n\t\t\tlog.Error(\"internal error: unknown event type: \", int(event.Type))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/vxlan/vxlan_windows.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vxlan\n\n// Some design notes:\n// VXLAN encapsulates L2 packets (though flannel is L3 only so don't expect to be able to send L2 packets across hosts)\n// Windows overlay decap works at L2 and so it needs the correct destination MAC for the remote host to work.\n// Windows does not expose an L3Miss interface so for now all possible remote IP/MAC pairs have to be configured upfront.\n//\n// In this scheme the scaling of table entries (per host) is:\n//  - 1 network entry for the overlay network\n//  - 1 endpoint per local container\n//  - N remote endpoints remote node (total endpoints = N * number of remote nodes)\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/Microsoft/hcsshim/hcn\"\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc init() {\n\tbackend.Register(\"vxlan\", New)\n}\n\nconst (\n\tdefaultVNI = 4096\n\tvxlanPort  = 4789\n)\n\ntype VXLANBackend struct {\n\tsubnetMgr subnet.Manager\n\textIface  *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbackend := &VXLANBackend{\n\t\tsubnetMgr: sm,\n\t\textIface:  extIface,\n\t}\n\n\treturn backend, nil\n}\n\nfunc newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*lease.LeaseAttrs, error) {\n\tvar hardwareAddress hardwareAddr\n\tif mac != nil {\n\t\thardwareAddress = hardwareAddr(mac)\n\t}\n\tleaseAttrs := &vxlanLeaseAttrs{\n\t\tVNI:     vnid,\n\t\tVtepMAC: hardwareAddress,\n\t}\n\tdata, err := json.Marshal(&leaseAttrs)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to marshal vxlanLeaseAttrs: %w\", err)\n\t}\n\n\treturn &lease.LeaseAttrs{\n\t\tPublicIP:    ip.FromIP(publicIP),\n\t\tBackendType: \"vxlan\",\n\t\tBackendData: json.RawMessage(data),\n\t}, nil\n}\n\nfunc (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\t// 1. Parse configuration\n\tcfg := struct {\n\t\tName          string\n\t\tMacPrefix     string\n\t\tVNI           int\n\t\tPort          int\n\t\tGBP           bool\n\t\tDirectRouting bool\n\t}{\n\t\tVNI:       defaultVNI,\n\t\tPort:      vxlanPort,\n\t\tMacPrefix: \"0E-2A\",\n\t}\n\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding VXLAN backend config: %v\", err)\n\t\t}\n\t}\n\n\t// 2. Verify configuration\n\tif cfg.VNI < defaultVNI {\n\t\treturn nil, fmt.Errorf(\"invalid VXLAN backend config. VNI [%v] must be greater than or equal to %v on Windows\", cfg.VNI, defaultVNI)\n\t}\n\tif cfg.Port != vxlanPort {\n\t\treturn nil, fmt.Errorf(\"invalid VXLAN backend config. Port [%v] is not supported on Windows. Omit the setting to default to port %v\", cfg.Port, vxlanPort)\n\t}\n\tif cfg.DirectRouting {\n\t\treturn nil, errors.New(\"invalid VXLAN backend config. DirectRouting is not supported on Windows\")\n\t}\n\tif cfg.GBP {\n\t\treturn nil, errors.New(\"invalid VXLAN backend config. GBP is not supported on Windows\")\n\t}\n\tif len(cfg.MacPrefix) == 0 || len(cfg.MacPrefix) != 5 || cfg.MacPrefix[2] != '-' {\n\t\treturn nil, fmt.Errorf(\"invalid VXLAN backend config.MacPrefix [%v] is invalid, prefix must be of the format xx-xx e.g. 0E-2A\", cfg.MacPrefix)\n\t}\n\tif len(cfg.Name) == 0 {\n\t\tcfg.Name = fmt.Sprintf(\"flannel.%v\", cfg.VNI)\n\t}\n\tlog.Infof(\"VXLAN config: Name=%s MacPrefix=%s VNI=%d Port=%d GBP=%v DirectRouting=%v\", cfg.Name, cfg.MacPrefix, cfg.VNI, cfg.Port, cfg.GBP, cfg.DirectRouting)\n\n\terr := hcn.RemoteSubnetSupported()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tsubnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tlease, err := be.subnetMgr.AcquireLease(ctx, subnetAttrs)\n\tswitch err {\n\tcase nil:\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %v\", err)\n\t}\n\n\tdevAttrs := vxlanDeviceAttrs{\n\t\tvni:           uint32(cfg.VNI),\n\t\tname:          cfg.Name,\n\t\taddressPrefix: lease.Subnet,\n\t\tinterfaceName: be.extIface.Iface.Name,\n\t}\n\n\tdev, err := newVXLANDevice(ctx, &devAttrs)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create VXLAN network: %w\", err)\n\t}\n\tdev.directRouting = cfg.DirectRouting\n\tdev.macPrefix = cfg.MacPrefix\n\n\tnetwork, err := newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\thcnNetwork, err := hcn.GetNetworkByName(cfg.Name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to get HNS network: %w\", err)\n\t}\n\n\tvar newDrMac string\n\tfor _, policy := range hcnNetwork.Policies {\n\t\tif policy.Type == hcn.DrMacAddress {\n\t\t\tpolicySettings := hcn.DrMacAddressNetworkPolicySetting{}\n\t\t\terr = json.Unmarshal(policy.Settings, &policySettings)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to unmarshal settings\")\n\t\t\t}\n\t\t\tnewDrMac = policySettings.Address\n\t\t}\n\t}\n\n\tmac, err := net.ParseMAC(string(newDrMac))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"cannot parse DR MAC %v: %+v\", newDrMac, err)\n\t}\n\n\tsubnetAttrs, err = newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), mac)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Before contacting the lease server (e.g. kube-api), we verify that the physical interface is ready\n\terr = checkHostNetworkReady(ctx, hcnNetwork)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"interface bound to %s took too long to get ready. Please check your network host configuration\", hcnNetwork.Name)\n\t}\n\n\tlease, err = be.subnetMgr.AcquireLease(ctx, subnetAttrs)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %w\", err)\n\t}\n\tnetwork.SubnetLease = lease\n\treturn network, nil\n}\n\n// So we can make it JSON (un)marshalable\ntype hardwareAddr net.HardwareAddr\n\nfunc (hw hardwareAddr) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(\"%q\", net.HardwareAddr(hw))), nil\n}\n\nfunc (hw *hardwareAddr) UnmarshalJSON(bytes []byte) error {\n\tif len(bytes) < 2 || bytes[0] != '\"' || bytes[len(bytes)-1] != '\"' {\n\t\treturn fmt.Errorf(\"error parsing hardware addr\")\n\t}\n\n\tbytes = bytes[1 : len(bytes)-1]\n\n\tmac, err := net.ParseMAC(string(bytes))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t*hw = hardwareAddr(mac)\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/backend/wireguard/device.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2021 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\npackage wireguard\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/vishvananda/netlink\"\n\t\"golang.zx2c4.com/wireguard/wgctrl\"\n\t\"golang.zx2c4.com/wireguard/wgctrl/wgtypes\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype wgDeviceAttrs struct {\n\tlistenPort int\n\tprivateKey *wgtypes.Key\n\tpublicKey  *wgtypes.Key\n\tpsk        *wgtypes.Key\n\tkeepalive  *time.Duration\n\tname       string\n\tMTU        int\n}\n\ntype wgDevice struct {\n\tlink  *netlink.GenericLink\n\tattrs *wgDeviceAttrs\n}\n\nfunc writePrivateKey(path string, content string) error {\n\tdir, _ := filepath.Split(path)\n\terr := os.MkdirAll(dir, 0755)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tf, err := os.Create(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = os.Chmod(path, 0400)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = f.WriteString(content)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = f.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (devAttrs *wgDeviceAttrs) setupKeys(psk string) error {\n\tkeyFile := \"/run/flannel/wgkey\"\n\n\tenvKeyFile, envExists := os.LookupEnv(\"WIREGUARD_KEY_FILE\")\n\tif envExists {\n\t\tkeyFile = envKeyFile\n\t}\n\n\tif _, err := os.Stat(keyFile); errors.Is(err, os.ErrNotExist) {\n\t\tprivateKey, err := wgtypes.GeneratePrivateKey()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not generate private key: %w\", err)\n\t\t}\n\t\tdevAttrs.privateKey = &privateKey\n\n\t\tpublicKey := privateKey.PublicKey()\n\t\tdevAttrs.publicKey = &publicKey\n\n\t\terr = writePrivateKey(keyFile, privateKey.String())\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not write key file: %w\", err)\n\t\t}\n\t} else {\n\t\tdata, err := os.ReadFile(keyFile)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tprivateKey, err := wgtypes.ParseKey(string(data))\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not parse private key from file: %w\", err)\n\t\t}\n\t\tdevAttrs.privateKey = &privateKey\n\t\tpublicKey := privateKey.PublicKey()\n\t\tdevAttrs.publicKey = &publicKey\n\t}\n\n\tif psk != \"\" {\n\t\tpresharedKey, err := wgtypes.ParseKey(psk)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not parse psk: %w\", err)\n\t\t}\n\t\tdevAttrs.psk = &presharedKey\n\t}\n\n\treturn nil\n}\n\nfunc newWGDevice(devAttrs *wgDeviceAttrs, ctx context.Context, wg *sync.WaitGroup) (*wgDevice, error) {\n\t// Create network device\n\tla := netlink.LinkAttrs{\n\t\tName: devAttrs.name,\n\t\tMTU:  devAttrs.MTU - overhead,\n\t}\n\tlink := &netlink.GenericLink{LinkAttrs: la, LinkType: \"wireguard\"}\n\n\tlink, err := ensureLink(link)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdev := wgDevice{\n\t\tlink:  link,\n\t\tattrs: devAttrs,\n\t}\n\n\t// Create wireguard interface\n\twgcfg := wgtypes.Config{\n\t\tPrivateKey:   dev.attrs.privateKey,\n\t\tListenPort:   &dev.attrs.listenPort,\n\t\tReplacePeers: true,\n\t}\n\n\tclient, err := wgctrl.New()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to open wgctrl: %w\", err)\n\t}\n\tdefer func() {\n\t\terr := client.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"failed to close wgctrl client: %v\", err)\n\t\t}\n\t}()\n\n\terr = client.ConfigureDevice(dev.attrs.name, wgcfg)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to configure device %w\", err)\n\t}\n\n\t// This code runs before flannel terminates.\n\t// We remove the device to undo any change we did to the system.\n\twg.Add(1)\n\tgo func() {\n\t\t<-ctx.Done()\n\t\terr := dev.remove()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Error while removing device: %v\", err)\n\t\t}\n\t\tlog.Infof(\"Removed wireguard device %s\", dev.attrs.name)\n\t\twg.Done()\n\t}()\n\n\treturn &dev, nil\n}\n\nfunc ensureLink(wglan *netlink.GenericLink) (*netlink.GenericLink, error) {\n\terr := netlink.LinkAdd(wglan)\n\tif err == syscall.EEXIST {\n\t\texisting, err := netlink.LinkByName(wglan.Name)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tlog.Warningf(\"%q already exists; recreating device\", wglan.Name)\n\t\terr = netlink.LinkDel(existing)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\terr = netlink.LinkAdd(wglan)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not create wireguard interface: %w\", err)\n\t\t}\n\t} else if err != nil {\n\t\treturn nil, fmt.Errorf(\"could not create wireguard interface: %w\", err)\n\t}\n\n\t_, err = netlink.LinkByIndex(wglan.Index)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"can't locate created wireguard device with index %v: %w\", wglan.Index, err)\n\t}\n\n\treturn wglan, nil\n}\n\nfunc (dev *wgDevice) remove() error {\n\terr := netlink.LinkDel(dev.link)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"could not remove wireguard device: %w\", err)\n\t}\n\treturn nil\n}\n\nfunc (dev *wgDevice) upAndAddRoute(dst *net.IPNet) error {\n\terr := netlink.LinkSetUp(dev.link)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to set interface %s to UP state: %w\", dev.attrs.name, err)\n\t}\n\n\terr = dev.addRoute(dst)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to add route to destination (%s) to interface (%s): %w\", dst, dev.attrs.name, err)\n\t}\n\treturn nil\n}\n\nfunc (dev *wgDevice) addRoute(dst *net.IPNet) error {\n\troute := netlink.Route{\n\t\tLinkIndex: dev.link.Attrs().Index,\n\t\tScope:     netlink.SCOPE_LINK,\n\t\tDst:       dst,\n\t}\n\n\terr := netlink.RouteReplace(&route)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to add route %s: %w\", dev.attrs.name, err)\n\t}\n\n\treturn nil\n}\n\nfunc (dev *wgDevice) Configure(devIP ip.IP4, flannelnet ip.IP4Net) error {\n\tnet := ip.IP4Net{IP: devIP, PrefixLen: 32}\n\terr := ip.EnsureV4AddressOnLink(net, flannelnet, dev.link)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to ensure address of interface %s: %w\", dev.attrs.name, err)\n\t}\n\n\tif err := dev.upAndAddRoute(flannelnet.ToIPNet()); err != nil {\n\t\treturn fmt.Errorf(\"failed to set up the route: %w\", err)\n\t}\n\n\treturn nil\n}\n\nfunc (dev *wgDevice) ConfigureV6(devIP *ip.IP6, flannelnet ip.IP6Net) error {\n\tnet := ip.IP6Net{IP: devIP, PrefixLen: 128}\n\terr := ip.EnsureV6AddressOnLink(net, flannelnet, dev.link)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to ensure address of interface %s: %w\", dev.attrs.name, err)\n\t}\n\n\tif err := dev.upAndAddRoute(flannelnet.ToIPNet()); err != nil {\n\t\treturn fmt.Errorf(\"failed to set up the route: %w\", err)\n\t}\n\n\treturn nil\n}\n\nfunc (dev *wgDevice) addPeer(publicEndpoint string, peerPublicKeyRaw string, peerSubnets []net.IPNet) error {\n\tudpEndpoint, err := net.ResolveUDPAddr(\"udp\", publicEndpoint)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to resolve UDP address: %w\", err)\n\t}\n\n\tpeerPublicKey, err := wgtypes.ParseKey(peerPublicKeyRaw)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to parse publicKey: %w\", err)\n\t}\n\n\twgcfg := wgtypes.Config{\n\t\tPrivateKey:   dev.attrs.privateKey,\n\t\tListenPort:   &dev.attrs.listenPort,\n\t\tReplacePeers: false,\n\t\tPeers: []wgtypes.PeerConfig{\n\t\t\t{\n\t\t\t\tPublicKey:                   peerPublicKey,\n\t\t\t\tPresharedKey:                dev.attrs.psk,\n\t\t\t\tPersistentKeepaliveInterval: dev.attrs.keepalive,\n\t\t\t\tEndpoint:                    udpEndpoint,\n\t\t\t\tReplaceAllowedIPs:           true,\n\t\t\t\tAllowedIPs:                  peerSubnets,\n\t\t\t},\n\t\t}}\n\n\tclient, err := wgctrl.New()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to open wgctrl: %w\", err)\n\t}\n\tdefer func() {\n\t\terr := client.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"failed to close wgctrl client: %v\", err)\n\t\t}\n\t}()\n\n\terr = client.ConfigureDevice(dev.attrs.name, wgcfg)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to configure device %w\", err)\n\t}\n\n\treturn nil\n}\n\nfunc (dev *wgDevice) removePeer(peerPublicKeyRaw string) error {\n\tpeerPublicKey, err := wgtypes.ParseKey(peerPublicKeyRaw)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to parse publicKey: %w\", err)\n\t}\n\n\twgcfg := wgtypes.Config{\n\t\tReplacePeers: false,\n\t\tPeers: []wgtypes.PeerConfig{\n\t\t\t{\n\t\t\t\tPublicKey: peerPublicKey,\n\t\t\t\tRemove:    true,\n\t\t\t},\n\t\t}}\n\n\tclient, err := wgctrl.New()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to open wgctrl: %w\", err)\n\t}\n\tdefer func() {\n\t\terr := client.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"failed to close wgctrl client: %v\", err)\n\t\t}\n\t}()\n\n\terr = client.ConfigureDevice(dev.attrs.name, wgcfg)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to remove peer %w\", err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/backend/wireguard/wireguard.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2021 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\npackage wireguard\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\ntype Mode string\n\nconst (\n\tSeparate Mode = \"separate\"\n\tAuto     Mode = \"auto\"\n\tIpv4     Mode = \"ipv4\"\n\tIpv6     Mode = \"ipv6\"\n)\n\nfunc init() {\n\tbackend.Register(\"wireguard\", New)\n}\n\ntype WireguardBackend struct {\n\tsm       subnet.Manager\n\textIface *backend.ExternalInterface\n}\n\nfunc New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {\n\tbe := &WireguardBackend{\n\t\tsm:       sm,\n\t\textIface: extIface,\n\t}\n\n\treturn be, nil\n}\n\nfunc newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, enableIPv4, enableIPv6 bool, publicKey string, v4Port, v6Port uint16) (*lease.LeaseAttrs, error) {\n\tv4Data, err := json.Marshal(&wireguardLeaseAttrs{\n\t\tPublicKey: publicKey,\n\t\tPort:      v4Port,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tv6Data, err := json.Marshal(&wireguardLeaseAttrs{\n\t\tPublicKey: publicKey,\n\t\tPort:      v6Port,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tleaseAttrs := &lease.LeaseAttrs{\n\t\tBackendType: \"wireguard\",\n\t}\n\n\tif publicIP != nil {\n\t\tleaseAttrs.PublicIP = ip.FromIP(publicIP)\n\t}\n\n\tif enableIPv4 {\n\t\tleaseAttrs.BackendData = json.RawMessage(v4Data)\n\t}\n\n\tif publicIPv6 != nil {\n\t\tleaseAttrs.PublicIPv6 = ip.FromIP6(publicIPv6)\n\t}\n\n\tif enableIPv6 {\n\t\tleaseAttrs.BackendV6Data = json.RawMessage(v6Data)\n\t}\n\n\treturn leaseAttrs, nil\n}\n\nfunc createWGDev(ctx context.Context, wg *sync.WaitGroup, name string, psk string, keepalive *time.Duration, listenPort int, mtu int) (*wgDevice, error) {\n\tdevAttrs := wgDeviceAttrs{\n\t\tkeepalive:  keepalive,\n\t\tlistenPort: listenPort,\n\t\tname:       name,\n\t\tMTU:        mtu,\n\t}\n\terr := devAttrs.setupKeys(psk)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newWGDevice(&devAttrs, ctx, wg)\n}\n\nfunc (be *WireguardBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {\n\t// Parse out configuration\n\tcfg := struct {\n\t\tListenPort                  int\n\t\tListenPortV6                int\n\t\tMTU                         int\n\t\tPSK                         string\n\t\tPersistentKeepaliveInterval time.Duration\n\t\tMode                        Mode\n\t}{\n\t\tListenPort:                  51820,\n\t\tListenPortV6:                51821,\n\t\tMTU:                         be.extIface.Iface.MTU,\n\t\tPersistentKeepaliveInterval: 0,\n\t\tMode:                        Separate,\n\t}\n\n\tif len(config.Backend) > 0 {\n\t\tif err := json.Unmarshal(config.Backend, &cfg); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error decoding backend config: %w\", err)\n\t\t}\n\t}\n\n\tkeepalive := cfg.PersistentKeepaliveInterval * time.Second\n\n\tvar err error\n\tvar dev, v6Dev *wgDevice\n\tvar publicKey string\n\tswitch cfg.Mode {\n\tcase Separate:\n\t\tif config.EnableIPv4 {\n\t\t\tdev, err = createWGDev(ctx, wg, \"flannel-wg\", cfg.PSK, &keepalive, cfg.ListenPort, cfg.MTU)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tpublicKey = dev.attrs.publicKey.String()\n\t\t}\n\t\tif config.EnableIPv6 {\n\t\t\tv6Dev, err = createWGDev(ctx, wg, \"flannel-wg-v6\", cfg.PSK, &keepalive, cfg.ListenPortV6, cfg.MTU)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tpublicKey = v6Dev.attrs.publicKey.String()\n\t\t}\n\tcase Auto, Ipv4, Ipv6:\n\t\tdev, err = createWGDev(ctx, wg, \"flannel-wg\", cfg.PSK, &keepalive, cfg.ListenPort, cfg.MTU)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tpublicKey = dev.attrs.publicKey.String()\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"no valid Mode configured\")\n\t}\n\n\tsubnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, config.EnableIPv4, config.EnableIPv6, publicKey, uint16(cfg.ListenPort), uint16(cfg.ListenPortV6))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tlease, err := be.sm.AcquireLease(ctx, subnetAttrs)\n\tswitch err {\n\tcase nil:\n\tcase context.Canceled, context.DeadlineExceeded:\n\t\treturn nil, err\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"failed to acquire lease: %w\", err)\n\n\t}\n\n\tif config.EnableIPv4 {\n\t\tif lease.Subnet.Empty() {\n\t\t\treturn nil, fmt.Errorf(\"failed to configure wg interface: IPv4 is enabled but the lease has no IPv4\")\n\t\t}\n\n\t\terr = dev.Configure(lease.Subnet.IP, config.Network)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif config.EnableIPv6 {\n\t\tif lease.IPv6Subnet.Empty() {\n\t\t\treturn nil, fmt.Errorf(\"failed to configure wg interface: IPv6 is enabled but the lease has no IPv6\")\n\t\t}\n\n\t\tif cfg.Mode == Separate {\n\t\t\terr = v6Dev.ConfigureV6(lease.IPv6Subnet.IP, config.IPv6Network)\n\t\t} else {\n\t\t\terr = dev.ConfigureV6(lease.IPv6Subnet.IP, config.IPv6Network)\n\t\t}\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn newNetwork(be.sm, be.extIface, dev, v6Dev, cfg.Mode, lease, cfg.MTU)\n}\n"
  },
  {
    "path": "pkg/backend/wireguard/wireguard_network.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2021 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\npackage wireguard\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\t/*\n\t\t20-byte IPv4 header or 40 byte IPv6 header\n\t\t8-byte UDP header\n\t\t4-byte type\n\t\t4-byte key index\n\t\t8-byte nonce\n\t\tN-byte encrypted data\n\t\t16-byte authentication tag\n\t*/\n\toverhead = 80\n)\n\ntype network struct {\n\tdev      *wgDevice\n\tv6Dev    *wgDevice\n\textIface *backend.ExternalInterface\n\tmode     Mode\n\tlease    *lease.Lease\n\tsm       subnet.Manager\n\tmtu      int\n}\n\nfunc newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface, dev, v6Dev *wgDevice, mode Mode, lease *lease.Lease, mtu int) (*network, error) {\n\tn := &network{\n\t\tdev:      dev,\n\t\tv6Dev:    v6Dev,\n\t\textIface: extIface,\n\t\tmode:     mode,\n\t\tlease:    lease,\n\t\tsm:       sm,\n\t\tmtu:      mtu,\n\t}\n\n\treturn n, nil\n}\n\nfunc (n *network) Lease() *lease.Lease {\n\treturn n.lease\n}\n\nfunc (n *network) MTU() int {\n\treturn n.mtu - overhead\n}\n\nfunc (n *network) Run(ctx context.Context) {\n\twg := sync.WaitGroup{}\n\n\tlog.Info(\"Watching for new subnet leases\")\n\tevents := make(chan []lease.Event)\n\twg.Add(1)\n\tgo func() {\n\t\tsubnet.WatchLeases(ctx, n.sm, n.lease, events)\n\t\twg.Done()\n\t}()\n\n\tdefer wg.Wait()\n\n\tfor {\n\t\tselect {\n\t\tcase evtBatch := <-events:\n\t\t\tn.handleSubnetEvents(ctx, evtBatch)\n\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\t}\n\t}\n}\n\ntype wireguardLeaseAttrs struct {\n\tPublicKey string\n\tPort      uint16\n}\n\n// Select the mode that is most likely to allow for a successful connection.\n// If both ipv4 and ipv6 addresses are provided:\n//   - Prefer ipv4 if the remote endpoint has a public ipv4 address\n//     and the external iface has an ipv4 address as well. Anything with\n//     an ipv4 address can likely connect to the public internet.\n//   - Use ipv6 if the remote endpoint has a publc address and the local\n//     interface has a public address as well. In which case it's likely that\n//     a connection can be made. The local interface having just an link-local\n//     address will only have a small chance of succeeding (ipv6 masquarading is\n//     very rare)\n//   - If neither is true default to ipv4 and cross fingers.\nfunc (n *network) selectMode(ip4 ip.IP4, ip6 *ip.IP6) Mode {\n\tif ip6 == nil {\n\t\treturn Ipv4\n\t}\n\tif !ip4.IsPrivate() && n.extIface.ExtAddr != nil {\n\t\treturn Ipv4\n\t}\n\tif !ip6.IsPrivate() && n.extIface.ExtV6Addr != nil && !ip.FromIP6(n.extIface.ExtV6Addr).IsPrivate() {\n\t\treturn Ipv6\n\t}\n\treturn Ipv4\n}\n\nfunc (n *network) handleSubnetEvents(ctx context.Context, batch []lease.Event) {\n\tfor _, event := range batch {\n\t\tswitch event.Type {\n\t\tcase lease.EventAdded:\n\n\t\t\tif event.Lease.Attrs.BackendType != \"wireguard\" {\n\t\t\t\tlog.Warningf(\"Ignoring non-wireguard subnet: type=%v\", event.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tvar v4wireguardAttrs, v6wireguardAttrs, wireguardAttrs wireguardLeaseAttrs\n\t\t\tvar subnets []*net.IPNet\n\t\t\tif event.Lease.EnableIPv4 {\n\t\t\t\tif len(event.Lease.Attrs.BackendData) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(event.Lease.Attrs.BackendData, &v4wireguardAttrs); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to unmarshal BackendData: %v\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twireguardAttrs = v4wireguardAttrs\n\t\t\t\tsubnets = append(subnets, event.Lease.Subnet.ToIPNet()) // only used if n.mode != Separate\n\t\t\t}\n\n\t\t\tif event.Lease.EnableIPv6 {\n\t\t\t\tif len(event.Lease.Attrs.BackendV6Data) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(event.Lease.Attrs.BackendV6Data, &v6wireguardAttrs); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to unmarshal BackendData: %v\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twireguardAttrs = v6wireguardAttrs\n\t\t\t\tsubnets = append(subnets, event.Lease.IPv6Subnet.ToIPNet()) // only used if n.mode != Separate\n\t\t\t}\n\n\t\t\t// default to the port in the attr, but use the device's listen port\n\t\t\t// if it's not set for backwards compatibility with older flannel\n\t\t\t// versions.\n\t\t\tv4Port := v4wireguardAttrs.Port\n\t\t\tif v4Port == 0 && n.dev != nil {\n\t\t\t\tv4Port = uint16(n.dev.attrs.listenPort)\n\t\t\t}\n\t\t\tv6Port := v6wireguardAttrs.Port\n\t\t\tif v6Port == 0 && n.v6Dev != nil {\n\t\t\t\tv6Port = uint16(n.v6Dev.attrs.listenPort)\n\t\t\t}\n\t\t\tv4PeerEndpoint := fmt.Sprintf(\"%s:%d\", event.Lease.Attrs.PublicIP.String(), v4Port)\n\t\t\tvar v6PeerEndpoint string\n\t\t\tif event.Lease.Attrs.PublicIPv6 != nil {\n\t\t\t\tv6PeerEndpoint = fmt.Sprintf(\"[%s]:%d\", event.Lease.Attrs.PublicIPv6.String(), v6Port)\n\t\t\t}\n\t\t\tif n.mode == Separate {\n\t\t\t\tif event.Lease.EnableIPv4 {\n\t\t\t\t\tlog.Infof(\"Subnet added: %v via %v\", event.Lease.Subnet, v4PeerEndpoint)\n\t\t\t\t\tif err := n.dev.addPeer(\n\t\t\t\t\t\tv4PeerEndpoint,\n\t\t\t\t\t\tv4wireguardAttrs.PublicKey,\n\t\t\t\t\t\t[]net.IPNet{*event.Lease.Subnet.ToIPNet()}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to setup ipv4 peer (%s): %v\", v4wireguardAttrs.PublicKey, err)\n\t\t\t\t\t}\n\t\t\t\t\tnetconf, err := n.sm.GetNetworkConfig(ctx)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Errorf(\"could not read network config: %v\", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := n.dev.addRoute(netconf.Network.ToIPNet()); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to add ipv4 route to (%s): %v\", netconf.Network, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif event.Lease.EnableIPv6 {\n\t\t\t\t\tlog.Infof(\"Subnet added: %v via %v\", event.Lease.IPv6Subnet, v6PeerEndpoint)\n\t\t\t\t\tif err := n.v6Dev.addPeer(\n\t\t\t\t\t\tv6PeerEndpoint,\n\t\t\t\t\t\tv6wireguardAttrs.PublicKey,\n\t\t\t\t\t\t[]net.IPNet{*event.Lease.IPv6Subnet.ToIPNet()}); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to setup ipv6 peer (%s): %v\", v6wireguardAttrs.PublicKey, err)\n\t\t\t\t\t}\n\t\t\t\t\tnetconf, err := n.sm.GetNetworkConfig(ctx)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Errorf(\"could not read network config: %v\", err)\n\t\t\t\t\t}\n\n\t\t\t\t\tif err := n.v6Dev.addRoute(netconf.IPv6Network.ToIPNet()); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to add ipv6 route to (%s): %v\", netconf.IPv6Network, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tvar publicEndpoint string\n\t\t\t\tmode := n.mode\n\t\t\t\tif mode != Ipv4 && mode != Ipv6 { // Auto mode\n\t\t\t\t\tmode = n.selectMode(event.Lease.Attrs.PublicIP, event.Lease.Attrs.PublicIPv6)\n\t\t\t\t}\n\t\t\t\tswitch mode {\n\t\t\t\tcase Ipv4:\n\t\t\t\t\tpublicEndpoint = v4PeerEndpoint\n\t\t\t\tcase Ipv6:\n\t\t\t\t\tpublicEndpoint = v6PeerEndpoint\n\t\t\t\t}\n\n\t\t\t\tlog.Infof(\"Subnet(s) added: %v via %v\", subnets, publicEndpoint)\n\t\t\t\tvar peers []net.IPNet\n\t\t\t\tfor _, v := range subnets {\n\t\t\t\t\tpeers = append(peers, *v)\n\t\t\t\t}\n\t\t\t\tif err := n.dev.addPeer(\n\t\t\t\t\tpublicEndpoint,\n\t\t\t\t\twireguardAttrs.PublicKey,\n\t\t\t\t\tpeers); err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to setup peer (%s): %v\", v4wireguardAttrs.PublicKey, err)\n\t\t\t\t}\n\t\t\t\tnetconf, err := n.sm.GetNetworkConfig(ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"could not read network config: %v\", err)\n\t\t\t\t}\n\n\t\t\t\tif err := n.dev.addRoute(netconf.Network.ToIPNet()); err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to add ipv4 route to (%s): %v\", netconf.Network, err)\n\t\t\t\t}\n\n\t\t\t\tif err := n.dev.addRoute(netconf.IPv6Network.ToIPNet()); err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to add ipv6 route to (%s): %v\", netconf.IPv6Network, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase lease.EventRemoved:\n\n\t\t\tif event.Lease.Attrs.BackendType != \"wireguard\" {\n\t\t\t\tlog.Warningf(\"Ignoring non-wireguard subnet: type=%v\", event.Lease.Attrs.BackendType)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tvar wireguardAttrs wireguardLeaseAttrs\n\t\t\tif event.Lease.EnableIPv4 && n.dev != nil {\n\t\t\t\tlog.Info(\"Subnet removed: \", event.Lease.Subnet)\n\t\t\t\tif len(event.Lease.Attrs.BackendData) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(event.Lease.Attrs.BackendData, &wireguardAttrs); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to unmarshal BackendData: %v\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif err := n.dev.removePeer(\n\t\t\t\t\twireguardAttrs.PublicKey,\n\t\t\t\t); err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to remove ipv4 peer (%s): %v\", wireguardAttrs.PublicKey, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif event.Lease.EnableIPv6 {\n\t\t\t\tlog.Info(\"Subnet removed: \", event.Lease.IPv6Subnet)\n\t\t\t\tif len(event.Lease.Attrs.BackendV6Data) > 0 {\n\t\t\t\t\tif err := json.Unmarshal(event.Lease.Attrs.BackendV6Data, &wireguardAttrs); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"failed to unmarshal BackendData: %v\", err)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tvar err error\n\t\t\t\tif n.mode == Separate && n.v6Dev != nil {\n\t\t\t\t\terr = n.v6Dev.removePeer(wireguardAttrs.PublicKey)\n\t\t\t\t} else {\n\t\t\t\t\terr = n.dev.removePeer(wireguardAttrs.PublicKey)\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"failed to remove ipv6 peer (%s): %v\", wireguardAttrs.PublicKey, err)\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tlog.Error(\"Internal error: unknown event type: \", int(event.Type))\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/backend/wireguard/wireguard_windows.go",
    "content": "//go:build windows\n// +build windows\n\n// Copyright 2021 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\npackage wireguard\n"
  },
  {
    "path": "pkg/ip/endianess.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\n// Taken from a patch by David Anderson who submitted it\n// but got rejected by the golang team\n\nimport (\n\t\"encoding/binary\"\n\t\"unsafe\"\n)\n\n// NativeEndian is the ByteOrder of the current system.\nvar NativeEndian binary.ByteOrder\n\nfunc init() {\n\t// Examine the memory layout of an int16 to determine system\n\t// endianness.\n\tvar one int16 = 1\n\tb := (*byte)(unsafe.Pointer(&one))\n\tif *b == 0 {\n\t\tNativeEndian = binary.BigEndian\n\t} else {\n\t\tNativeEndian = binary.LittleEndian\n\t}\n}\n\nfunc NativelyLittle() bool {\n\treturn NativeEndian == binary.LittleEndian\n}\n"
  },
  {
    "path": "pkg/ip/iface.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"slices\"\n\t\"syscall\"\n\n\t\"github.com/vishvananda/netlink\"\n\t\"golang.org/x/sys/unix\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc getIfaceAddrs(iface *net.Interface) ([]netlink.Addr, error) {\n\tlink := &netlink.Device{\n\t\tLinkAttrs: netlink.LinkAttrs{\n\t\t\tIndex: iface.Index,\n\t\t},\n\t}\n\n\treturn netlink.AddrList(link, syscall.AF_INET)\n}\n\nfunc getIfaceV6Addrs(iface *net.Interface) ([]netlink.Addr, error) {\n\tlink := &netlink.Device{\n\t\tLinkAttrs: netlink.LinkAttrs{\n\t\t\tIndex: iface.Index,\n\t\t},\n\t}\n\n\treturn netlink.AddrList(link, syscall.AF_INET6)\n}\n\nfunc GetInterfaceIP4Addrs(iface *net.Interface) ([]net.IP, error) {\n\taddrs, err := getIfaceAddrs(iface)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// sort addresses in preferred usage order\n\tslices.SortFunc(addrs, compareAddrs)\n\n\t// map rich ip information from netlink into the stdlib data structure\n\t// while filtering addresses that aren't relevant or usable\n\tipAddrs := make([]net.IP, 0)\n\tfor _, i := range addrs {\n\t\t// address must be IPv4, global unicast or link-local unicast and non-deprecated\n\t\tif i.IP.To4() != nil &&\n\t\t\t(i.IP.IsGlobalUnicast() || i.IP.IsLinkLocalUnicast()) &&\n\t\t\ti.Flags&syscall.IFA_F_DEPRECATED == 0 {\n\t\t\tipAddrs = append(ipAddrs, i.IP)\n\t\t}\n\t}\n\n\tif len(ipAddrs) > 0 {\n\t\treturn ipAddrs, nil\n\t}\n\n\treturn nil, errors.New(\"no IPv4 address found for given interface\")\n}\n\nfunc GetInterfaceIP6Addrs(iface *net.Interface) ([]net.IP, error) {\n\taddrs, err := getIfaceV6Addrs(iface)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// sort addresses in preferred usage order\n\tslices.SortFunc(addrs, compareAddrs)\n\n\t// map rich ip information from netlink into the stdlib data structure\n\t// while filtering addresses that aren't relevant or usable\n\tipAddrs := make([]net.IP, 0)\n\tfor _, i := range addrs {\n\t\t// address must be IPv6, global unicast or link-local unicast and non-deprecated\n\t\tif i.IP.To16() != nil &&\n\t\t\t(i.IP.IsGlobalUnicast() || i.IP.IsLinkLocalUnicast()) &&\n\t\t\ti.Flags&syscall.IFA_F_DEPRECATED == 0 {\n\t\t\tipAddrs = append(ipAddrs, i.IP)\n\t\t}\n\t}\n\n\tif len(ipAddrs) > 0 {\n\t\treturn ipAddrs, nil\n\t}\n\n\treturn nil, errors.New(\"no IPv6 address found for given interface\")\n}\n\nfunc GetInterfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {\n\taddrs, err := getIfaceAddrs(iface)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, addr := range addrs {\n\t\t// Attempt to parse the address in CIDR notation\n\t\t// and assert it is IPv4\n\t\tif addr.IP.To4() != nil {\n\t\t\tif addr.IP.To4().Equal(matchAddr) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\n\treturn errors.New(\"no IPv4 address found for given interface\")\n}\n\nfunc GetInterfaceIP6AddrMatch(iface *net.Interface, matchAddr net.IP) error {\n\taddrs, err := getIfaceV6Addrs(iface)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, addr := range addrs {\n\t\t// Attempt to parse the address in CIDR notation\n\t\t// and assert it is IPv6\n\t\tif addr.IP.To16() != nil {\n\t\t\tif addr.IP.To16().Equal(matchAddr) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\n\treturn errors.New(\"no IPv6 address found for given interface\")\n}\n\nfunc GetDefaultGatewayInterface() (*net.Interface, error) {\n\troutes, err := netlink.RouteList(nil, syscall.AF_INET)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, route := range routes {\n\t\tif route.Dst == nil || route.Dst.String() == \"0.0.0.0/0\" {\n\t\t\tif route.LinkIndex <= 0 {\n\t\t\t\treturn nil, errors.New(\"found default route but could not determine interface\")\n\t\t\t}\n\t\t\treturn net.InterfaceByIndex(route.LinkIndex)\n\t\t}\n\t}\n\n\treturn nil, errors.New(\"unable to find default route\")\n}\n\nfunc GetDefaultV6GatewayInterface() (*net.Interface, error) {\n\troutes, err := netlink.RouteList(nil, syscall.AF_INET6)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, route := range routes {\n\t\tif route.Dst == nil || route.Dst.String() == \"::/0\" {\n\t\t\tif route.LinkIndex <= 0 {\n\t\t\t\treturn nil, errors.New(\"found default v6 route but could not determine interface\")\n\t\t\t}\n\t\t\treturn net.InterfaceByIndex(route.LinkIndex)\n\t\t}\n\t}\n\n\treturn nil, errors.New(\"unable to find default v6 route\")\n}\n\nfunc GetInterfaceByIP(ip net.IP) (*net.Interface, error) {\n\tifaces, err := net.Interfaces()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, iface := range ifaces {\n\t\terr := GetInterfaceIP4AddrMatch(&iface, ip)\n\t\tif err == nil {\n\t\t\treturn &iface, nil\n\t\t}\n\t}\n\n\treturn nil, errors.New(\"no interface with given IP found\")\n}\n\nfunc GetInterfaceByIP6(ip net.IP) (*net.Interface, error) {\n\tifaces, err := net.Interfaces()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, iface := range ifaces {\n\t\terr := GetInterfaceIP6AddrMatch(&iface, ip)\n\t\tif err == nil {\n\t\t\treturn &iface, nil\n\t\t}\n\t}\n\n\treturn nil, errors.New(\"no interface with given IPv6 found\")\n}\n\nfunc GetInterfaceBySpecificIPRouting(ip net.IP) (*net.Interface, net.IP, error) {\n\troutes, err := netlink.RouteGet(ip)\n\tif err != nil {\n\t\treturn nil, nil, fmt.Errorf(\"couldn't lookup route to %v: %v\", ip, err)\n\t}\n\n\tfor _, route := range routes {\n\t\tiface, err := net.InterfaceByIndex(route.LinkIndex)\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"couldn't lookup interface: %v\", err)\n\t\t} else {\n\t\t\treturn iface, route.Src, nil\n\t\t}\n\t}\n\n\treturn nil, nil, errors.New(\"no interface with given IP found\")\n}\n\nfunc DirectRouting(ip net.IP) (bool, error) {\n\troutes, err := netlink.RouteGet(ip)\n\tif err != nil {\n\t\treturn false, fmt.Errorf(\"couldn't lookup route to %v: %v\", ip, err)\n\t}\n\tif len(routes) == 1 && routes[0].Gw == nil {\n\t\t// There is only a single route and there's no gateway (i.e. it's directly connected)\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\n// EnsureV4AddressOnLink ensures that there is only one v4 Addr on `link` within the `ipn` address space and it equals `ipa`.\nfunc EnsureV4AddressOnLink(ipa IP4Net, ipn IP4Net, link netlink.Link) error {\n\taddr := netlink.Addr{IPNet: ipa.ToIPNet()}\n\texistingAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar hasAddr bool\n\tfor _, existingAddr := range existingAddrs {\n\t\tif existingAddr.Equal(addr) {\n\t\t\thasAddr = true\n\t\t\tcontinue\n\t\t}\n\n\t\tif ipn.Contains(FromIP(existingAddr.IP)) {\n\t\t\tif err := netlink.AddrDel(link, &existingAddr); err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to remove IP address %s from %s: %s\", existingAddr.String(), link.Attrs().Name, err)\n\t\t\t}\n\t\t\tlog.Infof(\"removed IP address %s from %s\", existingAddr.String(), link.Attrs().Name)\n\t\t}\n\t}\n\n\t// Actually add the desired address to the interface if needed.\n\tif !hasAddr {\n\t\tif err := netlink.AddrAdd(link, &addr); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to add IP address %s to %s: %s\", addr.String(), link.Attrs().Name, err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// EnsureV6AddressOnLink ensures that there is only one v6 Addr on `link` and it equals `ipn`.\n// If there exist multiple addresses on link, it returns an error message to tell callers to remove additional address.\nfunc EnsureV6AddressOnLink(ipa IP6Net, ipn IP6Net, link netlink.Link) error {\n\taddr := netlink.Addr{IPNet: ipa.ToIPNet()}\n\texistingAddrs, err := netlink.AddrList(link, netlink.FAMILY_V6)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tonlyLinkLocal := true\n\tfor _, existingAddr := range existingAddrs {\n\t\tif !existingAddr.IP.IsLinkLocalUnicast() {\n\t\t\tif !existingAddr.Equal(addr) {\n\t\t\t\tif err := netlink.AddrDel(link, &existingAddr); err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"failed to remove v6 IP address %s from %s: %w\", ipn.String(), link.Attrs().Name, err)\n\t\t\t\t}\n\t\t\t\texistingAddrs = []netlink.Addr{}\n\t\t\t\tonlyLinkLocal = false\n\t\t\t} else {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\n\tif onlyLinkLocal {\n\t\texistingAddrs = []netlink.Addr{}\n\t}\n\n\t// Actually add the desired address to the interface if needed.\n\tif len(existingAddrs) == 0 {\n\t\tif err := netlink.AddrAdd(link, &addr); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to add v6 IP address %s to %s: %w\", ipn.String(), link.Attrs().Name, err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc AddBlackholeV4Route(ipV4Dest *net.IPNet) error {\n\troute := netlink.Route{Dst: ipV4Dest, Type: unix.RTN_BLACKHOLE, Family: netlink.FAMILY_V4}\n\troutes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &route, netlink.RT_FILTER_DST|netlink.RT_FILTER_TYPE)\n\tif err == nil && len(routes) == 0 {\n\t\terr = netlink.RouteAdd(&route)\n\t}\n\treturn err\n}\n\nfunc AddBlackholeV6Route(ipV6Dest *net.IPNet) error {\n\troute := netlink.Route{Dst: ipV6Dest, Type: unix.RTN_BLACKHOLE, Family: netlink.FAMILY_V6}\n\troutes, err := netlink.RouteListFiltered(netlink.FAMILY_V6, &route, netlink.RT_FILTER_DST|netlink.RT_FILTER_TYPE)\n\tif err == nil && len(routes) == 0 {\n\t\terr = netlink.RouteAdd(&route)\n\t}\n\treturn err\n}\n\n// compareAddrs compares two netlink address in terms of which one should be used preferably.\n// Addresses which are supposed to be completely disregarded are not considered specially and should be filtered separately\nfunc compareAddrs(a, b netlink.Addr) int {\n\t// global unicast addresses are preferable to link-local addresses because they probably have better reachability\n\tif a.IP.IsGlobalUnicast() && b.IP.IsLinkLocalUnicast() {\n\t\treturn -1\n\t} else if a.IP.IsLinkLocalUnicast() && b.IP.IsGlobalUnicast() {\n\t\treturn 1\n\t}\n\n\t// manually assigned addresses are preferable to auto-assigned or generated addresses\n\tif a.Flags&syscall.IFA_F_PERMANENT == syscall.IFA_F_PERMANENT && b.Flags&syscall.IFA_F_PERMANENT == 0 {\n\t\treturn -1\n\t} else if a.Flags&syscall.IFA_F_PERMANENT == 0 && b.Flags&syscall.IFA_F_PERMANENT == syscall.IFA_F_PERMANENT {\n\t\treturn 1\n\t}\n\n\t// non-temporary address are preferable to temporary addresses\n\tif a.Flags&syscall.IFA_F_TEMPORARY == 0 && b.Flags&syscall.IFA_F_TEMPORARY == syscall.IFA_F_TEMPORARY {\n\t\treturn -1\n\t} else if a.Flags&syscall.IFA_F_TEMPORARY == syscall.IFA_F_TEMPORARY && b.Flags&syscall.IFA_F_TEMPORARY == 0 {\n\t\treturn 1\n\t}\n\n\t// anything else doesn't really matter\n\treturn 0\n}\n"
  },
  {
    "path": "pkg/ip/iface_test.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"net\"\n\t\"testing\"\n\n\t\"github.com/flannel-io/flannel/pkg/ns\"\n\t\"github.com/vishvananda/netlink\"\n)\n\nfunc TestEnsureV4AddressOnLink(t *testing.T) {\n\tteardown := ns.SetUpNetlinkTest(t)\n\tdefer teardown()\n\tlo, err := netlink.LinkByName(\"lo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := netlink.LinkSetUp(lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// check changing address\n\tipn := IP4Net{IP: FromIP(net.ParseIP(\"127.0.0.2\")), PrefixLen: 24}\n\tif err := EnsureV4AddressOnLink(ipn, ipn, lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\taddrs, err := netlink.AddrList(lo, netlink.FAMILY_V4)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(addrs) != 1 || addrs[0].String() != \"127.0.0.2/24 lo\" {\n\t\tt.Fatalf(\"addrs %v is not expected\", addrs)\n\t}\n\n\t// check changing address if there exist unknown addresses\n\tif err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(\"127.0.1.1\"), Mask: net.CIDRMask(24, 32)}}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := EnsureV4AddressOnLink(ipn, ipn, lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\taddrs, err = netlink.AddrList(lo, netlink.FAMILY_V4)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(addrs) != 2 {\n\t\tt.Fatalf(\"two addresses expected, addrs: %v\", addrs)\n\t}\n}\n\nfunc TestEnsureV6AddressOnLink(t *testing.T) {\n\tteardown := ns.SetUpNetlinkTest(t)\n\tdefer teardown()\n\tlo, err := netlink.LinkByName(\"lo\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := netlink.LinkSetUp(lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// check changing address\n\tipn := IP6Net{IP: FromIP6(net.ParseIP(\"::2\")), PrefixLen: 64}\n\tif err := EnsureV6AddressOnLink(ipn, ipn, lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\taddrs, err := netlink.AddrList(lo, netlink.FAMILY_V6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(addrs) != 1 || addrs[0].String() != \"::2/64\" {\n\t\tt.Fatalf(\"v6 addrs %v is not expected\", addrs)\n\t}\n\n\t// check changing address if there exist multiple addresses\n\tif err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP(\"2001::4\"), Mask: net.CIDRMask(64, 128)}}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\taddrs, err = netlink.AddrList(lo, netlink.FAMILY_V6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(addrs) != 2 {\n\t\tt.Fatalf(\"two addresses expected, addrs: %v\", addrs)\n\t}\n\tif err := EnsureV6AddressOnLink(ipn, ipn, lo); err != nil {\n\t\tt.Fatal(err)\n\t}\n\taddrs, err = netlink.AddrList(lo, netlink.FAMILY_V6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(addrs) != 1 {\n\t\tt.Fatalf(\"only one address expected, addrs: %v\", addrs)\n\t}\n}\n"
  },
  {
    "path": "pkg/ip/iface_windows.go",
    "content": "//go:build windows\n// +build windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\n\t\"github.com/flannel-io/flannel/pkg/powershell\"\n)\n\n// GetInterfaceIP4Addrs returns the IPv4 address for the given network interface\nfunc GetInterfaceIP4Addrs(iface *net.Interface) ([]net.IP, error) {\n\taddrs, err := iface.Addrs()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tipAddrs := make([]net.IP, 0)\n\n\tfor _, addr := range addrs {\n\t\tvar ip net.IP\n\t\tswitch v := addr.(type) {\n\t\tcase *net.IPAddr:\n\t\t\tip = v.IP\n\t\tcase *net.IPNet:\n\t\t\tip = v.IP\n\t\t}\n\n\t\tif ip != nil && ip.To4() != nil {\n\t\t\tipAddrs = append(ipAddrs, ip)\n\t\t}\n\t}\n\n\tif len(ipAddrs) > 0 {\n\t\treturn ipAddrs, nil\n\t}\n\n\treturn nil, errors.New(\"no IPv4 address found for given interface\")\n}\n\n// GetDefaultGatewayInterface returns the first network interface found with a default gateway set\nfunc GetDefaultGatewayInterface() (*net.Interface, error) {\n\tindex, err := getDefaultGatewayInterfaceIndex()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn net.InterfaceByIndex(index)\n}\n\nfunc getDefaultGatewayInterfaceIndex() (int, error) {\n\tpowerShellJsonData := struct {\n\t\tIfIndex int `json:\"ifIndex\"`\n\t}{-1}\n\n\terr := powershell.RunCommandWithJsonResult(\"Get-NetRoute | Where { $_.DestinationPrefix -eq '0.0.0.0/0' } | Select-Object -Property ifIndex\", &powerShellJsonData)\n\tif err != nil {\n\t\treturn -1, err\n\t}\n\n\tif powerShellJsonData.IfIndex < 0 {\n\t\treturn -1, errors.New(\"unable to find default gateway interface index\")\n\t}\n\n\treturn powerShellJsonData.IfIndex, nil\n}\n\n// GetInterfaceByIP tries to get the network interface with the given ip address\nfunc GetInterfaceByIP(search net.IP) (*net.Interface, error) {\n\tifaces, err := net.Interfaces()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, i := range ifaces {\n\t\taddrs, err := i.Addrs()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tfor _, addr := range addrs {\n\t\t\tvar ip net.IP\n\t\t\tswitch v := addr.(type) {\n\t\t\tcase *net.IPNet:\n\t\t\t\tip = v.IP\n\t\t\tcase *net.IPAddr:\n\t\t\t\tip = v.IP\n\t\t\t}\n\n\t\t\tif ip != nil && ip.Equal(search) {\n\t\t\t\treturn &i, nil\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil, errors.New(\"no interface with given IP found\")\n}\n\n// EnableForwardingForInterface enables forwarding for given interface.\n// The process must run with elevated rights. Otherwise the function will fail with an \"Access Denied\" error.\nfunc EnableForwardingForInterface(iface *net.Interface) error {\n\treturn setForwardingForInterface(iface, true)\n}\n\n// DisableForwardingForInterface disables forwarding for given interface.\n// The process must run with elevated rights. Otherwise the function will fail with an \"Access Denied\" error.\nfunc DisableForwardingForInterface(iface *net.Interface) error {\n\treturn setForwardingForInterface(iface, false)\n}\n\nfunc setForwardingForInterface(iface *net.Interface, forwarding bool) error {\n\tvalue := \"Enabled\"\n\tif !forwarding {\n\t\tvalue = \"Disabled\"\n\t}\n\n\t_, err := powershell.RunCommandf(\"Set-NetIPInterface -ifIndex %d -AddressFamily IPv4 -Forwarding %s\", iface.Index, value)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc IsForwardingEnabledForInterface(iface *net.Interface) (bool, error) {\n\tpowerShellJsonData := struct {\n\t\tForwarding int `json:\"Forwarding\"`\n\t}{0}\n\n\terr := powershell.RunCommandWithJsonResult(fmt.Sprintf(\"Get-NetIPInterface -ifIndex %d -AddressFamily IPv4 | Select-Object -Property Forwarding\", iface.Index), &powerShellJsonData)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn powerShellJsonData.Forwarding == 1, nil\n}\n\nfunc GetInterfaceByIP6(ip net.IP) (*net.Interface, error)                       { return nil, nil }\nfunc GetInterfaceIP6Addrs(iface *net.Interface) ([]net.IP, error)               { return nil, nil }\nfunc GetInterfaceBySpecificIPRouting(ip net.IP) (*net.Interface, net.IP, error) { return nil, nil, nil }\nfunc GetDefaultV6GatewayInterface() (*net.Interface, error)                     { return nil, nil }\nfunc AddBlackholeV4Route(ipV4Dest *net.IPNet) error                             { return nil }\nfunc AddBlackholeV6Route(ipV6Dest *net.IPNet) error                             { return nil }\n"
  },
  {
    "path": "pkg/ip/iface_windows_test.go",
    "content": "//go:build windows\n// +build windows\n\n// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"testing\"\n)\n\nfunc TestGetInterfaceIP4Addr(t *testing.T) {\n\tiface, err := GetDefaultGatewayInterface()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t_, err = GetInterfaceIP4Addrs(iface)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestGetDefaultGatewayInterface(t *testing.T) {\n\t_, err := GetDefaultGatewayInterface()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestGetInterfaceByIP(t *testing.T) {\n\tdefaultIface, err := GetDefaultGatewayInterface() // use default gateway interface for test\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefaultIpv4Addr, err := GetInterfaceIP4Addrs(defaultIface)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, addr := range defaultIpv4Addr {\n\t\tiface, err := GetInterfaceByIP(addr)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif iface.Index != defaultIface.Index {\n\t\t\tt.Fatalf(\"iface.Index(%d) != defaultIface.Index(%d)\", iface.Index, defaultIface.Index)\n\t\t}\n\t}\n}\n\nfunc TestEnableForwardingForInterface(t *testing.T) {\n\tiface, err := GetDefaultGatewayInterface() // use default gateway interface for test\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = EnableForwardingForInterface(iface)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer func() { _ = DisableForwardingForInterface(iface) }() // try to reset forwarding\n\n\tenabled, err := IsForwardingEnabledForInterface(iface)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !enabled {\n\t\tt.Fatal(\"enabled == false\")\n\t}\n}\n"
  },
  {
    "path": "pkg/ip/ip6net.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/big\"\n\t\"net\"\n)\n\ntype IP6 big.Int\n\nfunc FromIP16Bytes(ip []byte) *IP6 {\n\treturn (*IP6)(big.NewInt(0).SetBytes(ip))\n}\n\nfunc FromIP6(ip net.IP) *IP6 {\n\tipv6 := ip.To16()\n\n\tif ipv6 == nil {\n\t\tpanic(\"Address is not an IPv6 address\")\n\t}\n\n\treturn FromIP16Bytes(ipv6)\n}\n\nfunc ParseIP6(s string) (*IP6, error) {\n\tip := net.ParseIP(s)\n\tif ip == nil {\n\t\treturn (*IP6)(big.NewInt(0)), errors.New(\"invalid IP address format\")\n\t}\n\treturn FromIP6(ip), nil\n}\n\nfunc Mask(prefixLen int) *big.Int {\n\tmask := net.CIDRMask(prefixLen, 128)\n\treturn big.NewInt(0).SetBytes(mask)\n}\n\nfunc IsEmpty(subnet *IP6) bool {\n\tif subnet == nil || (*big.Int)(subnet).Cmp(big.NewInt(0)) == 0 {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc GetIPv6SubnetMin(networkIP *IP6, subnetSize *big.Int) *IP6 {\n\treturn (*IP6)(big.NewInt(0).Add((*big.Int)(networkIP), subnetSize))\n}\n\nfunc GetIPv6SubnetMax(networkIP *IP6, subnetSize *big.Int) *IP6 {\n\treturn (*IP6)(big.NewInt(0).Sub((*big.Int)(networkIP), subnetSize))\n}\n\nfunc CheckIPv6Subnet(subnetIP *IP6, mask *big.Int) bool {\n\treturn (*big.Int)(subnetIP).Cmp(big.NewInt(0).And((*big.Int)(subnetIP), mask)) == 0\n}\n\nfunc MustParseIP6(s string) *IP6 {\n\tip, err := ParseIP6(s)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn ip\n}\n\nfunc (ip6 *IP6) ToIP() net.IP {\n\tip := net.IP((*big.Int)(ip6).Bytes())\n\tif ip.To4() != nil {\n\t\treturn ip\n\t}\n\ta := (*big.Int)(ip6).FillBytes(make([]byte, 16))\n\treturn a\n}\n\nfunc (ip6 IP6) String() string {\n\treturn ip6.ToIP().String()\n}\n\n// MarshalJSON: json.Marshaler impl\nfunc (ip6 IP6) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`\"%s\"`, ip6)), nil\n}\n\n// UnmarshalJSON: json.Unmarshaler impl\nfunc (ip6 *IP6) UnmarshalJSON(j []byte) error {\n\tj = bytes.Trim(j, \"\\\"\")\n\tif val, err := ParseIP6(string(j)); err != nil {\n\t\treturn err\n\t} else {\n\t\t*ip6 = *val\n\t\treturn nil\n\t}\n}\n\nfunc (ip6 *IP6) Cmp(other *IP6) int {\n\treturn (*big.Int)(ip6).Cmp((*big.Int)(other))\n}\n\n// IsPrivate reports whether ip is a private address, according to\n// RFC 4193 (IPv6 addresses).\nfunc (ip6 *IP6) IsPrivate() bool {\n\treturn (*big.Int)(ip6).Bytes()[0]&0xfe == 0xfc\n}\n\n// similar to net.IPNet but has uint based representation\ntype IP6Net struct {\n\tIP        *IP6\n\tPrefixLen uint\n}\n\nfunc (n IP6Net) String() string {\n\tif n.IP == nil {\n\t\tn.IP = (*IP6)(big.NewInt(0))\n\t}\n\treturn fmt.Sprintf(\"%s/%d\", n.IP.String(), n.PrefixLen)\n}\n\nfunc (n IP6Net) StringSep(hexSep, prefixSep string) string {\n\treturn fmt.Sprintf(\"%s%s%d\", n.IP.String(), prefixSep, n.PrefixLen)\n}\n\nfunc MapIP6ToString(nws []IP6Net) []string {\n\tres := make([]string, len(nws))\n\tfor i := range nws {\n\t\tres[i] = nws[i].String()\n\t}\n\treturn res\n}\n\nfunc (n IP6Net) Network() IP6Net {\n\tmask := net.CIDRMask(int(n.PrefixLen), 128)\n\treturn IP6Net{\n\t\tFromIP6(n.IP.ToIP().Mask(mask)),\n\t\tn.PrefixLen,\n\t}\n}\n\nfunc (n IP6Net) Next() IP6Net {\n\treturn IP6Net{\n\t\t(*IP6)(big.NewInt(0).Add((*big.Int)(n.IP), big.NewInt(0).Lsh(big.NewInt(1), 128-n.PrefixLen))),\n\t\tn.PrefixLen,\n\t}\n}\n\n// IncrementIP() increments the IP of IP6Net CIDR by 1\nfunc (n *IP6Net) IncrementIP() {\n\tn.IP = (*IP6)(big.NewInt(0).Add((*big.Int)(n.IP), big.NewInt(1)))\n}\n\nfunc FromIP6Net(n *net.IPNet) IP6Net {\n\tprefixLen, _ := n.Mask.Size()\n\treturn IP6Net{\n\t\tFromIP6(n.IP),\n\t\tuint(prefixLen),\n\t}\n}\n\nfunc (n IP6Net) ToIPNet() *net.IPNet {\n\treturn &net.IPNet{\n\t\tIP:   n.IP.ToIP(),\n\t\tMask: net.CIDRMask(int(n.PrefixLen), 128),\n\t}\n}\n\nfunc (n IP6Net) Overlaps(other IP6Net) bool {\n\tvar mask *big.Int\n\tif n.PrefixLen < other.PrefixLen {\n\t\tmask = n.Mask()\n\t} else {\n\t\tmask = other.Mask()\n\t}\n\treturn (IP6)(*big.NewInt(0).And((*big.Int)(n.IP), mask)).String() ==\n\t\t(IP6)(*big.NewInt(0).And((*big.Int)(other.IP), mask)).String()\n}\n\nfunc (n IP6Net) Equal(other IP6Net) bool {\n\treturn ((*big.Int)(n.IP).Cmp((*big.Int)(other.IP)) == 0) &&\n\t\tn.PrefixLen == other.PrefixLen\n}\n\nfunc (n IP6Net) Mask() *big.Int {\n\tmask := net.CIDRMask(int(n.PrefixLen), 128)\n\treturn big.NewInt(0).SetBytes(mask)\n}\n\nfunc (n IP6Net) Contains(ip *IP6) bool {\n\tnetwork := big.NewInt(0).And((*big.Int)(n.IP), n.Mask())\n\tsubnet := big.NewInt(0).And((*big.Int)(ip), n.Mask())\n\treturn (IP6)(*network).String() == (IP6)(*subnet).String()\n}\n\nfunc (n *IP6Net) ContainsCIDR(other *IP6Net) bool {\n\tones1 := n.Mask()\n\tones2 := other.Mask()\n\treturn ones1.Cmp(ones2) <= 0 && n.Contains(other.IP)\n}\n\nfunc (n IP6Net) Empty() bool {\n\treturn IsEmpty(n.IP) && n.PrefixLen == uint(0)\n}\n\n// MarshalJSON: json.Marshaler impl\nfunc (n IP6Net) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`\"%s\"`, n)), nil\n}\n\n// UnmarshalJSON: json.Unmarshaler impl\nfunc (n *IP6Net) UnmarshalJSON(j []byte) error {\n\tj = bytes.Trim(j, \"\\\"\")\n\tif _, val, err := net.ParseCIDR(string(j)); err != nil {\n\t\treturn err\n\t} else {\n\t\t*n = FromIP6Net(val)\n\t\treturn nil\n\t}\n}\n"
  },
  {
    "path": "pkg/ip/ip6net_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"testing\"\n)\n\nfunc mkIP6Net(s string, plen uint) IP6Net {\n\tip, err := ParseIP6(s)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn IP6Net{ip, plen}\n}\n\nfunc mkIP6(s string) *IP6 {\n\tip, err := ParseIP6(s)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn ip\n}\n\nfunc TestIP6(t *testing.T) {\n\tnip := net.ParseIP(\"fc00::1\")\n\tip := FromIP6(nip)\n\tipStr := ip.String()\n\tif ipStr != \"fc00::1\" {\n\t\tt.Error(\"FromIP6 failed\")\n\t}\n\n\tnip = net.ParseIP(\"::\")\n\tip = FromIP6(nip)\n\tipStr = ip.String()\n\tif ipStr != \"::\" {\n\t\tt.Error(\":: FromIP6 failed\")\n\t}\n\n\tif !IsEmpty(ip) {\n\t\tt.Error(\"IsEmpty failed\")\n\t}\n\n\tip, err := ParseIP6(\"fc00::1\")\n\tif err != nil {\n\t\tt.Error(\"ParseIP6 failed with: \", err)\n\t} else {\n\t\tipStr := ip.String()\n\t\tif ipStr != \"fc00::1\" {\n\t\t\tt.Error(\"ParseIP6 failed\")\n\t\t}\n\t}\n\n\tif ip.ToIP().String() != \"fc00::1\" {\n\t\tt.Error(\"ToIP failed\")\n\t}\n\n\tj, err := json.Marshal(ip)\n\tif err != nil {\n\t\tt.Error(\"Marshal of IP6 failed: \", err)\n\t} else if string(j) != `\"fc00::1\"` {\n\t\tt.Error(\"Marshal of IP6 failed with unexpected value: \", j)\n\t}\n\n\taddresses := []*struct {\n\t\tip      string\n\t\tprivate bool\n\t}{\n\t\t{\"fc00::1\", true},\n\t\t{\"fcff::1\", true},\n\t\t{\"fd00::1\", true},\n\t\t{\"fdff::1\", true},\n\n\t\t{\"2001::\", false},\n\t\t{\"fe00::\", false},\n\t}\n\n\tfor _, address := range addresses {\n\t\tip := mkIP6(address.ip)\n\t\tis_private := ip.IsPrivate()\n\t\tif is_private != address.private {\n\t\t\tt.Errorf(\"%v misdetected expected private=%v got private=%v\", address.ip, address.private, is_private)\n\t\t}\n\t}\n}\n\nfunc TestIP6Net(t *testing.T) {\n\tvar n IP6Net\n\tif !n.Empty() {\n\t\tt.Error(\"Empty failed\")\n\t}\n\n\tn = mkIP6Net(\"::\", 0)\n\tif !n.Empty() {\n\t\tt.Error(\"::/0 Empty failed\")\n\t}\n\n\tn = mkIP6Net(\"::\", 64)\n\tif n.Empty() {\n\t\tt.Error(\"::/64 Empty failed\")\n\t}\n\n\tn1 := mkIP6Net(\"fc00:1::\", 64)\n\tif n1.Empty() {\n\t\tt.Error(\"fc00:1::/64 Empty failed\")\n\t}\n\n\tif n1.ToIPNet().String() != \"fc00:1::/64\" {\n\t\tt.Error(\"ToIPNet failed\")\n\t}\n\n\tif !n1.Overlaps(n1) {\n\t\tt.Errorf(\"%s does not overlap %s\", n1, n1)\n\t}\n\n\tn2 := mkIP6Net(\"fc00::\", 16)\n\tif !n1.Overlaps(n2) {\n\t\tt.Errorf(\"%s does not overlap %s\", n1, n2)\n\t}\n\n\tn2 = mkIP6Net(\"fc00:2::\", 64)\n\tif n1.Overlaps(n2) {\n\t\tt.Errorf(\"%s overlaps %s\", n1, n2)\n\t}\n\n\tn2 = mkIP6Net(\"fb00:2::\", 48)\n\tif n1.Overlaps(n2) {\n\t\tt.Errorf(\"%s overlaps %s\", n1, n2)\n\t}\n\n\tif !n1.Contains(mkIP6(\"fc00:1::\")) {\n\t\tt.Error(\"Contains failed\")\n\t}\n\n\tif !n1.Contains(mkIP6(\"fc00:1::1\")) {\n\t\tt.Error(\"Contains failed\")\n\t}\n\n\tif n1.Contains(mkIP6(\"fc00:2::\")) {\n\t\tt.Error(\"Contains failed\")\n\t}\n\n\tj, err := json.Marshal(n1)\n\tif err != nil {\n\t\tt.Error(\"Marshal of IP6Net failed: \", err)\n\t} else if string(j) != `\"fc00:1::/64\"` {\n\t\tt.Error(\"Marshal of IP6Net failed with unexpected value: \", j)\n\t}\n\n\tn1.IncrementIP()\n\tif n1.String() != \"fc00:1::1/64\" {\n\t\tt.Error(\"IncrementIP() failed\")\n\t}\n}\n"
  },
  {
    "path": "pkg/ip/ipnet.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n)\n\ntype IP4 uint32\n\nfunc FromBytes(ip []byte) IP4 {\n\treturn IP4(uint32(ip[3]) |\n\t\t(uint32(ip[2]) << 8) |\n\t\t(uint32(ip[1]) << 16) |\n\t\t(uint32(ip[0]) << 24))\n}\n\nfunc FromIP(ip net.IP) IP4 {\n\tipv4 := ip.To4()\n\n\tif ipv4 == nil {\n\t\tpanic(\"Address is not an IPv4 address\")\n\t}\n\n\treturn FromBytes(ipv4)\n}\n\nfunc ParseIP4(s string) (IP4, error) {\n\tip := net.ParseIP(s)\n\tif ip == nil {\n\t\treturn IP4(0), errors.New(\"invalid IP address format\")\n\t}\n\treturn FromIP(ip), nil\n}\n\nfunc MustParseIP4(s string) IP4 {\n\tip, err := ParseIP4(s)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn ip\n}\n\nfunc (ip IP4) Octets() (a, b, c, d byte) {\n\ta, b, c, d = byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip)\n\treturn\n}\n\nfunc (ip IP4) ToIP() net.IP {\n\treturn net.IPv4(ip.Octets())\n}\n\nfunc (ip IP4) NetworkOrder() uint32 {\n\tif NativelyLittle() {\n\t\ta, b, c, d := byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip)\n\t\treturn uint32(a) | (uint32(b) << 8) | (uint32(c) << 16) | (uint32(d) << 24)\n\t} else {\n\t\treturn uint32(ip)\n\t}\n}\n\nfunc (ip IP4) String() string {\n\treturn ip.ToIP().String()\n}\n\nfunc (ip IP4) StringSep(sep string) string {\n\ta, b, c, d := ip.Octets()\n\treturn fmt.Sprintf(\"%d%s%d%s%d%s%d\", a, sep, b, sep, c, sep, d)\n}\n\n// MarshalJSON: json.Marshaler impl\nfunc (ip IP4) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`\"%s\"`, ip)), nil\n}\n\n// UnmarshalJSON: json.Unmarshaler impl\nfunc (ip *IP4) UnmarshalJSON(j []byte) error {\n\tj = bytes.Trim(j, \"\\\"\")\n\tif val, err := ParseIP4(string(j)); err != nil {\n\t\treturn err\n\t} else {\n\t\t*ip = val\n\t\treturn nil\n\t}\n}\n\n// IsPrivate reports whether ip is a private address, according to\n// RFC 1918 (IPv4 addresses)\nfunc (ip IP4) IsPrivate() bool {\n\ta, b, _, _ := ip.Octets()\n\n\treturn a == 10 || (a == 172 && b&0xf0 == 16) || (a == 192 && b == 168)\n}\n\n// similar to net.IPNet but has uint based representation\ntype IP4Net struct {\n\tIP        IP4\n\tPrefixLen uint\n}\n\nfunc (n IP4Net) String() string {\n\treturn fmt.Sprintf(\"%s/%d\", n.IP.String(), n.PrefixLen)\n}\n\nfunc (n IP4Net) StringSep(octetSep, prefixSep string) string {\n\treturn fmt.Sprintf(\"%s%s%d\", n.IP.StringSep(octetSep), prefixSep, n.PrefixLen)\n}\n\nfunc MapIP4ToString(nws []IP4Net) []string {\n\tres := make([]string, len(nws))\n\tfor i := range nws {\n\t\tres[i] = nws[i].String()\n\t}\n\treturn res\n}\n\nfunc (n IP4Net) Network() IP4Net {\n\treturn IP4Net{\n\t\tn.IP & IP4(n.Mask()),\n\t\tn.PrefixLen,\n\t}\n}\n\nfunc (n IP4Net) Next() IP4Net {\n\treturn IP4Net{\n\t\tn.IP + (1 << (32 - n.PrefixLen)),\n\t\tn.PrefixLen,\n\t}\n}\n\n// IncrementIP() increments the IP of IP4Net CIDR by 1\nfunc (n *IP4Net) IncrementIP() {\n\tn.IP++\n}\n\nfunc FromIPNet(n *net.IPNet) IP4Net {\n\tprefixLen, _ := n.Mask.Size()\n\treturn IP4Net{\n\t\tFromIP(n.IP),\n\t\tuint(prefixLen),\n\t}\n}\n\nfunc (n IP4Net) ToIPNet() *net.IPNet {\n\treturn &net.IPNet{\n\t\tIP:   n.IP.ToIP(),\n\t\tMask: net.CIDRMask(int(n.PrefixLen), 32),\n\t}\n}\n\nfunc (n IP4Net) Overlaps(other IP4Net) bool {\n\tvar mask uint32\n\tif n.PrefixLen < other.PrefixLen {\n\t\tmask = n.Mask()\n\t} else {\n\t\tmask = other.Mask()\n\t}\n\treturn (uint32(n.IP) & mask) == (uint32(other.IP) & mask)\n}\n\nfunc (n IP4Net) Equal(other IP4Net) bool {\n\treturn n.IP == other.IP && n.PrefixLen == other.PrefixLen\n}\n\nfunc (n IP4Net) Mask() uint32 {\n\tvar ones uint32 = 0xFFFFFFFF\n\treturn ones << (32 - n.PrefixLen)\n}\n\nfunc (n IP4Net) Contains(ip IP4) bool {\n\treturn (uint32(n.IP) & n.Mask()) == (uint32(ip) & n.Mask())\n}\n\nfunc (n *IP4Net) ContainsCIDR(other *IP4Net) bool {\n\tones1 := n.Mask()\n\tones2 := other.Mask()\n\treturn ones1 <= ones2 && n.Contains(other.IP)\n}\n\nfunc (n IP4Net) Empty() bool {\n\treturn n.IP == IP4(0) && n.PrefixLen == uint(0)\n}\n\n// MarshalJSON: json.Marshaler impl\nfunc (n IP4Net) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`\"%s\"`, n)), nil\n}\n\n// UnmarshalJSON: json.Unmarshaler impl\nfunc (n *IP4Net) UnmarshalJSON(j []byte) error {\n\tj = bytes.Trim(j, \"\\\"\")\n\tif _, val, err := net.ParseCIDR(string(j)); err != nil {\n\t\treturn err\n\t} else {\n\t\t*n = FromIPNet(val)\n\t\treturn nil\n\t}\n}\n"
  },
  {
    "path": "pkg/ip/ipnet_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ip\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"testing\"\n)\n\nfunc mkIP4Net(s string, plen uint) IP4Net {\n\tip, err := ParseIP4(s)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn IP4Net{ip, plen}\n}\n\nfunc mkIP4(s string) IP4 {\n\tip, err := ParseIP4(s)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn ip\n}\n\nfunc TestIP4(t *testing.T) {\n\tnip := net.ParseIP(\"1.2.3.4\")\n\tip := FromIP(nip)\n\ta, b, c, d := ip.Octets()\n\tif a != 1 || b != 2 || c != 3 || d != 4 {\n\t\tt.Error(\"FromIP failed\")\n\t}\n\n\tip, err := ParseIP4(\"1.2.3.4\")\n\tif err != nil {\n\t\tt.Error(\"ParseIP4 failed with: \", err)\n\t} else {\n\t\ta, b, c, d := ip.Octets()\n\t\tif a != 1 || b != 2 || c != 3 || d != 4 {\n\t\t\tt.Error(\"ParseIP4 failed\")\n\t\t}\n\t}\n\n\tif ip.ToIP().String() != \"1.2.3.4\" {\n\t\tt.Error(\"ToIP failed\")\n\t}\n\n\tif ip.String() != \"1.2.3.4\" {\n\t\tt.Error(\"String failed\")\n\t}\n\n\tif ip.StringSep(\"*\") != \"1*2*3*4\" {\n\t\tt.Error(\"StringSep failed\")\n\t}\n\n\tj, err := json.Marshal(ip)\n\tif err != nil {\n\t\tt.Error(\"Marshal of IP4 failed: \", err)\n\t} else if string(j) != `\"1.2.3.4\"` {\n\t\tt.Error(\"Marshal of IP4 failed with unexpected value: \", j)\n\t}\n\n\taddresses := []*struct {\n\t\tip      string\n\t\tprivate bool\n\t}{\n\t\t{\"192.168.0.1\", true},\n\t\t{\"172.16.0.1\", true},\n\t\t{\"172.31.0.1\", true},\n\t\t{\"10.1.2.3\", true},\n\n\t\t{\"8.8.8.8\", false},\n\t\t{\"172.32.0.1\", false},\n\t\t{\"192.167.0.1\", false},\n\t\t{\"192.169.0.1\", false},\n\t}\n\n\tfor _, address := range addresses {\n\t\tip := mkIP4(address.ip)\n\t\tis_private := ip.IsPrivate()\n\t\tif is_private != address.private {\n\t\t\tt.Errorf(\"%v misdetected expected private: %v got private: %v\", address.ip, address.private, is_private)\n\t\t}\n\t}\n}\n\nfunc TestIP4Net(t *testing.T) {\n\tn1 := mkIP4Net(\"1.2.3.0\", 24)\n\n\tif n1.ToIPNet().String() != \"1.2.3.0/24\" {\n\t\tt.Error(\"ToIPNet failed\")\n\t}\n\n\tif !n1.Overlaps(n1) {\n\t\tt.Errorf(\"%s does not overlap %s\", n1, n1)\n\t}\n\n\tn2 := mkIP4Net(\"1.2.0.0\", 16)\n\tif !n1.Overlaps(n2) {\n\t\tt.Errorf(\"%s does not overlap %s\", n1, n2)\n\t}\n\n\tn2 = mkIP4Net(\"1.2.4.0\", 24)\n\tif n1.Overlaps(n2) {\n\t\tt.Errorf(\"%s overlaps %s\", n1, n2)\n\t}\n\n\tn2 = mkIP4Net(\"7.2.4.0\", 22)\n\tif n1.Overlaps(n2) {\n\t\tt.Errorf(\"%s overlaps %s\", n1, n2)\n\t}\n\n\tif !n1.Contains(mkIP4(\"1.2.3.0\")) {\n\t\tt.Error(\"Contains failed\")\n\t}\n\n\tif !n1.Contains(mkIP4(\"1.2.3.4\")) {\n\t\tt.Error(\"Contains failed\")\n\t}\n\n\tif n1.Contains(mkIP4(\"1.2.4.0\")) {\n\t\tt.Error(\"Contains failed\")\n\t}\n\n\tj, err := json.Marshal(n1)\n\tif err != nil {\n\t\tt.Error(\"Marshal of IP4Net failed: \", err)\n\t} else if string(j) != `\"1.2.3.0/24\"` {\n\t\tt.Error(\"Marshal of IP4Net failed with unexpected value: \", j)\n\t}\n\n\tn1.IncrementIP()\n\tif n1.String() != \"1.2.3.1/24\" {\n\t\tt.Error(\"IncrementIP() failed\")\n\t}\n}\n"
  },
  {
    "path": "pkg/ip/tun.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage ip\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nconst (\n\ttunDevice  = \"/dev/net/tun\"\n\tifnameSize = 16\n)\n\ntype ifreqFlags struct {\n\tIfrnName  [ifnameSize]byte\n\tIfruFlags uint16\n}\n\nfunc ioctl(fd int, request, argp uintptr) error {\n\t_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(fd), request, argp)\n\tif errno != 0 {\n\t\treturn fmt.Errorf(\"ioctl failed with '%s'\", errno)\n\t}\n\treturn nil\n}\n\nfunc fromZeroTerm(s []byte) string {\n\treturn string(bytes.TrimRight(s, \"\\000\"))\n}\n\nfunc OpenTun(name string) (*os.File, string, error) {\n\ttun, err := os.OpenFile(tunDevice, os.O_RDWR, 0)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tvar ifr ifreqFlags\n\tcopy(ifr.IfrnName[:len(ifr.IfrnName)-1], []byte(name+\"\\000\"))\n\tifr.IfruFlags = syscall.IFF_TUN | syscall.IFF_NO_PI\n\n\terr = ioctl(int(tun.Fd()), syscall.TUNSETIFF, uintptr(unsafe.Pointer(&ifr)))\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tifname := fromZeroTerm(ifr.IfrnName[:ifnameSize])\n\treturn tun, ifname, nil\n}\n"
  },
  {
    "path": "pkg/ipmatch/match.go",
    "content": "// Copyright 2022 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ipmatch\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/flannel-io/flannel/pkg/backend\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\tipv4Stack int = iota\n\tipv6Stack\n\tdualStack\n\tnoneStack\n)\n\ntype PublicIPOpts struct {\n\tPublicIP   string\n\tPublicIPv6 string\n}\n\nfunc GetIPFamily(autoDetectIPv4, autoDetectIPv6 bool) (int, error) {\n\tif autoDetectIPv4 && !autoDetectIPv6 {\n\t\treturn ipv4Stack, nil\n\t} else if !autoDetectIPv4 && autoDetectIPv6 {\n\t\treturn ipv6Stack, nil\n\t} else if autoDetectIPv4 && autoDetectIPv6 {\n\t\treturn dualStack, nil\n\t}\n\treturn noneStack, errors.New(\"none defined stack\")\n}\n\nfunc LookupExtIface(ifname string, ifregexS string, ifcanreach string, ipStack int, opts PublicIPOpts) (*backend.ExternalInterface, error) {\n\tvar iface *net.Interface\n\tvar ifaceAddr net.IP\n\tvar ifaceV6Addr net.IP\n\tvar err error\n\tvar ifregex *regexp.Regexp\n\n\tif ifregexS != \"\" {\n\t\tifregex, err = regexp.Compile(ifregexS)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not compile the IP address regex '%s': %w\", ifregexS, err)\n\t\t}\n\t}\n\n\t// Check ip family stack\n\tif ipStack == noneStack {\n\t\treturn nil, fmt.Errorf(\"none matched ip stack\")\n\t}\n\n\tif len(ifname) > 0 {\n\t\tif ifaceAddr = net.ParseIP(ifname); ifaceAddr != nil {\n\t\t\tlog.Infof(\"Searching for interface using %s\", ifaceAddr)\n\t\t\tswitch ipStack {\n\t\t\tcase ipv4Stack:\n\t\t\t\tiface, err = ip.GetInterfaceByIP(ifaceAddr)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"error looking up interface %s: %s\", ifname, err)\n\t\t\t\t}\n\t\t\tcase ipv6Stack:\n\t\t\t\tiface, err = ip.GetInterfaceByIP6(ifaceAddr)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"error looking up v6 interface %s: %s\", ifname, err)\n\t\t\t\t}\n\t\t\t\tifaceV6Addr = ifaceAddr\n\t\t\tcase dualStack:\n\t\t\t\tif ifaceAddr.To4() != nil {\n\t\t\t\t\tiface, err = ip.GetInterfaceByIP(ifaceAddr)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, fmt.Errorf(\"error looking up interface %s: %s\", ifname, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(opts.PublicIPv6) > 0 {\n\t\t\t\t\tif ifaceV6Addr = net.ParseIP(opts.PublicIPv6); ifaceV6Addr != nil {\n\t\t\t\t\t\tv6Iface, err := ip.GetInterfaceByIP6(ifaceV6Addr)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn nil, fmt.Errorf(\"error looking up v6 interface %s: %s\", opts.PublicIPv6, err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif ifaceAddr.To4() == nil {\n\t\t\t\t\t\t\tiface = v6Iface\n\t\t\t\t\t\t\tifaceAddr = nil\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tif iface.Name != v6Iface.Name {\n\t\t\t\t\t\t\t\treturn nil, fmt.Errorf(\"v6 interface %s must be the same with v4 interface %s\", v6Iface.Name, iface.Name)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tiface, err = net.InterfaceByName(ifname)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error looking up interface %s: %s\", ifname, err)\n\t\t\t}\n\t\t}\n\t} else if ifregex != nil {\n\t\t// Use the regex if specified and the iface option for matching a specific ip or name is not used\n\t\tifaces, err := net.Interfaces()\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error listing all interfaces: %s\", err)\n\t\t}\n\n\t\t// Check IP\n\tifaceLoop:\n\t\tfor _, ifaceToMatch := range ifaces {\n\t\t\tswitch ipStack {\n\t\t\tcase ipv4Stack:\n\t\t\t\tifaceIPs, err := ip.GetInterfaceIP4Addrs(&ifaceToMatch)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Skip if there is no IPv4 address\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif matched := matchIP(ifregex, ifaceIPs); matched != nil {\n\t\t\t\t\tifaceAddr = matched\n\t\t\t\t\tiface = &ifaceToMatch\n\t\t\t\t\tbreak ifaceLoop\n\t\t\t\t}\n\t\t\tcase ipv6Stack:\n\t\t\t\tifaceIPs, err := ip.GetInterfaceIP6Addrs(&ifaceToMatch)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Skip if there is no IPv6 address\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif matched := matchIP(ifregex, ifaceIPs); matched != nil {\n\t\t\t\t\tifaceV6Addr = matched\n\t\t\t\t\tiface = &ifaceToMatch\n\t\t\t\t\tbreak ifaceLoop\n\t\t\t\t}\n\t\t\tcase dualStack:\n\t\t\t\tifaceIPs, err := ip.GetInterfaceIP4Addrs(&ifaceToMatch)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Skip if there is no IPv4 address\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tifaceV6IPs, err := ip.GetInterfaceIP6Addrs(&ifaceToMatch)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// Skip if there is no IPv6 address\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tif matched := matchIP(ifregex, ifaceIPs); matched != nil {\n\t\t\t\t\tifaceAddr = matched\n\t\t\t\t} else {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif matched := matchIP(ifregex, ifaceV6IPs); matched != nil {\n\t\t\t\t\tifaceV6Addr = matched\n\t\t\t\t\tiface = &ifaceToMatch\n\t\t\t\t\tbreak ifaceLoop\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Check Name\n\t\tif iface == nil && (ifaceAddr == nil || ifaceV6Addr == nil) {\n\t\t\tfor _, ifaceToMatch := range ifaces {\n\t\t\t\tif ifregex.MatchString(ifaceToMatch.Name) {\n\t\t\t\t\tiface = &ifaceToMatch\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Check that nothing was matched\n\t\tif iface == nil {\n\t\t\tvar availableFaces []string\n\t\t\tfor _, f := range ifaces {\n\t\t\t\tvar ipaddr []net.IP\n\t\t\t\tswitch ipStack {\n\t\t\t\tcase ipv4Stack, dualStack:\n\t\t\t\t\tipaddr, _ = ip.GetInterfaceIP4Addrs(&f) // We can safely ignore errors. We just won't log any ip\n\t\t\t\tcase ipv6Stack:\n\t\t\t\t\tipaddr, _ = ip.GetInterfaceIP6Addrs(&f) // We can safely ignore errors. We just won't log any ip\n\t\t\t\t}\n\t\t\t\tavailableFaces = append(availableFaces, fmt.Sprintf(\"%s:%v\", f.Name, ipaddr))\n\t\t\t}\n\n\t\t\treturn nil, fmt.Errorf(\"could not match pattern %s to any of the available network interfaces (%s)\", ifregexS, strings.Join(availableFaces, \", \"))\n\t\t}\n\t} else if len(ifcanreach) > 0 {\n\t\tif runtime.GOOS == \"windows\" {\n\t\t\treturn nil, fmt.Errorf(\"ifcanreach is not supported on windows\")\n\t\t}\n\t\tlog.Info(\"Determining interface to use based on given ifcanreach: \", ifcanreach)\n\t\tif iface, ifaceAddr, err = ip.GetInterfaceBySpecificIPRouting(net.ParseIP(ifcanreach)); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to get ifcanreach based interface: %s\", err)\n\t\t}\n\t} else {\n\t\tlog.Info(\"Determining IP address of default interface\")\n\t\tswitch ipStack {\n\t\tcase ipv4Stack:\n\t\t\tif iface, err = ip.GetDefaultGatewayInterface(); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to get default interface: %w\", err)\n\t\t\t}\n\t\tcase ipv6Stack:\n\t\t\tif iface, err = ip.GetDefaultV6GatewayInterface(); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to get default v6 interface: %w\", err)\n\t\t\t}\n\t\tcase dualStack:\n\t\t\tif iface, err = ip.GetDefaultGatewayInterface(); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to get default interface: %w\", err)\n\t\t\t}\n\t\t\tv6Iface, err := ip.GetDefaultV6GatewayInterface()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to get default v6 interface: %w\", err)\n\t\t\t}\n\t\t\tif iface.Name != v6Iface.Name {\n\t\t\t\treturn nil, fmt.Errorf(\"v6 default route interface %s \"+\n\t\t\t\t\t\"must be the same with v4 default route interface %s\", v6Iface.Name, iface.Name)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar ifaceAddrs []net.IP\n\tvar ifaceV6Addrs []net.IP\n\tif ipStack == ipv4Stack && ifaceAddr == nil {\n\t\tifaceAddrs, err = ip.GetInterfaceIP4Addrs(iface)\n\t\tif err != nil || len(ifaceAddrs) == 0 {\n\t\t\treturn nil, fmt.Errorf(\"failed to find IPv4 address for interface %s\", iface.Name)\n\t\t}\n\t\tifaceAddr = ifaceAddrs[0]\n\t} else if ipStack == ipv6Stack && ifaceV6Addr == nil {\n\t\tifaceV6Addrs, err = ip.GetInterfaceIP6Addrs(iface)\n\t\tif err != nil || len(ifaceV6Addrs) == 0 {\n\t\t\treturn nil, fmt.Errorf(\"failed to find IPv6 address for interface %s\", iface.Name)\n\t\t}\n\t\tifaceV6Addr = ifaceV6Addrs[0]\n\t} else if ipStack == dualStack {\n\t\tif ifaceAddr == nil {\n\t\t\tifaceAddrs, err = ip.GetInterfaceIP4Addrs(iface)\n\t\t\tif err != nil || len(ifaceAddrs) == 0 {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to find IPv4 address for interface %s\", iface.Name)\n\t\t\t}\n\t\t\tifaceAddr = ifaceAddrs[0]\n\t\t}\n\n\t\tif ifaceV6Addr == nil {\n\t\t\tifaceV6Addrs, err = ip.GetInterfaceIP6Addrs(iface)\n\t\t\tif err != nil || len(ifaceV6Addrs) == 0 {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to find IPv6 address for interface %s\", iface.Name)\n\t\t\t}\n\t\t\tifaceV6Addr = ifaceV6Addrs[0]\n\t\t}\n\t}\n\n\tif ifaceAddr != nil {\n\t\tlog.Infof(\"Using interface with name %s and address %s\", iface.Name, ifaceAddr)\n\t}\n\tif ifaceV6Addr != nil {\n\t\tlog.Infof(\"Using interface with name %s and v6 address %s\", iface.Name, ifaceV6Addr)\n\t}\n\n\tif iface.MTU == 0 {\n\t\treturn nil, fmt.Errorf(\"failed to determine MTU for %s interface\", ifaceAddr)\n\t}\n\n\tvar extAddr net.IP\n\tvar extV6Addr net.IP\n\n\tif len(opts.PublicIP) > 0 {\n\t\textAddr = net.ParseIP(opts.PublicIP)\n\t\tif extAddr == nil {\n\t\t\treturn nil, fmt.Errorf(\"invalid public IP address: %s\", opts.PublicIP)\n\t\t}\n\t\tlog.Infof(\"Using %s as external address\", extAddr)\n\t}\n\n\tif extAddr == nil && ipStack != ipv6Stack {\n\t\tlog.Infof(\"Defaulting external address to interface address (%s)\", ifaceAddr)\n\t\textAddr = ifaceAddr\n\t}\n\n\tif len(opts.PublicIPv6) > 0 {\n\t\textV6Addr = net.ParseIP(opts.PublicIPv6)\n\t\tif extV6Addr == nil {\n\t\t\treturn nil, fmt.Errorf(\"invalid public IPv6 address: %s\", opts.PublicIPv6)\n\t\t}\n\t\tlog.Infof(\"Using %s as external address\", extV6Addr)\n\t}\n\n\tif extV6Addr == nil && ipStack != ipv4Stack {\n\t\tlog.Infof(\"Defaulting external v6 address to interface address (%s)\", ifaceV6Addr)\n\t\textV6Addr = ifaceV6Addr\n\t}\n\n\treturn &backend.ExternalInterface{\n\t\tIface:       iface,\n\t\tIfaceName:   iface.Name,\n\t\tIfaceAddr:   ifaceAddr,\n\t\tIfaceV6Addr: ifaceV6Addr,\n\t\tExtAddr:     extAddr,\n\t\tExtV6Addr:   extV6Addr,\n\t}, nil\n}\n\nfunc matchIP(ifregex *regexp.Regexp, ifaceIPs []net.IP) net.IP {\n\tfor _, ifaceIP := range ifaceIPs {\n\t\tif ifregex.MatchString(ifaceIP.String()) {\n\t\t\treturn ifaceIP\n\t\t}\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/ipmatch/match_test.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2022 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ipmatch\n\nimport (\n\t\"os/exec\"\n\t\"testing\"\n)\n\nfunc TestLookupExtIface(t *testing.T) {\n\tvar err error\n\n\texecOrFail := func(name string, arg ...string) {\n\t\terr = exec.Command(name, arg...).Run()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"exec ip link command failed.\\nmaybe you need adjust sudo config to allow user exec root command without input password.\\nerr=%v\", err)\n\t\t}\n\t}\n\texecOrFail(\"sudo\", \"ip\", \"link\", \"add\", \"name\", \"dummy0\", \"type\", \"dummy\")\n\texecOrFail(\"sudo\", \"ip\", \"addr\", \"add\", \"1.10.100.1\", \"dev\", \"dummy0\")\n\texecOrFail(\"sudo\", \"ip\", \"addr\", \"add\", \"192.168.200.128\", \"dev\", \"dummy0\")\n\texecOrFail(\"sudo\", \"ip\", \"addr\", \"add\", \"172.16.30.18\", \"dev\", \"dummy0\")\n\texecOrFail(\"sudo\", \"ip\", \"addr\", \"add\", \"172.16.31.200\", \"dev\", \"dummy0\")\n\texecOrFail(\"sudo\", \"ip\", \"addr\", \"add\", \"172.16.32.100\", \"dev\", \"dummy0\")\n\texecOrFail(\"sudo\", \"ip\", \"addr\", \"add\", \"2001:db8::1/64\", \"dev\", \"dummy0\")\n\texecOrFail(\"sudo\", \"ip\", \"link\", \"set\", \"dummy0\", \"up\")\n\texecOrFail(\"sudo\", \"ip\", \"route\", \"add\", \"172.16.32.254\", \"via\", \"172.16.32.100\", \"dev\", \"dummy0\")\n\n\tdefer func() {\n\t\t_ = exec.Command(\"sudo\", \"ip\", \"link\", \"set\", \"dummy0\", \"down\").Run()\n\t\t_ = exec.Command(\"sudo\", \"ip\", \"link\", \"delete\", \"dummy0\").Run()\n\t}()\n\n\tt.Run(\"ByIfRegexForIPv4\", func(t *testing.T) {\n\t\tbackendInterface, err := LookupExtIface(\"\", `192\\.168\\.200\\.\\d+`, \"\", ipv4Stack, PublicIPOpts{})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != \"dummy0\" {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", \"dummy0\", backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != \"192.168.200.128\" {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", \"192.168.200.128\", backendInterface.IfaceAddr.String())\n\t\t}\n\t})\n\n\tt.Run(\"ByIfRegexForName\", func(t *testing.T) {\n\t\tbackendInterface, err := LookupExtIface(\"\", `dummy\\d+`, \"\", ipv4Stack, PublicIPOpts{})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != \"dummy0\" {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", \"dummy0\", backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != \"1.10.100.1\" {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", \"1.10.100.1\", backendInterface.IfaceAddr.String())\n\t\t}\n\t})\n\n\tt.Run(\"ByName\", func(t *testing.T) {\n\t\tbackendInterface, err := LookupExtIface(\"dummy0\", \"\", \"\", ipv4Stack, PublicIPOpts{})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != \"dummy0\" {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", \"dummy0\", backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != \"1.10.100.1\" {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", \"1.10.100.1\", backendInterface.IfaceAddr.String())\n\t\t}\n\t})\n\n\tt.Run(\"ByIPv4\", func(t *testing.T) {\n\t\tbackendInterface, err := LookupExtIface(\"172.16.30.18\", \"\", \"\", ipv4Stack, PublicIPOpts{})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != \"dummy0\" {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", \"dummy0\", backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != \"172.16.30.18\" {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", \"172.16.30.18\", backendInterface.IfaceAddr.String())\n\t\t}\n\t})\n\n\tt.Run(\"ByIPv4DualStack\", func(t *testing.T) {\n\t\tbackendInterface, err := LookupExtIface(\"172.16.30.18\", \"\", \"\", dualStack, PublicIPOpts{})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != \"dummy0\" {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", \"dummy0\", backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != \"172.16.30.18\" {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", \"172.16.30.18\", backendInterface.IfaceAddr.String())\n\t\t}\n\t\tif backendInterface.IfaceV6Addr.String() != \"2001:db8::1/64\" {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", \"2001:db8::1/64\", backendInterface.IfaceV6Addr.String())\n\t\t}\n\t})\n\n\tt.Run(\"ByIfRegexMatchPublicIPv4\", func(t *testing.T) {\n\t\texpectedIfaceName := \"dummy0\"\n\t\texpectedIP := \"172.16.30.18\"\n\t\texpectedPublicIP := \"172.16.31.200\"\n\t\tbackendInterface, err := LookupExtIface(\"\", `172\\.16\\.30\\.\\d+`, \"\", ipv4Stack, PublicIPOpts{\n\t\t\tPublicIP: expectedPublicIP,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != \"dummy0\" {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", expectedIfaceName, backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != expectedIP {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", expectedIP, backendInterface.IfaceAddr.String())\n\t\t}\n\t\tif backendInterface.ExtAddr.String() != expectedPublicIP {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", expectedPublicIP, backendInterface.ExtAddr.String())\n\t\t}\n\t})\n\n\tt.Run(\"ByIfCanReach\", func(t *testing.T) {\n\t\texpectedIfaceName := \"dummy0\"\n\t\texpectedIP := \"172.16.32.100\"\n\t\tbackendInterface, err := LookupExtIface(\"\", \"\", \"172.16.32.254\", ipv4Stack, PublicIPOpts{})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"backendInterface=%+v iface=%+v\", backendInterface, *backendInterface.Iface)\n\n\t\tif backendInterface.Iface.Name != expectedIfaceName {\n\t\t\tt.Fatalf(\"iface name not equal, expected=%v actual=%v\", expectedIfaceName, backendInterface.Iface.Name)\n\t\t}\n\t\tif backendInterface.IfaceAddr.String() != expectedIP {\n\t\t\tt.Fatalf(\"iface addr not equal, expected=%v actual=%v\", expectedIP, backendInterface.IfaceAddr.String())\n\t\t}\n\t})\n}\n"
  },
  {
    "path": "pkg/lease/lease.go",
    "content": "// Copyright 2022 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage lease\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\tEventAdded EventType = iota\n\tEventRemoved\n)\n\ntype (\n\tEventType int\n\n\tEvent struct {\n\t\tType  EventType `json:\"type\"`\n\t\tLease Lease     `json:\"lease,omitempty\"`\n\t}\n)\n\n// LeaseAttrs includes extra information for the lease\ntype LeaseAttrs struct {\n\tPublicIP      ip.IP4\n\tPublicIPv6    *ip.IP6\n\tBackendType   string          `json:\",omitempty\"`\n\tBackendData   json.RawMessage `json:\",omitempty\"`\n\tBackendV6Data json.RawMessage `json:\",omitempty\"`\n}\n\n// Lease includes information about the lease\ntype Lease struct {\n\tEnableIPv4 bool\n\tEnableIPv6 bool\n\tSubnet     ip.IP4Net\n\tIPv6Subnet ip.IP6Net\n\tAttrs      LeaseAttrs\n\tExpiration time.Time\n\n\tAsof int64 //Only used in etcd\n}\n\ntype LeaseWatchResult struct {\n\t// Either Events or Snapshot will be set.  If Events is empty, it means\n\t// the cursor was out of range and Snapshot contains the current list\n\t// of items, even if empty.\n\tEvents   []Event     `json:\"events\"`\n\tSnapshot []Lease     `json:\"snapshot\"` //Only used in etcd\n\tCursor   interface{} `json:\"cursor\"`   //Only used in etcd\n}\n\ntype LeaseWatcher struct {\n\tOwnLease *Lease  //Lease with the subnet of the local node\n\tLeases   []Lease //Leases with subnets from other nodes\n}\n\nfunc (la *LeaseAttrs) String() string {\n\tvar buffer bytes.Buffer\n\tbuffer.WriteString(fmt.Sprintf(\"BackendType: %s, PublicIP: %s, \", la.BackendType, la.PublicIP.String()))\n\tif la.PublicIPv6 != nil {\n\t\tbuffer.WriteString(fmt.Sprintf(\"PublicIPv6: %s, \", la.PublicIPv6.String()))\n\t} else {\n\t\tbuffer.WriteString(\"PublicIPv6: (nil), \")\n\t}\n\tif la.BackendData != nil {\n\t\tj, err := json.Marshal(la.BackendData)\n\t\tif err != nil {\n\t\t\tbuffer.WriteString(\"BackendData: (nil), \")\n\t\t}\n\t\tbuffer.WriteString(fmt.Sprintf(\"BackendData: %s, \", string(j)))\n\t} else {\n\t\tbuffer.WriteString(\"BackendData: (nil), \")\n\t}\n\tif la.BackendV6Data != nil {\n\t\tj, err := json.Marshal(la.BackendV6Data)\n\t\tif err != nil {\n\t\t\tbuffer.WriteString(\"BackendV6Data: (nil)\")\n\t\t}\n\t\tbuffer.WriteString(fmt.Sprintf(\"BackendV6Data: %s\", string(j)))\n\t} else {\n\t\tbuffer.WriteString(\"BackendV6Data: (nil)\")\n\t}\n\treturn buffer.String()\n}\n\n// Reset is called by etcd-subnet when using a snapshot\nfunc (lw *LeaseWatcher) Reset(leases []Lease) []Event {\n\tbatch := []Event{}\n\n\tfor _, nl := range leases {\n\t\tfound := false\n\t\tif sameSubnet(nl.EnableIPv4, nl.EnableIPv6, *lw.OwnLease, nl) {\n\t\t\tcontinue\n\t\t}\n\n\t\tfor i, ol := range lw.Leases {\n\t\t\tif sameSubnet(ol.EnableIPv4, ol.EnableIPv6, ol, nl) {\n\t\t\t\tlw.Leases = append(lw.Leases[:i], lw.Leases[i+1:]...)\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif !found {\n\t\t\t// new lease\n\t\t\tbatch = append(batch, Event{EventAdded, nl})\n\t\t}\n\t}\n\n\tfor _, l := range lw.Leases {\n\t\tbatch = append(batch, Event{EventRemoved, l})\n\t}\n\n\t// copy the leases over (caution: don't just assign a slice)\n\tlw.Leases = make([]Lease, len(leases))\n\tcopy(lw.Leases, leases)\n\n\treturn batch\n}\n\n// Update reads the leases in the events and depending on Type, adds them or removes them\nfunc (lw *LeaseWatcher) Update(events []Event) []Event {\n\tbatch := []Event{}\n\n\tfor _, e := range events {\n\t\tif sameSubnet(e.Lease.EnableIPv4, e.Lease.EnableIPv6, *lw.OwnLease, e.Lease) {\n\t\t\tcontinue\n\t\t}\n\n\t\tswitch e.Type {\n\t\tcase EventAdded:\n\t\t\tbatch = append(batch, lw.add(&e.Lease))\n\n\t\tcase EventRemoved:\n\t\t\tbatch = append(batch, lw.remove(&e.Lease))\n\t\t}\n\t}\n\n\treturn batch\n}\n\n// add updates lw.Leases, adding the passed lease (either overwriting or appending). It makes lw.Leases a set\nfunc (lw *LeaseWatcher) add(lease *Lease) Event {\n\tfor i, l := range lw.Leases {\n\t\tif sameSubnet(l.EnableIPv4, l.EnableIPv6, l, *lease) {\n\t\t\tlw.Leases[i] = *lease\n\t\t\treturn Event{EventAdded, lw.Leases[i]}\n\t\t}\n\t}\n\n\tlw.Leases = append(lw.Leases, *lease)\n\n\treturn Event{EventAdded, lw.Leases[len(lw.Leases)-1]}\n}\n\n// remove updates lw.Leases, removing the passed lease\nfunc (lw *LeaseWatcher) remove(lease *Lease) Event {\n\tfor i, l := range lw.Leases {\n\t\tif sameSubnet(l.EnableIPv4, l.EnableIPv6, l, *lease) {\n\t\t\tlw.Leases = append(lw.Leases[:i], lw.Leases[i+1:]...)\n\t\t\treturn Event{EventRemoved, l}\n\t\t}\n\t}\n\n\tlog.Errorf(\"Removed subnet (%s) and ipv6 subnet (%s) were not found\", lease.Subnet, lease.IPv6Subnet)\n\treturn Event{EventRemoved, *lease}\n}\n\n// sameSubnet checks if the subnets are the same in ipv4-only, ipv6-only and dualStack cases\nfunc sameSubnet(ipv4Enabled, ipv6Enabled bool, firstLease, secondLease Lease) bool {\n\t// ipv4 only case\n\tif ipv4Enabled && !ipv6Enabled && firstLease.Subnet.Equal(secondLease.Subnet) {\n\t\treturn true\n\t}\n\t// ipv6 only case\n\tif !ipv4Enabled && ipv6Enabled && firstLease.IPv6Subnet.Equal(secondLease.IPv6Subnet) {\n\t\treturn true\n\t}\n\t// dualStack case\n\tif ipv4Enabled && ipv6Enabled && firstLease.Subnet.Equal(secondLease.Subnet) && firstLease.IPv6Subnet.Equal(secondLease.IPv6Subnet) {\n\t\treturn true\n\t}\n\t// etcd case\n\tif !ipv4Enabled && !ipv6Enabled && firstLease.Subnet.Equal(secondLease.Subnet) {\n\t\treturn true\n\t}\n\n\treturn false\n}\n"
  },
  {
    "path": "pkg/mac/mac.go",
    "content": "// Copyright 2021 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mac\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"net\"\n)\n\n// NewHardwareAddr generates a new random hardware (MAC) address, local and\n// unicast.\nfunc NewHardwareAddr() (net.HardwareAddr, error) {\n\thardwareAddr := make(net.HardwareAddr, 6)\n\tif _, err := rand.Read(hardwareAddr); err != nil {\n\t\treturn nil, fmt.Errorf(\"could not generate random MAC address: %w\", err)\n\t}\n\n\t// Ensure that address is locally administered and unicast.\n\thardwareAddr[0] = (hardwareAddr[0] & 0xfe) | 0x02\n\n\treturn hardwareAddr, nil\n}\n"
  },
  {
    "path": "pkg/mac/mac_test.go",
    "content": "// Copyright 2021 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mac\n\nimport (\n\t\"testing\"\n)\n\nfunc TestNewHardwareAddr(t *testing.T) {\n\t// Ignore the actual address, since it's random.\n\t// But an error should never be returned.\n\t_, err := NewHardwareAddr()\n\tif err != nil {\n\t\tt.Fatalf(\"err: %v\", err)\n\t}\n}\n"
  },
  {
    "path": "pkg/ns/ns.go",
    "content": "//go:build !windows\n// +build !windows\n\n// Copyright 2017 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ns\n\nimport (\n\t\"runtime\"\n\t\"testing\"\n\n\t\"github.com/vishvananda/netns\"\n)\n\nfunc SetUpNetlinkTest(t *testing.T) func() {\n\t// new temporary namespace so we don't pollute the host\n\t// lock thread since the namespace is thread local\n\truntime.LockOSThread()\n\tvar err error\n\tns, err := netns.New()\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create newns: %v\", err)\n\t}\n\n\treturn func() {\n\t\terr := ns.Close()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Failed to close netns: %v\", err)\n\t\t}\n\t\truntime.UnlockOSThread()\n\t}\n}\n"
  },
  {
    "path": "pkg/powershell/powershell.go",
    "content": "//go:build windows\n// +build windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage powershell\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\n// commandWrapper ensures that exceptions are written to stdout and the powershell process exit code is -1\nconst commandWrapper = `$ErrorActionPreference=\"Stop\";try { %s } catch { Write-Host $_; os.Exit(-1) }`\n\n// RunCommand executes a given powershell command.\n//\n// When the command throws a powershell exception, RunCommand will return the exception message as error.\nfunc RunCommand(command string) ([]byte, error) {\n\tcmd := exec.Command(\"powershell.exe\", \"-NoLogo\", \"-NoProfile\", \"-NonInteractive\", \"-Command\", fmt.Sprintf(commandWrapper, command))\n\n\tstdout, err := cmd.Output()\n\tif err != nil {\n\t\tif cmd.ProcessState.ExitCode() != 0 {\n\t\t\tmessage := fmt.Sprintf(\"error: %v while running the command: %v.\", err, cmd) + \" Command output: \" + strings.TrimSpace(string(stdout))\n\t\t\treturn []byte{}, errors.New(message)\n\t\t}\n\n\t\treturn []byte{}, err\n\t}\n\n\treturn stdout, nil\n}\n\n// RunCommandf executes a given powershell command. Command argument formats according to a format specifier (See fmt.Sprintf).\n//\n// When the command throws a powershell exception, RunCommandf will return the exception message as error.\nfunc RunCommandf(command string, a ...interface{}) ([]byte, error) {\n\treturn RunCommand(fmt.Sprintf(command, a...))\n}\n\n// RunCommandWithJsonResult executes a given powershell command.\n// The command will be wrapped with ConvertTo-Json.\n//\n// You can Wrap your command with @(<cmd>) to ensure that the returned json is an array\n//\n// When the command throws a powershell exception, RunCommandf will return the exception message as error.\nfunc RunCommandWithJsonResult(command string, v interface{}) error {\n\twrappedCommand := fmt.Sprintf(commandWrapper, \"ConvertTo-Json (%s)\")\n\twrappedCommand = fmt.Sprintf(wrappedCommand, command)\n\n\tstdout, err := RunCommandf(wrappedCommand)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error while executing command: %w\", err)\n\t}\n\n\terr = json.Unmarshal(stdout, v)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error while unmarshalling stdout: %w\", err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/retry/retry.go",
    "content": "// Copyright 2023 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage retry\n\nimport (\n\tlog \"k8s.io/klog/v2\"\n\n\tretry_v4 \"github.com/avast/retry-go/v4\"\n)\n\n// Do executes f until it does not return an error\n// By default, the number of attempts is 10 with increasing delay between each\nfunc Do(f func() error) error {\n\n\treturn retry_v4.Do(f,\n\t\tretry_v4.OnRetry(func(n uint, err error) {\n\t\t\tlog.Errorf(\"#%d: %s\\n\", n, err)\n\t\t}),\n\t)\n\n}\n"
  },
  {
    "path": "pkg/routing/router.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage routing\n\nimport (\n\t\"bytes\"\n\t\"net\"\n)\n\n// Router manages network routes\ntype Router interface {\n\t// GetAllRoutes returns all existing routes\n\tGetAllRoutes() ([]Route, error)\n\n\t// GetRoutesFromInterfaceToSubnet returns all routes from the given Interface to the given subnet\n\tGetRoutesFromInterfaceToSubnet(interfaceIndex int, destinationSubnet *net.IPNet) ([]Route, error)\n\n\t// CreateRoute creates a new route\n\tCreateRoute(interfaceIndex int, destinationSubnet *net.IPNet, gatewayAddress net.IP) error\n\n\t// DeleteRoute removes an existing route\n\tDeleteRoute(interfaceIndex int, destinationSubnet *net.IPNet, gatewayAddress net.IP) error\n}\n\n// Route present a specific route\ntype Route struct {\n\tInterfaceIndex    int\n\tDestinationSubnet *net.IPNet\n\tGatewayAddress    net.IP\n}\n\nfunc (r *Route) Equal(other Route) bool {\n\treturn r.DestinationSubnet.IP.Equal(other.DestinationSubnet.IP) && bytes.Equal(r.DestinationSubnet.Mask, other.DestinationSubnet.Mask) && r.GatewayAddress.Equal(other.GatewayAddress)\n}\n"
  },
  {
    "path": "pkg/routing/router_windows.go",
    "content": "//go:build windows\n// +build windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage routing\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\n\t\"github.com/flannel-io/flannel/pkg/powershell\"\n)\n\n// Router manages network routes on Windows OS using MSFT_NetRoute\n// See also https://docs.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh872448(v%3Dvs.85)\ntype RouterWindows struct{}\n\nfunc (r RouterWindows) GetAllRoutes() ([]Route, error) {\n\treturn parseNetRoutes(\"@(Get-NetRoute | Select-Object -Property IfIndex,DestinationPrefix,NextHop)\")\n}\n\nfunc (r RouterWindows) GetRoutesFromInterfaceToSubnet(interfaceIndex int, destinationSubnet *net.IPNet) ([]Route, error) {\n\treturn parseNetRoutes(fmt.Sprintf(\"@(Get-NetRoute -InterfaceIndex %d -DestinationPrefix %s | Select-Object -Property IfIndex,DestinationPrefix,NextHop)\", interfaceIndex, destinationSubnet.String()))\n}\n\nfunc (r RouterWindows) CreateRoute(interfaceIndex int, destinationSubnet *net.IPNet, gatewayAddress net.IP) error {\n\t_, err := powershell.RunCommandf(\"New-NetRoute -InterfaceIndex %d -DestinationPrefix %s -NextHop  %s\", interfaceIndex, destinationSubnet.String(), gatewayAddress.String())\n\treturn err\n}\n\nfunc (r RouterWindows) DeleteRoute(interfaceIndex int, destinationSubnet *net.IPNet, gatewayAddress net.IP) error {\n\t_, err := powershell.RunCommandf(\"Remove-NetRoute -InterfaceIndex %d -DestinationPrefix %s -NextHop %s -Verbose -Confirm:$false\", interfaceIndex, destinationSubnet.String(), gatewayAddress.String())\n\treturn err\n}\n\ntype winNetRoute struct {\n\tIfIndex           int\n\tDestinationPrefix string\n\tNextHop           string\n}\n\nfunc parseNetRoutes(cmd string) ([]Route, error) {\n\tpowerShellJsonData := make([]winNetRoute, 0)\n\n\terr := powershell.RunCommandWithJsonResult(cmd, &powerShellJsonData)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\troutes := make([]Route, 0)\n\tfor _, r := range powerShellJsonData {\n\t\troute := Route{\n\t\t\tInterfaceIndex: r.IfIndex,\n\t\t}\n\n\t\t_, destinationSubnet, err := net.ParseCIDR(r.DestinationPrefix)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\troute.DestinationSubnet = destinationSubnet\n\n\t\tgatewayAddress := net.ParseIP(r.NextHop)\n\t\tif gatewayAddress == nil {\n\t\t\tcontinue\n\t\t}\n\t\troute.GatewayAddress = gatewayAddress\n\n\t\troutes = append(routes, route)\n\t}\n\n\treturn routes, nil\n}\n"
  },
  {
    "path": "pkg/routing/router_windows_test.go",
    "content": "//go:build windows\n// +build windows\n\n// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage routing\n\nimport (\n\t\"net\"\n\t\"testing\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n)\n\nfunc TestGetAllRoutes(t *testing.T) {\n\trouter := RouterWindows{}\n\troutes, err := router.GetAllRoutes()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(routes) == 0 {\n\t\tt.Fatal(\"len(routes) == 0\")\n\t}\n}\n\nfunc TestCreateAndRemoveRoute(t *testing.T) {\n\tdefaultInterface, err := ip.GetDefaultGatewayInterface()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\trouter := RouterWindows{}\n\tallRoutes, err := router.GetAllRoutes()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(allRoutes) == 0 {\n\t\tt.Fatal(\"len(routes) == 0\")\n\t}\n\n\tdestinationSubnet := allRoutes[0].DestinationSubnet\n\tgatewayAddress := net.ParseIP(\"192.168.199.123\")\n\n\troutes, err := router.GetRoutesFromInterfaceToSubnet(defaultInterface.Index, destinationSubnet)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfirstRouteLen := len(routes)\n\tif len(routes) == 0 {\n\t\tt.Fatal(\"len(routes) == 0\")\n\t}\n\n\terr = router.CreateRoute(defaultInterface.Index, destinationSubnet, gatewayAddress)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\troutes, err = router.GetRoutesFromInterfaceToSubnet(defaultInterface.Index, destinationSubnet)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedLen := firstRouteLen + 1\n\tgivenLen := len(routes)\n\tif givenLen != expectedLen {\n\t\tt.Fatalf(\"givenLen:%d != expectedLen:%d\", givenLen, expectedLen)\n\t}\n\n\terr = router.DeleteRoute(defaultInterface.Index, destinationSubnet, gatewayAddress)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\troutes, err = router.GetRoutesFromInterfaceToSubnet(defaultInterface.Index, destinationSubnet)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(routes) != firstRouteLen {\n\t\tt.Fatal(\"len(routes) != startRouteLen\")\n\t}\n}\n"
  },
  {
    "path": "pkg/subnet/config.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage subnet\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/big\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n)\n\ntype Config struct {\n\tEnableIPv4     bool\n\tEnableIPv6     bool\n\tEnableNFTables bool\n\tNetwork        ip.IP4Net\n\tIPv6Network    ip.IP6Net\n\tSubnetMin      ip.IP4\n\tSubnetMax      ip.IP4\n\tIPv6SubnetMin  *ip.IP6\n\tIPv6SubnetMax  *ip.IP6\n\tSubnetLen      uint\n\tIPv6SubnetLen  uint\n\tBackendType    string          `json:\"-\"`\n\tBackend        json.RawMessage `json:\",omitempty\"`\n}\n\nfunc parseBackendType(be json.RawMessage) (string, error) {\n\tvar bt struct {\n\t\tType string\n\t}\n\n\tif len(be) == 0 {\n\t\treturn \"udp\", nil\n\t} else if err := json.Unmarshal(be, &bt); err != nil {\n\t\treturn \"\", fmt.Errorf(\"error decoding Backend property of config: %v\", err)\n\t}\n\n\treturn bt.Type, nil\n}\n\nfunc ParseConfig(s string) (*Config, error) {\n\tcfg := new(Config)\n\t// Enable ipv4 by default\n\tcfg.EnableIPv4 = true\n\terr := json.Unmarshal([]byte(s), cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbt, err := parseBackendType(cfg.Backend)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcfg.BackendType = bt\n\n\treturn cfg, nil\n}\n\n// CheckNetworkConfig checks the coherence of the flannel configuration.\n// It is used only with the local network manager, not with the kubernetes-based manager.\nfunc CheckNetworkConfig(config *Config) error {\n\tif config.EnableIPv4 {\n\t\tif config.Network.Empty() {\n\t\t\treturn errors.New(\"please define a correct Network parameter in the flannel config\")\n\t\t}\n\t\tif config.SubnetLen > 0 {\n\t\t\t// SubnetLen needs to allow for a tunnel and bridge device on each host.\n\t\t\tif config.SubnetLen > 30 {\n\t\t\t\treturn errors.New(\"SubnetLen must be less than /31\")\n\t\t\t}\n\n\t\t\t// SubnetLen needs to fit _more_ than twice into the Network.\n\t\t\t// the first subnet isn't used, so splitting into two one only provide one usable host.\n\t\t\tif config.SubnetLen < config.Network.PrefixLen+2 {\n\t\t\t\treturn errors.New(\"network must be able to accommodate at least four subnets\")\n\t\t\t}\n\t\t} else {\n\t\t\t// If the network is smaller than a /28 then the network isn't big enough for flannel so return an error.\n\t\t\t// Default to giving each host at least a /24 (as long as the network is big enough to support at least four hosts)\n\t\t\t// Otherwise, if the network is too small to give each host a /24 just split the network into four.\n\t\t\tif config.Network.PrefixLen > 28 {\n\t\t\t\t// Each subnet needs at least four addresses (/30) and the network needs to accommodate at least four\n\t\t\t\t// since the first subnet isn't used, so splitting into two would only provide one usable host.\n\t\t\t\t// So the min useful PrefixLen is /28\n\t\t\t\treturn errors.New(\"network is too small. Minimum useful network prefix is /28\")\n\t\t\t} else if config.Network.PrefixLen <= 22 {\n\t\t\t\t// Network is big enough to give each host a /24\n\t\t\t\tconfig.SubnetLen = 24\n\t\t\t} else {\n\t\t\t\t// Use +2 to provide four hosts per subnet.\n\t\t\t\tconfig.SubnetLen = config.Network.PrefixLen + 2\n\t\t\t}\n\t\t}\n\n\t\tsubnetSize := ip.IP4(1 << (32 - config.SubnetLen))\n\n\t\tif config.SubnetMin == ip.IP4(0) {\n\t\t\t// skip over the first subnet otherwise it causes problems. e.g.\n\t\t\t// if Network is 10.100.0.0/16, having an interface with 10.100.0.0\n\t\t\t// conflicts with the network address.\n\t\t\tconfig.SubnetMin = config.Network.IP + subnetSize\n\t\t} else if !config.Network.Contains(config.SubnetMin) {\n\t\t\treturn errors.New(\"SubnetMin is not in the range of the Network\")\n\t\t}\n\n\t\tif config.SubnetMax == ip.IP4(0) {\n\t\t\tconfig.SubnetMax = config.Network.Next().IP - subnetSize\n\t\t} else if !config.Network.Contains(config.SubnetMax) {\n\t\t\treturn errors.New(\"SubnetMax is not in the range of the Network\")\n\t\t}\n\n\t\t// The SubnetMin and SubnetMax need to be aligned to a SubnetLen boundary\n\t\tmask := ip.IP4(0xFFFFFFFF << (32 - config.SubnetLen))\n\t\tif config.SubnetMin != config.SubnetMin&mask {\n\t\t\treturn fmt.Errorf(\"SubnetMin is not on a SubnetLen boundary: %v\", config.SubnetMin)\n\t\t}\n\n\t\tif config.SubnetMax != config.SubnetMax&mask {\n\t\t\treturn fmt.Errorf(\"SubnetMax is not on a SubnetLen boundary: %v\", config.SubnetMax)\n\t\t}\n\t}\n\tif config.EnableIPv6 {\n\t\tif config.IPv6Network.Empty() {\n\t\t\treturn errors.New(\"please define a correct IPv6Network parameter in the flannel config\")\n\t\t}\n\t\tif config.IPv6SubnetLen > 0 {\n\t\t\t// SubnetLen needs to allow for a tunnel and bridge device on each host.\n\t\t\tif config.IPv6SubnetLen > 126 {\n\t\t\t\treturn errors.New(\"SubnetLen must be less than /127\")\n\t\t\t}\n\n\t\t\t// SubnetLen needs to fit _more_ than twice into the Network.\n\t\t\t// the first subnet isn't used, so splitting into two one only provide one usable host.\n\t\t\tif config.IPv6SubnetLen < config.IPv6Network.PrefixLen+2 {\n\t\t\t\treturn errors.New(\"network must be able to accommodate at least four subnets\")\n\t\t\t}\n\t\t} else {\n\t\t\t// If the network is smaller than a /124 then the network isn't big enough for flannel so return an error.\n\t\t\t// Default to giving each host at least a /64 (as long as the network is big enough to support at least four hosts)\n\t\t\t// Otherwise, if the network is too small to give each host a /64 just split the network into four.\n\t\t\tif config.IPv6Network.PrefixLen > 124 {\n\t\t\t\t// Each subnet needs at least four addresses (/126) and the network needs to accommodate at least four\n\t\t\t\t// since the first subnet isn't used, so splitting into two would only provide one usable host.\n\t\t\t\t// So the min useful PrefixLen is /124\n\t\t\t\treturn errors.New(\"IPv6Network is too small. Minimum useful network prefix is /124\")\n\t\t\t} else if config.IPv6Network.PrefixLen <= 62 {\n\t\t\t\t// Network is big enough to give each host a /64\n\t\t\t\tconfig.IPv6SubnetLen = 64\n\t\t\t} else {\n\t\t\t\t// Use +2 to provide four hosts per subnet.\n\t\t\t\tconfig.IPv6SubnetLen = config.IPv6Network.PrefixLen + 2\n\t\t\t}\n\t\t}\n\n\t\tipv6SubnetSize := big.NewInt(0).Lsh(big.NewInt(1), 128-config.IPv6SubnetLen)\n\n\t\tif ip.IsEmpty(config.IPv6SubnetMin) {\n\t\t\t// skip over the first subnet otherwise it causes problems. e.g.\n\t\t\t// if Network is fc00::/48, having an interface with fc00::\n\t\t\t// conflicts with the broadcast address.\n\t\t\tconfig.IPv6SubnetMin = ip.GetIPv6SubnetMin(config.IPv6Network.IP, ipv6SubnetSize)\n\t\t} else if !config.IPv6Network.Contains(config.IPv6SubnetMin) {\n\t\t\treturn errors.New(\"IPv6SubnetMin is not in the range of the IPv6Network\")\n\t\t}\n\n\t\tif ip.IsEmpty(config.IPv6SubnetMax) {\n\t\t\tconfig.IPv6SubnetMax = ip.GetIPv6SubnetMax(config.IPv6Network.Next().IP, ipv6SubnetSize)\n\t\t} else if !config.IPv6Network.Contains(config.IPv6SubnetMax) {\n\t\t\treturn errors.New(\"IPv6SubnetMax is not in the range of the IPv6Network\")\n\t\t}\n\n\t\t// The SubnetMin and SubnetMax need to be aligned to a SubnetLen boundary\n\t\tmask := ip.Mask(int(config.IPv6SubnetLen))\n\t\tif !ip.CheckIPv6Subnet(config.IPv6SubnetMin, mask) {\n\t\t\treturn fmt.Errorf(\"IPv6SubnetMin is not on a SubnetLen boundary: %v\", config.IPv6SubnetMin)\n\t\t}\n\n\t\tif !ip.CheckIPv6Subnet(config.IPv6SubnetMax, mask) {\n\t\t\treturn fmt.Errorf(\"IPv6SubnetMax is not on a SubnetLen boundary: %v\", config.IPv6SubnetMax)\n\t\t}\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/subnet/config_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage subnet\n\nimport (\n\t\"testing\"\n)\n\nfunc TestConfigDefaults(t *testing.T) {\n\ts := `{ \"network\": \"10.3.0.0/16\" }`\n\n\tcfg, err := ParseConfig(s)\n\tif err != nil {\n\t\tt.Fatalf(\"ParseConfig failed: %s\", err)\n\t}\n\terr = CheckNetworkConfig(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"CheckNetworkConfig failed: %s\", err)\n\t}\n\n\texpectedNet := \"10.3.0.0/16\"\n\tif cfg.Network.String() != expectedNet {\n\t\tt.Errorf(\"Network mismatch: expected %s, got %s\", expectedNet, cfg.Network)\n\t}\n\n\tif cfg.SubnetMin.String() != \"10.3.1.0\" {\n\t\tt.Errorf(\"SubnetMin mismatch, expected 10.3.1.0, got %s\", cfg.SubnetMin)\n\t}\n\n\tif cfg.SubnetMax.String() != \"10.3.255.0\" {\n\t\tt.Errorf(\"SubnetMax mismatch, expected 10.3.255.0, got %s\", cfg.SubnetMax)\n\t}\n\n\tif cfg.SubnetLen != 24 {\n\t\tt.Errorf(\"SubnetLen mismatch: expected 24, got %d\", cfg.SubnetLen)\n\t}\n}\n\nfunc TestIPv6ConfigDefaults(t *testing.T) {\n\ts := `{ \"enableIPv6\": true, \"ipv6Network\": \"fc00::/48\", \"enableIPv4\": false }`\n\n\tcfg, err := ParseConfig(s)\n\tif err != nil {\n\t\tt.Fatalf(\"ParseConfig failed: %s\", err)\n\t}\n\terr = CheckNetworkConfig(cfg)\n\tif err != nil {\n\t\tt.Fatalf(\"CheckNetworkConfig failed: %s\", err)\n\t}\n\n\texpectedNet := \"fc00::/48\"\n\tif cfg.IPv6Network.String() != expectedNet {\n\t\tt.Errorf(\"IPv6Network mismatch: expected %s, got %s\", expectedNet, cfg.IPv6Network)\n\t}\n\n\tif cfg.IPv6SubnetMin.String() != \"fc00:0:0:1::\" {\n\t\tt.Errorf(\"IPv6SubnetMin mismatch, expected fc00:0:0:1::, got %s\", cfg.IPv6SubnetMin)\n\t}\n\n\tif cfg.IPv6SubnetMax.String() != \"fc00:0:0:ffff::\" {\n\t\tt.Errorf(\"IPv6SubnetMax mismatch, expected fc00:0:0:ffff::, got %s\", cfg.IPv6SubnetMax)\n\t}\n\n\tif cfg.IPv6SubnetLen != 64 {\n\t\tt.Errorf(\"IPv6SubnetLen mismatch: expected 64, got %d\", cfg.IPv6SubnetLen)\n\t}\n}\n\nfunc TestConfigOverrides(t *testing.T) {\n\ts := `{ \"Network\": \"10.3.0.0/16\", \"SubnetMin\": \"10.3.5.0\", \"SubnetMax\": \"10.3.8.0\", \"SubnetLen\": 28 }`\n\n\tcfg, err := ParseConfig(s)\n\tif err != nil {\n\t\tt.Fatalf(\"ParseConfig failed: %s\", err)\n\t}\n\n\texpectedNet := \"10.3.0.0/16\"\n\tif cfg.Network.String() != expectedNet {\n\t\tt.Errorf(\"Network mismatch: expected %s, got %s\", expectedNet, cfg.Network)\n\t}\n\n\tif cfg.SubnetMin.String() != \"10.3.5.0\" {\n\t\tt.Errorf(\"SubnetMin mismatch: expected 10.3.5.0, got %s\", cfg.SubnetMin)\n\t}\n\n\tif cfg.SubnetMax.String() != \"10.3.8.0\" {\n\t\tt.Errorf(\"SubnetMax mismatch: expected 10.3.8.0, got %s\", cfg.SubnetMax)\n\t}\n\n\tif cfg.SubnetLen != 28 {\n\t\tt.Errorf(\"SubnetLen mismatch: expected 28, got %d\", cfg.SubnetLen)\n\t}\n}\n\nfunc TestIPv6ConfigOverrides(t *testing.T) {\n\ts := `{ \"EnableIPv6\": true, \"IPv6Network\": \"fc00::/48\", \"IPv6SubnetMin\": \"fc00:0:0:1::\", \"IPv6SubnetMax\": \"fc00:0:0:f::\", \"IPv6SubnetLen\": 124, \"enableIPv4\": false }`\n\n\tcfg, err := ParseConfig(s)\n\tif err != nil {\n\t\tt.Fatalf(\"ParseConfig failed: %s\", err)\n\t}\n\n\texpectedNet := \"fc00::/48\"\n\tif cfg.IPv6Network.String() != expectedNet {\n\t\tt.Errorf(\"IPv6Network mismatch: expected %s, got %s\", expectedNet, cfg.IPv6Network)\n\t}\n\n\tif cfg.IPv6SubnetMin.String() != \"fc00:0:0:1::\" {\n\t\tt.Errorf(\"IPv6SubnetMin mismatch: expected fc00:0:0:1::, got %s\", cfg.IPv6SubnetMin)\n\t}\n\n\tif cfg.IPv6SubnetMax.String() != \"fc00:0:0:f::\" {\n\t\tt.Errorf(\"IPv6SubnetMax mismatch: expected fc00:0:0:f::, got %s\", cfg.IPv6SubnetMax)\n\t}\n\n\tif cfg.IPv6SubnetLen != 124 {\n\t\tt.Errorf(\"IPv6SubnetLen mismatch: expected 124, got %d\", cfg.IPv6SubnetLen)\n\t}\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/local_manager.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"go.etcd.io/etcd/api/v3/v3rpc/rpctypes\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\traceRetries = 10\n\tsubnetTTL   = 24 * time.Hour\n)\n\nvar (\n\terrInterrupted   = errors.New(\"interrupted\")\n\terrCanceled      = errors.New(\"canceled\")\n\terrUnimplemented = errors.New(\"unimplemented\")\n)\n\ntype LocalManager struct {\n\tregistry               Registry\n\tpreviousSubnet         ip.IP4Net\n\tpreviousIPv6Subnet     ip.IP6Net\n\tsubnetLeaseRenewMargin int\n}\n\ntype watchCursor struct {\n\tindex int64\n}\n\nfunc isErrEtcdNodeExist(e error) bool {\n\tif e == nil {\n\t\treturn false\n\t}\n\treturn e == rpctypes.ErrDuplicateKey\n}\n\nfunc (c watchCursor) String() string {\n\treturn strconv.FormatInt(c.index, 10)\n}\n\nfunc NewLocalManager(ctx context.Context, config *EtcdConfig, prevSubnet ip.IP4Net, prevIPv6Subnet ip.IP6Net, subnetLeaseRenewMargin int) (subnet.Manager, error) {\n\tr, err := newEtcdSubnetRegistry(ctx, config, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newLocalManager(r, prevSubnet, prevIPv6Subnet, subnetLeaseRenewMargin), nil\n}\n\nfunc newLocalManager(r Registry, prevSubnet ip.IP4Net, prevIPv6Subnet ip.IP6Net, subnetLeaseRenewMargin int) subnet.Manager {\n\treturn &LocalManager{\n\t\tregistry:               r,\n\t\tpreviousSubnet:         prevSubnet,\n\t\tpreviousIPv6Subnet:     prevIPv6Subnet,\n\t\tsubnetLeaseRenewMargin: subnetLeaseRenewMargin,\n\t}\n}\n\nfunc (m *LocalManager) GetStoredMacAddresses(ctx context.Context) (string, string) {\n\treturn \"\", \"\"\n}\n\nfunc (m *LocalManager) GetStoredPublicIP(ctx context.Context) (string, string) {\n\treturn \"\", \"\"\n}\n\nfunc (m *LocalManager) GetNetworkConfig(ctx context.Context) (*subnet.Config, error) {\n\tcfg, err := m.registry.getNetworkConfig(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfig, err := subnet.ParseConfig(cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = subnet.CheckNetworkConfig(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn config, nil\n}\n\nfunc (m *LocalManager) AcquireLease(ctx context.Context, attrs *lease.LeaseAttrs) (*lease.Lease, error) {\n\tconfig, err := m.GetNetworkConfig(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor i := 0; i < raceRetries; i++ {\n\t\tl, err := m.tryAcquireLease(ctx, config, attrs.PublicIP, attrs)\n\t\tswitch err {\n\t\tcase nil:\n\t\t\treturn l, nil\n\t\tcase errTryAgain:\n\t\t\tcontinue\n\t\tdefault:\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn nil, errors.New(\"max retries reached trying to acquire a subnet\")\n}\n\nfunc findLeaseByIP(leases []lease.Lease, pubIP ip.IP4) *lease.Lease {\n\tfor _, l := range leases {\n\t\tif pubIP == l.Attrs.PublicIP {\n\t\t\treturn &l\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc findLeaseBySubnet(leases []lease.Lease, subnet ip.IP4Net) *lease.Lease {\n\tfor _, l := range leases {\n\t\tif subnet.Equal(l.Subnet) {\n\t\t\treturn &l\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (m *LocalManager) tryAcquireLease(ctx context.Context, config *subnet.Config, extIaddr ip.IP4, attrs *lease.LeaseAttrs) (*lease.Lease, error) {\n\tleases, _, err := m.registry.getSubnets(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Try to reuse a subnet if there's one that matches our IP\n\tif l := findLeaseByIP(leases, extIaddr); l != nil {\n\t\t// Make sure the existing subnet is still within the configured network\n\t\tif isSubnetConfigCompat(config, l.Subnet) && isIPv6SubnetConfigCompat(config, l.IPv6Subnet) {\n\t\t\tlog.Infof(\"Found lease (ip: %v ipv6: %v) for current IP (%v), reusing\", l.Subnet, l.IPv6Subnet, extIaddr)\n\n\t\t\tttl := time.Duration(0)\n\t\t\tif !l.Expiration.IsZero() {\n\t\t\t\t// Not a reservation\n\t\t\t\tttl = subnetTTL\n\t\t\t}\n\t\t\texp, err := m.registry.updateSubnet(ctx, l.Subnet, l.IPv6Subnet, attrs, ttl, 0)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tl.Attrs = *attrs\n\t\t\tl.Expiration = exp\n\t\t\treturn l, nil\n\t\t} else {\n\t\t\tlog.Infof(\"Found lease (%+v) for current IP (%v) but not compatible with current config, deleting\", l, extIaddr)\n\t\t\tif err := m.registry.deleteSubnet(ctx, l.Subnet, l.IPv6Subnet); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\t// no existing match, check if there was a previous subnet to use\n\tvar sn ip.IP4Net\n\tvar sn6 ip.IP6Net\n\tif !m.previousSubnet.Empty() {\n\t\t// use previous subnet\n\t\tif l := findLeaseBySubnet(leases, m.previousSubnet); l == nil {\n\t\t\t// Check if the previous subnet is a part of the network and of the right subnet length\n\t\t\tif isSubnetConfigCompat(config, m.previousSubnet) && isIPv6SubnetConfigCompat(config, m.previousIPv6Subnet) {\n\t\t\t\tlog.Infof(\"Found previously leased subnet (%v), reusing\", m.previousSubnet)\n\t\t\t\tsn = m.previousSubnet\n\t\t\t\tsn6 = m.previousIPv6Subnet\n\t\t\t} else {\n\t\t\t\tlog.Errorf(\"Found previously leased subnet (%v) that is not compatible with the Etcd network config, ignoring\", m.previousSubnet)\n\t\t\t}\n\t\t}\n\t}\n\n\tif sn.Empty() {\n\t\t// no existing match, grab a new one\n\t\tsn, sn6, err = m.allocateSubnet(config, leases)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\texp, err := m.registry.createSubnet(ctx, sn, sn6, attrs, subnetTTL)\n\tswitch {\n\tcase err == nil:\n\t\tlog.Infof(\"Allocated lease (ip: %v ipv6: %v) to current node (%v) \", sn, sn6, extIaddr)\n\t\treturn &lease.Lease{\n\t\t\tEnableIPv4: true,\n\t\t\tSubnet:     sn,\n\t\t\tEnableIPv6: !sn6.Empty(),\n\t\t\tIPv6Subnet: sn6,\n\t\t\tAttrs:      *attrs,\n\t\t\tExpiration: exp,\n\t\t}, nil\n\tcase isErrEtcdNodeExist(err):\n\t\treturn nil, errTryAgain\n\tdefault:\n\t\treturn nil, err\n\t}\n}\n\nfunc (m *LocalManager) allocateSubnet(config *subnet.Config, leases []lease.Lease) (ip.IP4Net, ip.IP6Net, error) {\n\tlog.Infof(\"Picking subnet in range %s ... %s\", config.SubnetMin, config.SubnetMax)\n\tif config.EnableIPv6 {\n\t\tlog.Infof(\"Picking ipv6 subnet in range %s ... %s\", config.IPv6SubnetMin, config.IPv6SubnetMax)\n\t}\n\n\tvar availableIPs []ip.IP4\n\tvar availableIPv6s []*ip.IP6\n\n\tsn := ip.IP4Net{IP: config.SubnetMin, PrefixLen: config.SubnetLen}\n\tvar sn6 ip.IP6Net\n\tif config.EnableIPv6 {\n\t\tsn6 = ip.IP6Net{IP: config.IPv6SubnetMin, PrefixLen: config.IPv6SubnetLen}\n\t}\n\nOuterLoop:\n\tfor ; sn.IP <= config.SubnetMax && len(availableIPs) < 100; sn = sn.Next() {\n\t\tfor _, l := range leases {\n\t\t\tif sn.Overlaps(l.Subnet) {\n\t\t\t\tcontinue OuterLoop\n\t\t\t}\n\t\t}\n\t\tavailableIPs = append(availableIPs, sn.IP)\n\t}\n\n\tif !sn6.Empty() {\n\tOuterLoopv6:\n\t\tfor ; sn6.IP.Cmp(config.IPv6SubnetMax) <= 0 && len(availableIPv6s) < 100; sn6 = sn6.Next() {\n\t\t\tfor _, l := range leases {\n\t\t\t\tif sn6.Overlaps(l.IPv6Subnet) {\n\t\t\t\t\tcontinue OuterLoopv6\n\t\t\t\t}\n\t\t\t}\n\t\t\tavailableIPv6s = append(availableIPv6s, sn6.IP)\n\t\t}\n\t}\n\n\tif len(availableIPs) == 0 || (!sn6.Empty() && len(availableIPv6s) == 0) {\n\t\treturn ip.IP4Net{}, ip.IP6Net{}, errors.New(\"out of subnets\")\n\t} else {\n\t\ti := randInt(0, len(availableIPs))\n\t\tipnet := ip.IP4Net{IP: availableIPs[i], PrefixLen: config.SubnetLen}\n\n\t\tif sn6.Empty() {\n\t\t\treturn ipnet, ip.IP6Net{}, nil\n\t\t}\n\t\ti = randInt(0, len(availableIPv6s))\n\t\treturn ipnet, ip.IP6Net{IP: availableIPv6s[i], PrefixLen: config.IPv6SubnetLen}, nil\n\t}\n}\n\nfunc (m *LocalManager) RenewLease(ctx context.Context, lease *lease.Lease) error {\n\texp, err := m.registry.updateSubnet(ctx, lease.Subnet, lease.IPv6Subnet, &lease.Attrs, subnetTTL, 0)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tlease.Expiration = exp\n\treturn nil\n}\n\nfunc getNextIndex(cursor interface{}) (int64, error) {\n\tnextIndex := int64(0)\n\n\tif wc, ok := cursor.(watchCursor); ok {\n\t\tnextIndex = wc.index\n\t} else if s, ok := cursor.(string); ok {\n\t\tvar err error\n\t\tnextIndex, err = strconv.ParseInt(s, 10, 64)\n\t\tif err != nil {\n\t\t\treturn 0, fmt.Errorf(\"failed to parse cursor: %v\", err)\n\t\t}\n\t} else {\n\t\treturn 0, fmt.Errorf(\"internal error: watch cursor is of unknown type\")\n\t}\n\n\treturn nextIndex + 1, nil\n}\n\nfunc (m *LocalManager) leaseWatchReset(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) (lease.LeaseWatchResult, error) {\n\tl, index, err := m.registry.getSubnet(ctx, sn, sn6)\n\tif err != nil {\n\t\treturn lease.LeaseWatchResult{}, err\n\t}\n\n\treturn lease.LeaseWatchResult{\n\t\tSnapshot: []lease.Lease{*l},\n\t\tCursor:   watchCursor{index},\n\t}, nil\n}\n\nfunc (m *LocalManager) WatchLease(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, receiver chan []lease.LeaseWatchResult) error {\n\twr, err := m.leaseWatchReset(ctx, sn, sn6)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tlog.Info(\"manager.WatchLease: sending reset results...\")\n\t//send the result of leaseWatchResult to allow the listener\n\t//to catch-up to the current state\n\treceiver <- []lease.LeaseWatchResult{wr}\n\n\tnextIndex, err := getNextIndex(wr.Cursor)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = m.registry.watchSubnet(ctx, nextIndex, sn, sn6, receiver)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (m *LocalManager) WatchLeases(ctx context.Context, receiver chan []lease.LeaseWatchResult) error {\n\twr, err := m.registry.leasesWatchReset(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// send the result of leasesWatchReset to the listener\n\t// to catch-up on the state if the registry\n\t// before starting to watch changes\n\treceiver <- []lease.LeaseWatchResult{wr}\n\n\tnextIndex, err := getNextIndex(wr.Cursor)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = m.registry.watchSubnets(ctx, receiver, nextIndex)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// CompleteLease monitor lease\nfunc (m *LocalManager) CompleteLease(ctx context.Context, myLease *lease.Lease, wg *sync.WaitGroup) error {\n\t// Use the subnet manager to start watching leases.\n\tevts := make(chan lease.Event)\n\n\twg.Add(1)\n\tgo func() {\n\t\tl := myLease\n\t\tsubnet.WatchLease(ctx, m, l.Subnet, l.IPv6Subnet, evts)\n\t\twg.Done()\n\t}()\n\n\trenewMargin := time.Duration(m.subnetLeaseRenewMargin) * time.Minute\n\tdur := time.Until(myLease.Expiration) - renewMargin\n\n\tfor {\n\t\tselect {\n\t\tcase <-time.After(dur):\n\t\t\terr := m.RenewLease(ctx, myLease)\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(\"Error renewing lease (trying again in 1 min): \", err)\n\t\t\t\tdur = time.Minute\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tlog.Info(\"Lease renewed, new expiration: \", myLease.Expiration)\n\t\t\tdur = time.Until(myLease.Expiration) - renewMargin\n\n\t\tcase e, ok := <-evts:\n\t\t\tif !ok {\n\t\t\t\tlog.Infof(\"Stopped monitoring lease\")\n\t\t\t\treturn errCanceled\n\t\t\t}\n\t\t\tswitch e.Type {\n\t\t\tcase lease.EventAdded:\n\t\t\t\tmyLease.Expiration = e.Lease.Expiration\n\t\t\t\tdur = time.Until(myLease.Expiration) - renewMargin\n\t\t\t\tlog.Infof(\"Waiting for %s to renew lease\", dur)\n\n\t\t\tcase lease.EventRemoved:\n\t\t\t\tlog.Error(\"Lease has been revoked. Shutting down daemon.\")\n\t\t\t\treturn errInterrupted\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc isIndexTooSmall(err error) bool {\n\treturn err == rpctypes.ErrGRPCCompacted\n}\n\nfunc isSubnetConfigCompat(config *subnet.Config, sn ip.IP4Net) bool {\n\tif sn.IP < config.SubnetMin || sn.IP > config.SubnetMax {\n\t\treturn false\n\t}\n\n\treturn sn.PrefixLen == config.SubnetLen\n}\n\nfunc isIPv6SubnetConfigCompat(config *subnet.Config, sn6 ip.IP6Net) bool {\n\tif !config.EnableIPv6 {\n\t\treturn sn6.Empty()\n\t}\n\tif sn6.Empty() || sn6.IP.Cmp(config.IPv6SubnetMin) < 0 || sn6.IP.Cmp(config.IPv6SubnetMax) > 0 {\n\t\treturn false\n\t}\n\n\treturn sn6.PrefixLen == config.IPv6SubnetLen\n}\n\nfunc (m *LocalManager) Name() string {\n\tpreviousSubnet := m.previousSubnet.String()\n\tif m.previousSubnet.Empty() {\n\t\tpreviousSubnet = \"None\"\n\t}\n\treturn fmt.Sprintf(\"Etcd Local Manager with Previous Subnet: %s\", previousSubnet)\n}\n\n// For etcd subnet manager, the file never changes so we just write it once at startup\nfunc (m *LocalManager) HandleSubnetFile(path string, config *subnet.Config, ipMasq bool, sn ip.IP4Net, ipv6sn ip.IP6Net, mtu int) error {\n\treturn subnet.WriteSubnetFile(path, config, ipMasq, sn, ipv6sn, mtu)\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/mock_registry.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/jonboulle/clockwork\"\n\t\"go.etcd.io/etcd/api/v3/v3rpc/rpctypes\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nvar clock clockwork.Clock = clockwork.NewRealClock()\n\ntype netwk struct {\n\tconfig        string\n\tsubnets       []lease.Lease\n\tsubnetsEvents chan event\n\n\tmux          sync.Mutex\n\tsubnetEvents map[ip.IP4Net]chan event\n}\n\nfunc (n *netwk) sendSubnetEvent(sn ip.IP4Net, e event) {\n\tlog.Infof(\"sendSubnetEvent: sn=[ %s ], e=[ %v ]\", sn, e)\n\tn.subnetsEvents <- e\n\n\tn.mux.Lock()\n\tc, ok := n.subnetEvents[sn]\n\tif !ok {\n\t\tc = make(chan event, 10)\n\t\tn.subnetEvents[sn] = c\n\t}\n\tn.mux.Unlock()\n\tc <- e\n}\n\ntype event struct {\n\tevt   lease.Event\n\tindex int64\n}\n\ntype MockSubnetRegistry struct {\n\tmux     sync.Mutex\n\tnetwork *netwk\n\tindex   int64\n}\n\nfunc NewMockRegistry(config string, initialSubnets []lease.Lease) *MockSubnetRegistry {\n\tmsr := &MockSubnetRegistry{\n\t\tindex: 1000,\n\t\tnetwork: &netwk{\n\t\t\tconfig:        config,\n\t\t\tsubnets:       initialSubnets,\n\t\t\tsubnetsEvents: make(chan event, 1000),\n\t\t\tsubnetEvents:  make(map[ip.IP4Net]chan event)},\n\t}\n\n\treturn msr\n}\n\nfunc (msr *MockSubnetRegistry) getNetworkConfig(ctx context.Context) (string, error) {\n\treturn msr.network.config, nil\n}\n\nfunc (msr *MockSubnetRegistry) setConfig(config string) error {\n\tmsr.network.config = config\n\treturn nil\n}\n\nfunc (msr *MockSubnetRegistry) getSubnets(ctx context.Context) ([]lease.Lease, int64, error) {\n\tmsr.mux.Lock()\n\tdefer msr.mux.Unlock()\n\n\tsubs := make([]lease.Lease, len(msr.network.subnets))\n\tcopy(subs, msr.network.subnets)\n\treturn subs, msr.index, nil\n}\n\n// TODO ignores ipv6\nfunc (msr *MockSubnetRegistry) getSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) (*lease.Lease, int64, error) {\n\tmsr.mux.Lock()\n\tdefer msr.mux.Unlock()\n\n\tfor _, l := range msr.network.subnets {\n\t\tif l.Subnet.Equal(sn) {\n\t\t\treturn &l, msr.index, nil\n\t\t}\n\t}\n\treturn nil, msr.index, fmt.Errorf(\"subnet %s not found\", sn)\n}\n\n// TOODO ignores ipv6\nfunc (msr *MockSubnetRegistry) createSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, attrs *lease.LeaseAttrs, ttl time.Duration) (time.Time, error) {\n\tmsr.mux.Lock()\n\tdefer msr.mux.Unlock()\n\n\t// check for existing\n\tif _, _, err := msr.network.findSubnet(sn); err == nil {\n\t\treturn time.Time{}, rpctypes.ErrGRPCKeyNotFound\n\t}\n\n\tmsr.index += 1\n\n\texp := time.Time{}\n\tif ttl != 0 {\n\t\texp = clock.Now().Add(ttl)\n\t}\n\n\tl := lease.Lease{\n\t\tSubnet:     sn,\n\t\tAttrs:      *attrs,\n\t\tExpiration: exp,\n\t\tAsof:       msr.index,\n\t}\n\tmsr.network.subnets = append(msr.network.subnets, l)\n\n\tevt := lease.Event{\n\t\tType:  lease.EventAdded,\n\t\tLease: l,\n\t}\n\n\tmsr.network.sendSubnetEvent(sn, event{evt, msr.index})\n\n\treturn exp, nil\n}\n\n// TODO ignores ipv6\nfunc (msr *MockSubnetRegistry) updateSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, attrs *lease.LeaseAttrs, ttl time.Duration, asof int64) (time.Time, error) {\n\tmsr.mux.Lock()\n\tdefer msr.mux.Unlock()\n\n\tmsr.index += 1\n\n\texp := time.Time{}\n\tif ttl != 0 {\n\t\texp = clock.Now().Add(ttl)\n\t}\n\n\tsub, i, err := msr.network.findSubnet(sn)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\tsub.Attrs = *attrs\n\tsub.Asof = msr.index\n\tsub.Expiration = exp\n\tmsr.network.subnets[i] = sub\n\tmsr.network.sendSubnetEvent(sn, event{\n\t\tlease.Event{\n\t\t\tType:  lease.EventAdded,\n\t\t\tLease: sub,\n\t\t}, msr.index,\n\t})\n\n\treturn sub.Expiration, nil\n}\n\nfunc (msr *MockSubnetRegistry) deleteSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) error {\n\tmsr.mux.Lock()\n\tdefer msr.mux.Unlock()\n\n\tmsr.index += 1\n\n\tsub, i, err := msr.network.findSubnet(sn)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmsr.network.subnets[i] = msr.network.subnets[len(msr.network.subnets)-1]\n\tmsr.network.subnets = msr.network.subnets[:len(msr.network.subnets)-1]\n\tsub.Asof = msr.index\n\tmsr.network.sendSubnetEvent(sn, event{\n\t\tlease.Event{\n\t\t\tType:  lease.EventRemoved,\n\t\t\tLease: sub,\n\t\t}, msr.index,\n\t})\n\n\treturn nil\n}\n\nfunc (msr *MockSubnetRegistry) watchSubnets(ctx context.Context, leaseWatchChan chan []lease.LeaseWatchResult, since int64) error {\n\tlog.Infof(\"watchSubnets started with since= [ %d]\", since)\n\tfor {\n\t\tmsr.mux.Lock()\n\t\tindex := msr.index\n\t\tmsr.mux.Unlock()\n\n\t\tif since < index {\n\t\t\treturn rpctypes.ErrGRPCCompacted\n\t\t}\n\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\tclose(leaseWatchChan)\n\t\t\treturn ctx.Err()\n\n\t\tcase e := <-msr.network.subnetsEvents:\n\t\t\tif e.index > since {\n\t\t\t\tleaseWatchChan <- []lease.LeaseWatchResult{\n\t\t\t\t\t{Events: []lease.Event{e.evt},\n\t\t\t\t\t\tCursor: e.index}}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// TODO ignores ip6\nfunc (msr *MockSubnetRegistry) watchSubnet(ctx context.Context, since int64, sn ip.IP4Net, sn6 ip.IP6Net, leaseWatchChan chan []lease.LeaseWatchResult) error {\n\treturn errUnimplemented\n}\n\nfunc (msr *MockSubnetRegistry) leasesWatchReset(ctx context.Context) (lease.LeaseWatchResult, error) {\n\twr := lease.LeaseWatchResult{}\n\tleases, index, err := msr.getSubnets(ctx)\n\tif err != nil {\n\t\treturn wr, fmt.Errorf(\"failed to retrieve subnet leases: %v\", err)\n\t}\n\n\twr.Cursor = watchCursor{index}\n\twr.Snapshot = leases\n\treturn wr, nil\n}\n\nfunc (n *netwk) findSubnet(sn ip.IP4Net) (lease.Lease, int, error) {\n\tfor i, sub := range n.subnets {\n\t\tif sub.Subnet.Equal(sn) {\n\t\t\treturn sub, i, nil\n\t\t}\n\t}\n\treturn lease.Lease{}, 0, fmt.Errorf(\"subnet not found\")\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/mock_subnet.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n)\n\nfunc NewMockManager(registry *MockSubnetRegistry) subnet.Manager {\n\treturn newLocalManager(registry, ip.IP4Net{}, ip.IP6Net{}, 60)\n}\n\nfunc NewMockManagerWithSubnet(registry *MockSubnetRegistry, sn ip.IP4Net, sn6 ip.IP6Net) subnet.Manager {\n\treturn newLocalManager(registry, sn, sn6, 60)\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/rand.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"math/rand\"\n\t\"time\"\n)\n\nvar rnd *rand.Rand\n\nfunc init() {\n\tseed := time.Now().UnixNano()\n\trnd = rand.New(rand.NewSource(seed))\n}\n\nfunc randInt(lo, hi int) int {\n\treturn lo + int(rnd.Int31n(int32(hi-lo)))\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/registry.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"path\"\n\t\"regexp\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"go.etcd.io/etcd/api/v3/mvccpb\"\n\t\"go.etcd.io/etcd/api/v3/v3rpc/rpctypes\"\n\t\"go.etcd.io/etcd/client/pkg/v3/tlsutil\"\n\tetcd \"go.etcd.io/etcd/client/v3\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nvar (\n\terrTryAgain            = errors.New(\"try again\")\n\terrConfigNotFound      = errors.New(\"flannel config not found in etcd store. Did you create your config using etcdv3 API?\")\n\terrNoWatchChannel      = errors.New(\"no watch channel\")\n\terrSubnetAlreadyexists = errors.New(\"subnet already exists\")\n)\n\ntype Registry interface {\n\tgetNetworkConfig(ctx context.Context) (string, error)\n\tgetSubnets(ctx context.Context) ([]lease.Lease, int64, error)\n\tgetSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) (*lease.Lease, int64, error)\n\tcreateSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, attrs *lease.LeaseAttrs, ttl time.Duration) (time.Time, error)\n\tupdateSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, attrs *lease.LeaseAttrs, ttl time.Duration, asof int64) (time.Time, error)\n\tdeleteSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) error\n\twatchSubnets(ctx context.Context, leaseWatchChan chan []lease.LeaseWatchResult, since int64) error\n\twatchSubnet(ctx context.Context, since int64, sn ip.IP4Net, sn6 ip.IP6Net, leaseWatchChan chan []lease.LeaseWatchResult) error\n\tleasesWatchReset(ctx context.Context) (lease.LeaseWatchResult, error)\n}\n\ntype EtcdConfig struct {\n\tEndpoints []string\n\tKeyfile   string\n\tCertfile  string\n\tCAFile    string\n\tPrefix    string\n\tUsername  string\n\tPassword  string\n}\n\ntype etcdNewFunc func(ctx context.Context, c *EtcdConfig) (*etcd.Client, etcd.KV, error)\n\ntype etcdSubnetRegistry struct {\n\tcliNewFunc   etcdNewFunc\n\tmux          sync.Mutex\n\tkvApi        etcd.KV\n\tcli          *etcd.Client\n\tetcdCfg      *EtcdConfig\n\tnetworkRegex *regexp.Regexp\n}\n\nfunc newTlsConfig(c *EtcdConfig) (*tls.Config, error) {\n\ttlscfg := tls.Config{\n\t\tMinVersion: tls.VersionTLS12,\n\t}\n\n\tif c.Keyfile == \"\" || c.Certfile == \"\" {\n\t\tlog.Warning(\"no certificate provided: connecting to etcd with http. This is insecure\")\n\t\treturn nil, nil\n\t} else {\n\t\tcert, err := tlsutil.NewCert(c.Certfile, c.Keyfile, nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif cert != nil {\n\t\t\ttlscfg.Certificates = []tls.Certificate{*cert}\n\t\t}\n\t\tif c.CAFile != \"\" {\n\t\t\ttlscfg.RootCAs, err = tlsutil.NewCertPool([]string{c.CAFile})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn &tlscfg, nil\n}\n\nfunc newEtcdClient(ctx context.Context, c *EtcdConfig) (*etcd.Client, etcd.KV, error) {\n\ttlscfg, err := newTlsConfig(c)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tcli, err := etcd.New(etcd.Config{\n\t\tEndpoints: c.Endpoints,\n\t\tUsername:  c.Username,\n\t\tPassword:  c.Password,\n\t\tTLS:       tlscfg,\n\t})\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tkv := etcd.NewKV(cli)\n\n\t//make sure the Client is closed properly\n\tgo func() {\n\t\t<-ctx.Done()\n\t\terr := cli.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Failed to close etcd client: %v\", err)\n\t\t}\n\t}()\n\treturn cli, kv, nil\n}\n\nfunc newEtcdSubnetRegistry(ctx context.Context, config *EtcdConfig, cliNewFunc etcdNewFunc) (Registry, error) {\n\tr := &etcdSubnetRegistry{\n\t\tetcdCfg:      config,\n\t\tnetworkRegex: regexp.MustCompile(config.Prefix + `/([^/]*)(/|/config)?$`),\n\t}\n\tif cliNewFunc != nil {\n\t\tr.cliNewFunc = cliNewFunc\n\t} else {\n\t\tr.cliNewFunc = newEtcdClient\n\t}\n\n\tvar err error\n\tr.cli, r.kvApi, err = r.cliNewFunc(ctx, config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn r, nil\n}\n\nfunc (esr *etcdSubnetRegistry) getNetworkConfig(ctx context.Context) (string, error) {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"config\")\n\tresp, err := esr.kv().Get(ctx, key)\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif len(resp.Kvs) == 0 {\n\t\treturn \"\", errConfigNotFound\n\t}\n\n\treturn string(resp.Kvs[0].Value), nil\n}\n\n// getSubnets queries etcd to get a list of currently allocated leases for a given network.\n// It returns the leases along with the \"as-of\" etcd-index that can be used as the starting\n// point for etcd watch.\nfunc (esr *etcdSubnetRegistry) getSubnets(ctx context.Context) ([]lease.Lease, int64, error) {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\")\n\tresp, err := esr.kv().Get(ctx, key, etcd.WithPrefix())\n\tif err != nil {\n\t\tif err == rpctypes.ErrGRPCKeyNotFound {\n\t\t\t// key not found: treat it as empty set\n\t\t\treturn []lease.Lease{}, 0, nil\n\t\t}\n\t\treturn nil, 0, err\n\t}\n\n\tleases := []lease.Lease{}\n\tfor _, kv := range resp.Kvs {\n\t\tttlresp, err := esr.cli.TimeToLive(ctx, etcd.LeaseID(kv.Lease))\n\t\tif err != nil {\n\t\t\tlog.Warningf(\"Could not read ttl: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tl, err := kvToIPLease(kv, ttlresp.TTL)\n\t\tif err != nil {\n\t\t\tlog.Warningf(\"Ignoring bad subnet node: %v\", err)\n\t\t\tcontinue\n\t\t}\n\n\t\tleases = append(leases, *l)\n\t}\n\n\treturn leases, resp.Header.Revision, nil\n}\n\nfunc (esr *etcdSubnetRegistry) getSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) (*lease.Lease, int64, error) {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\", subnet.MakeSubnetKey(sn, sn6))\n\tresp, err := esr.kv().Get(ctx, key)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\n\tif len(resp.Kvs) == 0 {\n\t\treturn nil, 0, rpctypes.ErrGRPCKeyNotFound\n\t}\n\n\tttlresp, err := esr.cli.TimeToLive(ctx, etcd.LeaseID(resp.Kvs[0].Lease))\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tl, err := kvToIPLease(resp.Kvs[0], ttlresp.TTL)\n\treturn l, resp.Header.Revision, err\n}\n\nfunc (esr *etcdSubnetRegistry) createSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, attrs *lease.LeaseAttrs, ttl time.Duration) (time.Time, error) {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\", subnet.MakeSubnetKey(sn, sn6))\n\tvalue, err := json.Marshal(attrs)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\tlresp, err := esr.cli.Grant(ctx, int64(ttl.Seconds()))\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\t//Use a transaction to check if key was not already present in etcd\n\treq := etcd.OpPut(key, string(value), etcd.WithLease(lresp.ID))\n\tcond := etcd.Compare(etcd.Version(key), \"=\", 0)\n\ttresp, err := esr.cli.Txn(ctx).If(cond).Then(req).Commit()\n\tif err != nil {\n\t\t_, rerr := esr.cli.Revoke(ctx, lresp.ID)\n\t\tif rerr != nil {\n\t\t\tlog.Error(rerr)\n\t\t}\n\t\treturn time.Time{}, err\n\t}\n\tif !tresp.Succeeded {\n\t\t_, rerr := esr.cli.Revoke(ctx, lresp.ID)\n\t\tif rerr != nil {\n\t\t\tlog.Error(rerr)\n\t\t}\n\t\treturn time.Time{}, errSubnetAlreadyexists\n\t}\n\n\texp := time.Now().Add(time.Duration(lresp.TTL) * time.Second)\n\treturn exp, nil\n}\n\nfunc (esr *etcdSubnetRegistry) updateSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, attrs *lease.LeaseAttrs, ttl time.Duration, asof int64) (time.Time, error) {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\", subnet.MakeSubnetKey(sn, sn6))\n\tvalue, err := json.Marshal(attrs)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\tlresp, lerr := esr.cli.Grant(ctx, int64(ttl.Seconds()))\n\tif lerr != nil {\n\t\treturn time.Time{}, lerr\n\t}\n\n\t_, perr := esr.kv().Put(ctx, key, string(value), etcd.WithLease(lresp.ID))\n\tif perr != nil {\n\t\t_, rerr := esr.cli.Revoke(ctx, lresp.ID)\n\t\tif rerr != nil {\n\t\t\tlog.Error(rerr)\n\t\t}\n\t\treturn time.Time{}, perr\n\t}\n\n\texp := time.Now().Add(time.Duration(lresp.TTL) * time.Second)\n\n\treturn exp, nil\n}\n\nfunc (esr *etcdSubnetRegistry) deleteSubnet(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net) error {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\", subnet.MakeSubnetKey(sn, sn6))\n\t_, err := esr.kv().Delete(ctx, key)\n\treturn err\n}\n\nfunc (esr *etcdSubnetRegistry) watchSubnets(ctx context.Context, leaseWatchChan chan []lease.LeaseWatchResult, since int64) error {\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\")\n\tinitialBackoff := 100 * time.Millisecond // Initial backoff duration\n\texponentialBackoff := initialBackoff\n\tmaxBackoff := 5 * time.Second // Cap for backoff duration\n\n\tfor {\n\t\twctx, cancel := context.WithCancel(ctx)\n\t\tdefer cancel()\n\t\tlog.Infof(\"registry: watching subnets starting from rev %d\", since)\n\t\trch := esr.cli.Watch(etcd.WithRequireLeader(wctx), key, etcd.WithPrefix(), etcd.WithRev(since))\n\t\tif rch == nil {\n\t\t\tlog.Errorf(\"Failed to establish etcd watch channel\")\n\t\t\tcancel()\n\t\t\ttime.Sleep(exponentialBackoff)                             // Avoid CPU spinning\n\t\t\texponentialBackoff = min(exponentialBackoff*2, maxBackoff) // Exponential backoff\n\t\t\tcontinue\n\t\t}\n\n\tinnerLoop:\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\terr := esr.cli.Close()\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"Failed to close etcd client: %v\", err)\n\t\t\t\t}\n\t\t\t\tclose(leaseWatchChan)\n\t\t\t\treturn ctx.Err()\n\t\t\tcase wresp, ok := <-rch:\n\t\t\t\terr := wresp.Err()\n\t\t\t\tif !ok || err != nil {\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Warningf(\"etcd watch channel for %s closed with error %v, reconnecting...\", key, err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlog.Warningf(\"etcd watch channel for %s closed, reconnecting...\", key)\n\t\t\t\t\t}\n\t\t\t\t\tcancel()\n\t\t\t\t\ttime.Sleep(exponentialBackoff)\n\t\t\t\t\texponentialBackoff = min(exponentialBackoff*2, maxBackoff)\n\t\t\t\t\tbreak innerLoop\n\t\t\t\t}\n\t\t\t\texponentialBackoff = initialBackoff // Reset backoff on success\n\t\t\t\tresults := make([]lease.LeaseWatchResult, 0)\n\t\t\t\tfor _, etcdEvent := range wresp.Events {\n\t\t\t\t\tsubnetEvent, err := parseSubnetWatchResponse(ctx, esr.cli, etcdEvent)\n\t\t\t\t\tswitch {\n\n\t\t\t\t\tcase err == nil:\n\t\t\t\t\t\tlog.Infof(\"watchSubnets: got valid subnet event with revision %d\", wresp.Header.Revision)\n\t\t\t\t\t\t// TODO only vxlan backend and kube subnet manager support dual stack now.\n\t\t\t\t\t\tsubnetEvent.Lease.EnableIPv4 = true\n\t\t\t\t\t\twr := lease.LeaseWatchResult{\n\t\t\t\t\t\t\tEvents: []lease.Event{subnetEvent},\n\t\t\t\t\t\t\tCursor: watchCursor{wresp.Header.Revision},\n\t\t\t\t\t\t}\n\t\t\t\t\t\tresults = append(results, wr)\n\n\t\t\t\t\tcase isIndexTooSmall(err):\n\t\t\t\t\t\tlog.Warning(\"Watch of subnet leases failed because etcd index outside history window\")\n\t\t\t\t\t\twr, err := esr.leasesWatchReset(ctx)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tlog.Errorf(\"error resetting etcd watch: %s\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tresults = append(results, wr)\n\t\t\t\t\tcase wresp.Header.Revision != 0:\n\t\t\t\t\t\tlog.Warning(\"Watch of subnet leases failed because header revision != 0\")\n\t\t\t\t\t\tresults = append(results, lease.LeaseWatchResult{Cursor: watchCursor{wresp.Header.Revision}})\n\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tlog.Warningf(\"Watch of subnet failed with error %s\", err)\n\t\t\t\t\t\tresults = append(results, lease.LeaseWatchResult{})\n\t\t\t\t\t}\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Errorf(\"error parsing etcd event: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(results) > 0 {\n\t\t\t\t\tleaseWatchChan <- results\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (esr *etcdSubnetRegistry) watchSubnet(ctx context.Context, since int64, sn ip.IP4Net, sn6 ip.IP6Net, leaseWatchChan chan []lease.LeaseWatchResult) error {\n\tsubnetKey := subnet.MakeSubnetKey(sn, sn6)\n\tkey := path.Join(esr.etcdCfg.Prefix, \"subnets\", subnetKey)\n\tinitialBackoff := 100 * time.Millisecond // Initial backoff duration\n\texponentialBackoff := initialBackoff\n\tmaxBackoff := 5 * time.Second // Cap for backoff duration\n\n\tfor {\n\t\twctx, cancel := context.WithCancel(ctx)\n\t\tdefer cancel()\n\t\trch := esr.cli.Watch(etcd.WithRequireLeader(wctx), key, etcd.WithPrefix(), etcd.WithRev(since))\n\t\tif rch == nil {\n\t\t\tlog.Errorf(\"Failed to establish etcd watch channel\")\n\t\t\tcancel()\n\t\t\ttime.Sleep(exponentialBackoff)                             // Avoid CPU spinning\n\t\t\texponentialBackoff = min(exponentialBackoff*2, maxBackoff) // Exponential backoff\n\t\t\tcontinue\n\t\t}\n\n\tinnerLoop:\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\terr := esr.cli.Close()\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorf(\"Failed to close etcd client: %v\", err)\n\t\t\t\t}\n\t\t\t\tclose(leaseWatchChan)\n\t\t\t\treturn ctx.Err()\n\t\t\tcase wresp, ok := <-rch:\n\t\t\t\terr := wresp.Err()\n\t\t\t\tif !ok || err != nil {\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Warningf(\"etcd watch channel for %s closed with error %v, reconnecting...\", key, err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlog.Warningf(\"etcd watch channel for %s closed, reconnecting...\", key)\n\t\t\t\t\t}\n\t\t\t\t\tcancel()\n\t\t\t\t\ttime.Sleep(exponentialBackoff)\n\t\t\t\t\texponentialBackoff = min(exponentialBackoff*2, maxBackoff)\n\t\t\t\t\tbreak innerLoop\n\t\t\t\t}\n\t\t\t\texponentialBackoff = initialBackoff // Reset backoff on success\n\t\t\t\tbatch := make([]lease.LeaseWatchResult, 0)\n\t\t\t\tfor _, etcdEvent := range wresp.Events {\n\t\t\t\t\tsubnetEvent, err := parseSubnetWatchResponse(ctx, esr.cli, etcdEvent)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase err == nil:\n\t\t\t\t\t\twr := lease.LeaseWatchResult{\n\t\t\t\t\t\t\tEvents: []lease.Event{subnetEvent},\n\t\t\t\t\t\t\tCursor: watchCursor{wresp.Header.Revision},\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbatch = append(batch, wr)\n\t\t\t\t\tcase isIndexTooSmall(err):\n\t\t\t\t\t\tlog.Warning(\"Watch of subnet leases failed because etcd index outside history window\")\n\t\t\t\t\t\twr, err := esr.leasesWatchReset(ctx)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tlog.Errorf(\"error resetting etcd watch: %s\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbatch = append(batch, wr)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tlog.Errorf(\"couldn't read etcd event: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(batch) > 0 {\n\t\t\t\t\tleaseWatchChan <- batch\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (esr *etcdSubnetRegistry) kv() etcd.KV {\n\tesr.mux.Lock()\n\tdefer esr.mux.Unlock()\n\treturn esr.kvApi\n}\n\nfunc parseSubnetWatchResponse(ctx context.Context, cli *etcd.Client, ev *etcd.Event) (lease.Event, error) {\n\tsn, tsn6 := subnet.ParseSubnetKey(string(ev.Kv.Key))\n\tif sn == nil {\n\t\treturn lease.Event{}, fmt.Errorf(\"%v %q: not a subnet, skipping\", ev.Type, string(ev.Kv.Key))\n\t}\n\n\tvar sn6 ip.IP6Net\n\tif tsn6 != nil {\n\t\tsn6 = *tsn6\n\t}\n\n\tswitch ev.Type {\n\tcase etcd.EventTypeDelete:\n\t\treturn lease.Event{\n\t\t\tType: lease.EventRemoved,\n\t\t\tLease: lease.Lease{\n\t\t\t\tEnableIPv4: true,\n\t\t\t\tSubnet:     *sn,\n\t\t\t\tEnableIPv6: !sn6.Empty(),\n\t\t\t\tIPv6Subnet: sn6,\n\t\t\t},\n\t\t}, nil\n\n\tdefault:\n\t\tattrs := &lease.LeaseAttrs{}\n\t\terr := json.Unmarshal(ev.Kv.Value, attrs)\n\t\tif err != nil {\n\t\t\treturn lease.Event{}, err\n\t\t}\n\n\t\tlresp, lerr := cli.TimeToLive(ctx, etcd.LeaseID(ev.Kv.Lease))\n\t\tif lerr != nil {\n\t\t\treturn lease.Event{}, lerr\n\t\t}\n\t\texp := time.Now().Add(time.Duration(lresp.TTL) * time.Second)\n\t\tevt := lease.Event{\n\t\t\tType: lease.EventAdded,\n\t\t\tLease: lease.Lease{\n\t\t\t\tEnableIPv4: true,\n\t\t\t\tSubnet:     *sn,\n\t\t\t\tEnableIPv6: !sn6.Empty(),\n\t\t\t\tIPv6Subnet: sn6,\n\t\t\t\tAttrs:      *attrs,\n\t\t\t\tExpiration: exp,\n\t\t\t},\n\t\t}\n\t\treturn evt, nil\n\t}\n}\n\nfunc kvToIPLease(kv *mvccpb.KeyValue, ttl int64) (*lease.Lease, error) {\n\tsn, tsn6 := subnet.ParseSubnetKey(string(kv.Key))\n\tif sn == nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse subnet key %s\", kv.Key)\n\t}\n\n\tvar sn6 ip.IP6Net\n\tif tsn6 != nil {\n\t\tsn6 = *tsn6\n\t}\n\n\tattrs := &lease.LeaseAttrs{}\n\tif err := json.Unmarshal([]byte(kv.Value), attrs); err != nil {\n\t\treturn nil, err\n\t}\n\n\texp := time.Now().Add(time.Duration(ttl) * time.Second)\n\n\tlease := lease.Lease{\n\t\tEnableIPv4: true,\n\t\tEnableIPv6: !sn6.Empty(),\n\t\tSubnet:     *sn,\n\t\tIPv6Subnet: sn6,\n\t\tAttrs:      *attrs,\n\t\tExpiration: exp,\n\t\tAsof:       kv.ModRevision,\n\t}\n\n\treturn &lease, nil\n}\n\n// leasesWatchReset is called when incremental lease watch failed and we need to grab a snapshot\nfunc (esr *etcdSubnetRegistry) leasesWatchReset(ctx context.Context) (lease.LeaseWatchResult, error) {\n\twr := lease.LeaseWatchResult{}\n\n\tleases, index, err := esr.getSubnets(ctx)\n\tif err != nil {\n\t\treturn wr, fmt.Errorf(\"failed to retrieve subnet leases: %v\", err)\n\t}\n\n\twr.Cursor = watchCursor{index}\n\twr.Snapshot = leases\n\treturn wr, nil\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/registry_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\tetcd \"go.etcd.io/etcd/client/v3\"\n\t\"go.etcd.io/etcd/tests/v3/framework/integration\"\n)\n\nfunc newTestEtcdRegistry(t *testing.T, ctx context.Context, client *etcd.Client) (Registry, etcd.KV) {\n\tcfg := &EtcdConfig{\n\t\tEndpoints: []string{\"http://127.0.0.1:4001\", \"http://127.0.0.1:2379\"},\n\t\tPrefix:    \"/coreos.com/network\",\n\t}\n\n\tr, err := newEtcdSubnetRegistry(ctx, cfg,\n\t\tfunc(ctx context.Context, c *EtcdConfig) (*etcd.Client, etcd.KV, error) {\n\t\t\treturn client, client.KV, nil\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to create etcd subnet registry\")\n\t}\n\n\treturn r, r.(*etcdSubnetRegistry).kvApi\n}\n\nfunc watchSubnets(t *testing.T, r Registry, ctx context.Context, sn ip.IP4Net, nextIndex int64, result chan error) {\n\ttype leaseEvent struct {\n\t\tetype  lease.EventType\n\t\tsubnet ip.IP4Net\n\t\tfound  bool\n\t}\n\texpectedEvents := []leaseEvent{\n\t\t{lease.EventAdded, sn, false},\n\t\t{lease.EventRemoved, sn, false},\n\t}\n\n\treceiver := make(chan []lease.LeaseWatchResult)\n\tnumFound := 0\n\n\tgo func() {\n\t\terr := r.watchSubnets(ctx, receiver, nextIndex)\n\t\tif err != nil {\n\t\t\tresult <- errNoWatchChannel\n\t\t\treturn\n\t\t}\n\t}()\n\n\tfor watchResults := range receiver {\n\t\tfor _, wr := range watchResults {\n\t\t\tfor _, evt := range wr.Events {\n\t\t\t\tfor _, exp := range expectedEvents {\n\t\t\t\t\tif evt.Type != exp.etype {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif exp.found == true {\n\t\t\t\t\t\tresult <- fmt.Errorf(\"Subnet event type already found: %v\", exp)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tif !evt.Lease.Subnet.Equal(exp.subnet) {\n\t\t\t\t\t\tresult <- fmt.Errorf(\"Subnet event lease %v mismatch (expected %v)\", evt.Lease.Subnet, exp.subnet)\n\t\t\t\t\t}\n\t\t\t\t\texp.found = true\n\t\t\t\t\tnumFound += 1\n\t\t\t\t}\n\t\t\t\tif numFound == len(expectedEvents) {\n\t\t\t\t\t// All done; success\n\t\t\t\t\tresult <- nil\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t}\n\n\t}\n}\n\nfunc TestEtcdRegistry(t *testing.T) {\n\tintegration.BeforeTestExternal(t)\n\n\tclus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1})\n\t// NewClusterV3(t, &integration.ClusterConfig{Size: 1})\n\tdefer clus.Terminate(t)\n\n\tclient := clus.RandClient()\n\n\tctx := context.Background()\n\n\tr, kvApi := newTestEtcdRegistry(t, ctx, client)\n\n\t_, err := r.getNetworkConfig(ctx)\n\tif err != errConfigNotFound {\n\t\tt.Fatal(\"Should hit error getting config\")\n\t}\n\n\t// Populate etcd with a network\n\tnetKey := \"/coreos.com/network/config\"\n\tnetValue := \"{ \\\"Network\\\": \\\"10.1.0.0/16\\\", \\\"Backend\\\": { \\\"Type\\\": \\\"host-gw\\\" } }\"\n\t_, err = kvApi.Put(ctx, netKey, netValue)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to create new entry\", err)\n\t}\n\n\tconfig, err := r.getNetworkConfig(ctx)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to get network config\", err)\n\t}\n\tif config != netValue {\n\t\tt.Fatal(\"Failed to match network config\")\n\t}\n\n\tsn := ip.IP4Net{\n\t\tIP:        ip.MustParseIP4(\"10.1.5.0\"),\n\t\tPrefixLen: 24,\n\t}\n\n\twg := sync.WaitGroup{}\n\twg.Add(1)\n\tstartWg := sync.WaitGroup{}\n\tstartWg.Add(1)\n\tresult := make(chan error, 1)\n\tgo func() {\n\t\tstartWg.Done()\n\t\twatchSubnets(t, r, ctx, sn, 0, result)\n\t\twg.Done()\n\t}()\n\n\tstartWg.Wait()\n\t// Lease a subnet for the network\n\tattrs := &lease.LeaseAttrs{\n\t\tPublicIP: ip.MustParseIP4(\"1.2.3.4\"),\n\t}\n\texp, err := r.createSubnet(ctx, sn, ip.IP6Net{}, attrs, 24*time.Hour)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to create subnet lease\")\n\t}\n\tif !exp.After(time.Now()) {\n\t\tt.Fatalf(\"Subnet lease duration %v not in the future\", exp)\n\t}\n\n\t// Make sure the lease got created\n\tresp, err := kvApi.Get(ctx, \"/coreos.com/network/subnets/10.1.5.0-24\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to verify subnet lease directly in etcd: %v\", err)\n\t}\n\tif resp == nil || resp.Kvs == nil {\n\t\tt.Fatal(\"Failed to retrive node in subnet lease\")\n\t}\n\n\tif len(resp.Kvs) != 1 || !bytes.Equal(resp.Kvs[0].Value, []byte(\"{\\\"PublicIP\\\":\\\"1.2.3.4\\\",\\\"PublicIPv6\\\":null}\")) {\n\t\tt.Fatalf(\"Unexpected subnet lease node %s value %s\", resp.Kvs[0].Key, resp.Kvs[0].Value)\n\t}\n\n\tleases, _, err := r.getSubnets(ctx)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to get Subnets\")\n\t}\n\tif len(leases) != 1 {\n\t\tt.Fatalf(\"Unexpected number of leases %d (expected 1)\", len(leases))\n\t}\n\tif !leases[0].Subnet.Equal(sn) {\n\t\tt.Fatalf(\"Mismatched subnet %v (expected %v)\", leases[0].Subnet, sn)\n\t}\n\n\tlease, _, err := r.getSubnet(ctx, sn, ip.IP6Net{})\n\tif lease == nil {\n\t\tt.Fatal(\"Missing subnet lease\")\n\t}\n\tif err != nil {\n\t\tt.Fatal(\"Failed to get Subnet\")\n\t}\n\n\terr = r.deleteSubnet(ctx, sn, ip.IP6Net{})\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to delete subnet %v: %v\", sn, err)\n\t}\n\n\t// Make sure the lease got deleted\n\tresp, err = kvApi.Get(ctx, \"/coreos.com/network/subnets/10.1.5.0-24\")\n\tif err != nil {\n\t\tt.Fatal(\"Failed to get Subnet\" + err.Error())\n\t}\n\tif len(resp.Kvs) > 0 {\n\t\tt.Fatal(\"Unexpected success getting deleted subnet\")\n\t}\n\n\twg.Wait()\n\n\t// Check errors from watch goroutine\n\twatchResult := <-result\n\tif watchResult != nil {\n\t\tt.Fatalf(\"Error watching keys: %v\", watchResult)\n\t}\n\n\t// TODO: watchSubnet and watchNetworks\n}\n"
  },
  {
    "path": "pkg/subnet/etcd/subnet_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage etcd\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"path\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\tetcd \"go.etcd.io/etcd/client/v3\"\n\t\"go.etcd.io/etcd/tests/v3/framework/integration\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nfunc initTestRegistry(ctx context.Context, t *testing.T, r Registry, kvApi etcd.KV) {\n\t// Populate etcd with a network\n\tnetKey := \"/coreos.com/network/config\"\n\tnetValue := `{ \"Network\": \"10.3.0.0/16\", \"SubnetMin\": \"10.3.1.0\", \"SubnetMax\": \"10.3.25.0\" }`\n\t_, err := kvApi.Put(ctx, netKey, netValue)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to create new entry\", err)\n\t}\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: ip.MustParseIP4(\"1.1.1.1\"),\n\t}\n\n\texp := time.Now().Add(24 * time.Hour)\n\n\tsubnets := []lease.Lease{\n\t\t// leases within SubnetMin-SubnetMax range\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.1.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 10},\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.2.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 11},\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.4.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 12},\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.5.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 13},\n\n\t\t// hand created lease outside the range of subnetMin-SubnetMax for testing removal\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.31.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 13},\n\t}\n\n\tfor _, lease := range subnets {\n\t\t_, err = r.createSubnet(ctx, lease.Subnet, lease.IPv6Subnet, &attrs, time.Until(lease.Expiration))\n\t\tif err != nil {\n\t\t\tt.Fatal(\"Failed to create new entry\", err)\n\t\t}\n\t}\n}\n\nfunc newDummyRegistry() *MockSubnetRegistry {\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: ip.MustParseIP4(\"1.1.1.1\"),\n\t}\n\n\texp := time.Time{}\n\n\tsubnets := []lease.Lease{\n\t\t// leases within SubnetMin-SubnetMax range\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.1.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 10},\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.2.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 11},\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.4.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 12},\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.5.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 13},\n\n\t\t// hand created lease outside the range of subnetMin-SubnetMax for testing removal\n\t\t{EnableIPv4: true, EnableIPv6: false, Subnet: ip.IP4Net{IP: ip.MustParseIP4(\"10.3.31.0\"), PrefixLen: 24}, IPv6Subnet: ip.IP6Net{}, Attrs: attrs, Expiration: exp, Asof: 13},\n\t}\n\n\tconfig := `{ \"Network\": \"10.3.0.0/16\", \"SubnetMin\": \"10.3.1.0\", \"SubnetMax\": \"10.3.25.0\" }`\n\treturn NewMockRegistry(config, subnets)\n}\n\nfunc TestAcquireLease(t *testing.T) {\n\tmsr := newDummyRegistry()\n\tsm := NewMockManager(msr)\n\n\textIaddr, _ := ip.ParseIP4(\"1.2.3.4\")\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: extIaddr,\n\t}\n\n\tl, err := sm.AcquireLease(context.Background(), &attrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\n\tif !inAllocatableRange(context.Background(), sm, l.Subnet) {\n\t\tt.Fatal(\"Subnet mismatch: expected 10.3.3.0/24, got: \", l.Subnet)\n\t}\n\n\t// Acquire again, should reuse\n\tl2, err := sm.AcquireLease(context.Background(), &attrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\n\tif !l.Subnet.Equal(l2.Subnet) {\n\t\tt.Fatalf(\"AcquireLease did not reuse subnet; expected %v, got %v\", l.Subnet, l2.Subnet)\n\t}\n\n\t// Test if a previous subnet will be used\n\tmsr2 := newDummyRegistry()\n\tprevSubnet := ip.IP4Net{IP: ip.MustParseIP4(\"10.3.6.0\"), PrefixLen: 24}\n\tsm2 := NewMockManagerWithSubnet(msr2, prevSubnet, ip.IP6Net{})\n\tprev, err := sm2.AcquireLease(context.Background(), &attrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\tif !prev.Subnet.Equal(prevSubnet) {\n\t\tt.Fatalf(\"AcquireLease did not reuse subnet from previous run; expected %v, got %v\", prevSubnet, prev.Subnet)\n\t}\n\n\t// Test that a previous subnet will not be used if it does not match the registry config\n\tmsr3 := newDummyRegistry()\n\tinvalidSubnet := ip.IP4Net{IP: ip.MustParseIP4(\"10.4.1.0\"), PrefixLen: 24}\n\tsm3 := NewMockManagerWithSubnet(msr3, invalidSubnet, ip.IP6Net{})\n\tl3, err := sm3.AcquireLease(context.Background(), &attrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\tif l3.Subnet.Equal(invalidSubnet) {\n\t\tt.Fatalf(\"AcquireLease reused invalid subnet from previous run; reused %v\", l3.Subnet)\n\t}\n}\n\nfunc TestConfigChanged(t *testing.T) {\n\tmsr := newDummyRegistry()\n\tsm := NewMockManager(msr)\n\n\textIaddr, _ := ip.ParseIP4(\"1.2.3.4\")\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: extIaddr,\n\t}\n\n\tl, err := sm.AcquireLease(context.Background(), &attrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\n\tif !inAllocatableRange(context.Background(), sm, l.Subnet) {\n\t\tt.Fatal(\"Acquired subnet outside of valid range: \", l.Subnet)\n\t}\n\n\t// Change config\n\tconfig := `{ \"Network\": \"10.4.0.0/16\" }`\n\terr = msr.setConfig(config)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to set the Config\", err)\n\t}\n\n\t// Acquire again, should not reuse\n\tif l, err = sm.AcquireLease(context.Background(), &attrs); err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\n\tif !inAllocatableRange(context.Background(), sm, l.Subnet) {\n\t\tt.Fatal(\"Acquired subnet outside of valid range: \", l.Subnet)\n\t}\n}\n\nfunc acquireLease(ctx context.Context, t *testing.T, sm subnet.Manager) *lease.Lease {\n\textIaddr, _ := ip.ParseIP4(\"1.2.3.4\")\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: extIaddr,\n\t}\n\n\tl, err := sm.AcquireLease(ctx, &attrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\n\treturn l\n}\n\nfunc TestWatchLeaseAdded(t *testing.T) {\n\tintegration.BeforeTestExternal(t)\n\n\tclus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1})\n\tdefer clus.Terminate(t)\n\n\tclient := clus.RandClient()\n\n\tctx := context.Background()\n\n\tr, kvApi := newTestEtcdRegistry(t, ctx, client)\n\tinitTestRegistry(ctx, t, r, kvApi)\n\tsm := newLocalManager(r, ip.IP4Net{}, ip.IP6Net{}, 60)\n\n\tl := acquireLease(ctx, t, sm)\n\n\tevents := make(chan []lease.Event)\n\tgo subnet.WatchLeases(ctx, sm, l, events)\n\n\tselect {\n\tcase evtBatch := <-events:\n\t\tfor _, evt := range evtBatch {\n\t\t\tif generateKey(evt.Lease.Subnet, evt.Lease.IPv6Subnet) == generateKey(l.Subnet, l.IPv6Subnet) {\n\t\t\t\tt.Errorf(\"WatchLeases returned our own lease\")\n\t\t\t}\n\t\t}\n\tcase <-time.After(5 * time.Second):\n\t\tlog.Info(\"no event for our own lease received, we're good\")\n\t}\n\n\texpected := ip.IP4Net{\n\t\tIP:        ip.MustParseIP4(\"10.3.30.0\"),\n\t\tPrefixLen: 24,\n\t}\n\t// Sanity check to make sure acquired lease is not this.\n\t// It shouldn't be as SubnetMin/SubnetMax in config is [10.3.1.0/24 to 10.3.25.0/24]\n\tif l.Subnet.Equal(expected) {\n\t\tt.Fatalf(\"Acquired lease conflicts with one about to create\")\n\t}\n\n\tattrs := &lease.LeaseAttrs{\n\t\tPublicIP: ip.MustParseIP4(\"1.1.1.1\"),\n\t}\n\t_, err := r.createSubnet(ctx, expected, ip.IP6Net{}, attrs, 0)\n\tif err != nil {\n\t\tt.Fatalf(\"createSubnet filed: %v\", err)\n\t}\n\n\tevtBatch := <-events\n\n\tif len(evtBatch) != 1 {\n\t\tt.Fatalf(\"WatchLeases produced wrong sized event batch: got %v, expected 1\", len(evtBatch))\n\t}\n\n\tevt := evtBatch[0]\n\n\tif evt.Type != lease.EventAdded {\n\t\tt.Fatalf(\"WatchLeases produced wrong event type\")\n\t}\n\n\tactual := evt.Lease.Subnet\n\tif !actual.Equal(expected) {\n\t\tt.Errorf(\"WatchSubnet produced wrong subnet: expected %s, got %s\", expected, actual)\n\t}\n\tlog.Info(\"test complete!\")\n}\n\nfunc TestWatchLeaseRemoved(t *testing.T) {\n\tintegration.BeforeTestExternal(t)\n\n\tclus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1})\n\tdefer clus.Terminate(t)\n\n\tclient := clus.RandClient()\n\n\tctx := context.Background()\n\n\tr, kvApi := newTestEtcdRegistry(t, ctx, client)\n\tnetKey := \"/coreos.com/network/config\"\n\tnetValue := `{ \"Network\": \"10.3.0.0/16\", \"SubnetMin\": \"10.3.1.0\", \"SubnetMax\": \"10.3.25.0\" }`\n\t_, err := kvApi.Put(ctx, netKey, netValue)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to create new entry\", err)\n\t}\n\tsm := newLocalManager(r, ip.IP4Net{}, ip.IP6Net{}, 60)\n\n\tl := acquireLease(ctx, t, sm)\n\n\tevents := make(chan []lease.Event)\n\tgo subnet.WatchLeases(ctx, sm, l, events)\n\n\t// evtBatch := <-events\n\n\t// for _, evt := range evtBatch {\n\t// \tif evt.Lease.Key() == l.Key() {\n\t// \t\tt.Errorf(\"WatchLeases returned our own lease\")\n\t// \t}\n\t// }\n\n\texpected := ip.IP4Net{IP: ip.MustParseIP4(\"10.3.31.0\"), PrefixLen: 24}\n\t// Sanity check to make sure acquired lease is not this.\n\t// It shouldn't be as SubnetMin/SubnetMax in config is [10.3.1.0/24 to 10.3.25.0/24]\n\tif l.Subnet.Equal(expected) {\n\t\tt.Fatalf(\"Acquired lease conflicts with one about to create\")\n\t}\n\tattrs := lease.LeaseAttrs{\n\t\tPublicIP: ip.MustParseIP4(\"1.1.1.1\"),\n\t}\n\t_, err = r.createSubnet(ctx, expected, ip.IP6Net{}, &attrs, time.Until(time.Now().Add(3*time.Second)))\n\tif err != nil {\n\t\tt.Errorf(\"could not create subnet: %s\", err)\n\t}\n\t//1. check that the subnet was created\n\tevtBatch := <-events\n\tif len(evtBatch) != 1 {\n\t\tt.Fatalf(\"WatchLeases produced wrong sized event batch: %#v\", evtBatch)\n\t}\n\tif len(evtBatch) != 1 {\n\t\tt.Fatalf(\"WatchLeases produced wrong sized event batch: %#v\", evtBatch)\n\t}\n\n\tevt := evtBatch[0]\n\n\tif evt.Type != lease.EventAdded {\n\t\tt.Fatalf(\"WatchLeases produced wrong event type\")\n\t}\n\n\t//2. check that the subnet was deleted after 3 seconds\n\tevtBatch = <-events\n\tif len(evtBatch) != 1 {\n\t\tt.Fatalf(\"WatchLeases produced wrong sized event batch: %#v\", evtBatch)\n\t}\n\n\tevt = evtBatch[0]\n\n\tif evt.Type != lease.EventRemoved {\n\t\tt.Fatalf(\"WatchLeases produced wrong event type\")\n\t}\n\n\tactual := evt.Lease.Subnet\n\tif !actual.Equal(expected) {\n\t\tt.Errorf(\"WatchSubnet produced wrong subnet: expected %s, got %s\", expected, actual)\n\t}\n}\n\nfunc TestCompleteLease(t *testing.T) {\n\tintegration.BeforeTestExternal(t)\n\n\tclus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1})\n\tdefer clus.Terminate(t)\n\n\tclient := clus.RandClient()\n\n\tctx := context.Background()\n\n\tr, kvApi := newTestEtcdRegistry(t, ctx, client)\n\tinitTestRegistry(ctx, t, r, kvApi)\n\tsm := newLocalManager(r, ip.IP4Net{}, ip.IP6Net{}, 60)\n\n\tl := acquireLease(ctx, t, sm)\n\n\tevts := make(chan lease.Event)\n\n\tgo func() {\n\t\tmyLease := l\n\t\tsubnet.WatchLease(ctx, sm, myLease.Subnet, myLease.IPv6Subnet, evts)\n\t}()\n\n\tevent := <-evts\n\tlog.Infof(\"got event: type: %d, subnet: %s\", event.Type, event.Lease.Subnet.String())\n\tif event.Type != lease.EventAdded {\n\t\tt.Fatal(\"WatchLease: wrong event, expected EventAdded\")\n\t} else {\n\t\tlog.Info(\"WatchLease: got EventAdded (lease creation)\")\n\t}\n\n\terr := sm.RenewLease(ctx, l)\n\tif err != nil {\n\t\tt.Errorf(\"failed to renew lease: %s\", err)\n\t}\n\n\tevent = <-evts\n\tlog.Infof(\"got event: type: %d, subnet: %s\", event.Type, event.Lease.Subnet.String())\n\tif event.Type != lease.EventAdded {\n\t\tt.Fatal(\"WatchLease: wrong event, expected EventAdded\")\n\t} else {\n\t\tlog.Info(\"WatchLease: got EventAdded (lease renewal)\")\n\t}\n\n\tleaseKey := path.Join(\"/coreos.com/network/\", \"subnets\", subnet.MakeSubnetKey(l.Subnet, ip.IP6Net{}))\n\t_, err = kvApi.Delete(ctx, leaseKey)\n\tif err != nil {\n\t\tt.Errorf(\"could not delete lease: %s\", err)\n\t}\n\n\tlog.Info(\"lease deleted manually\")\n\tevent = <-evts\n\tlog.Infof(\"got event: type: %d, subnet: %s\", event.Type, event.Lease.Subnet.String())\n\tif event.Type != lease.EventRemoved {\n\t\tt.Fatal(\"WatchLease: wrong event, expected EventRemoved\")\n\t}\n\n}\n\ntype leaseData struct {\n\tDummy string\n}\n\nfunc TestRenewLease(t *testing.T) {\n\n\tintegration.BeforeTestExternal(t)\n\n\tclus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1})\n\tdefer clus.Terminate(t)\n\n\tclient := clus.RandClient()\n\n\tctx := context.Background()\n\n\tr, kvApi := newTestEtcdRegistry(t, ctx, client)\n\tnetKey := \"/coreos.com/network/config\"\n\tnetValue := `{ \"Network\": \"10.3.0.0/16\", \"SubnetMin\": \"10.3.1.0\", \"SubnetMax\": \"10.3.25.0\" }`\n\t_, err := kvApi.Put(ctx, netKey, netValue)\n\tif err != nil {\n\t\tt.Fatal(\"Failed to create new entry\", err)\n\t}\n\tsm := newLocalManager(r, ip.IP4Net{}, ip.IP6Net{}, 60)\n\n\t// Create LeaseAttrs\n\textIaddr, _ := ip.ParseIP4(\"1.2.3.4\")\n\texpectedAttrs := lease.LeaseAttrs{\n\t\tPublicIP:    extIaddr,\n\t\tBackendType: \"vxlan\",\n\t}\n\n\tld, err := json.Marshal(&leaseData{Dummy: \"test string\"})\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to marshal leaseData: %v\", err)\n\t}\n\texpectedAttrs.BackendData = json.RawMessage(ld)\n\n\t// Acquire lease\n\tl, err := sm.AcquireLease(ctx, &expectedAttrs)\n\tif err != nil {\n\t\tt.Fatal(\"AcquireLease failed: \", err)\n\t}\n\n\t//wait a bit so that RenewLease has an effect\n\ttime.Sleep(10 * time.Second)\n\tif err := sm.RenewLease(ctx, l); err != nil {\n\t\tt.Fatal(\"RenewLease failed: \", err)\n\t}\n\t//we expect the new lease to have an expiration date in exactly 24h\n\tacceptableMargin := 10 * time.Second\n\texpectedExpiration := time.Now().Add(subnetTTL).Round(time.Duration(acceptableMargin))\n\n\tetcdResp, err := kvApi.Get(ctx, \"/coreos.com/network/subnets\", etcd.WithPrefix())\n\tif err != nil {\n\t\tt.Errorf(\"failed to renew lease: could not read leases: %s\", err)\n\t}\n\tfor _, resp := range etcdResp.Kvs {\n\t\tlog.Infof(\"found key: %s\", resp.Key)\n\t\tsn, _ := subnet.ParseSubnetKey(string(resp.Key))\n\t\tif sn.Equal(l.Subnet) {\n\n\t\t\tttlResp, err := client.TimeToLive(ctx, etcd.LeaseID(resp.Lease))\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Failed to renew lease: could not \")\n\t\t\t}\n\t\t\tleaseExpiration := time.Now().Add(time.Duration(ttlResp.TTL) * time.Second).Round(time.Duration(acceptableMargin))\n\n\t\t\tif diff := leaseExpiration.Sub(expectedExpiration); diff < -acceptableMargin || diff > acceptableMargin {\n\t\t\t\tt.Errorf(\"Failed to renew lease: bad expiration; expected %v, got %v\", expectedExpiration, leaseExpiration)\n\t\t\t}\n\t\t\t//check that the value is correct\n\t\t\tattrs := &lease.LeaseAttrs{}\n\t\t\terr = json.Unmarshal(resp.Value, attrs)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(\"Failed to renew lease: could not unmarshal attrs\")\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(attrs, &expectedAttrs) {\n\t\t\t\tt.Errorf(\"LeaseAttrs changed: was %#v, now %#v\", expectedAttrs, attrs)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\n\tt.Fatal(\"Failed to find acquired lease\")\n}\n\nfunc inAllocatableRange(ctx context.Context, sm subnet.Manager, ipn ip.IP4Net) bool {\n\tcfg, err := sm.GetNetworkConfig(ctx)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\treturn ipn.IP >= cfg.SubnetMin || ipn.IP <= cfg.SubnetMax\n}\n\nfunc generateKey(ipv4Subnet ip.IP4Net, ipv6Subnet ip.IP6Net) string {\n\treturn subnet.MakeSubnetKey(ipv4Subnet, ipv6Subnet)\n}\n"
  },
  {
    "path": "pkg/subnet/kube/annotations.go",
    "content": "// Copyright 2018 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage kube\n\nimport (\n\t\"errors\"\n\t\"regexp\"\n\t\"strings\"\n)\n\ntype annotations struct {\n\tSubnetKubeManaged          string\n\tBackendData                string\n\tBackendV6Data              string\n\tBackendType                string\n\tBackendPublicIP            string\n\tBackendPublicIPv6          string\n\tBackendNodePublicIP        string\n\tBackendNodePublicIPv6      string\n\tBackendPublicIPOverwrite   string\n\tBackendPublicIPv6Overwrite string\n}\n\nfunc newAnnotations(prefix string) (annotations, error) {\n\tslashCnt := strings.Count(prefix, \"/\")\n\tif slashCnt > 1 {\n\t\treturn annotations{}, errors.New(\"subnet/kube: prefix can contain at most single slash\")\n\t}\n\tif slashCnt == 0 {\n\t\tprefix += \"/\"\n\t}\n\tif !strings.HasSuffix(prefix, \"/\") && !strings.HasSuffix(prefix, \"-\") {\n\t\tprefix += \"-\"\n\t}\n\n\t// matches is a regexp matching the format used by the kubernetes for\n\t// annotations. Following rules apply:\n\t//\n\t//\t- must start with FQDN - must contain at most one slash \"/\"\n\t//\t- must contain only lowercase letters, nubers, underscores,\n\t//\t  hyphens, dots and slash\n\tmatches, err := regexp.MatchString(`(?:[a-z0-9_-]+\\.)+[a-z0-9_-]+/(?:[a-z0-9_-]+-)?$`, prefix)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tif !matches {\n\t\treturn annotations{}, errors.New(\"subnet/kube: prefix must be in a format: fqdn/[0-9a-z-_]*\")\n\t}\n\n\ta := annotations{\n\t\tSubnetKubeManaged:          prefix + \"kube-subnet-manager\",\n\t\tBackendData:                prefix + \"backend-data\",\n\t\tBackendV6Data:              prefix + \"backend-v6-data\",\n\t\tBackendType:                prefix + \"backend-type\",\n\t\tBackendPublicIP:            prefix + \"public-ip\",\n\t\tBackendNodePublicIP:        prefix + \"node-public-ip\",\n\t\tBackendPublicIPOverwrite:   prefix + \"public-ip-overwrite\",\n\t\tBackendPublicIPv6:          prefix + \"public-ipv6\",\n\t\tBackendNodePublicIPv6:      prefix + \"node-public-ipv6\",\n\t\tBackendPublicIPv6Overwrite: prefix + \"public-ipv6-overwrite\",\n\t}\n\n\treturn a, nil\n}\n"
  },
  {
    "path": "pkg/subnet/kube/annotations_test.go",
    "content": "// Copyright 2018 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage kube\n\nimport \"testing\"\n\nfunc Test_newAnnotations(t *testing.T) {\n\ttestCases := []struct {\n\t\tprefix              string\n\t\texpectedBackendType string\n\t\thasError            bool\n\t}{\n\t\t{\n\t\t\tprefix:              \"flannel.alpha.coreos.com\",\n\t\t\texpectedBackendType: \"flannel.alpha.coreos.com/backend-type\",\n\t\t},\n\t\t{\n\t\t\tprefix:              \"flannel.alpha.coreos.com/\",\n\t\t\texpectedBackendType: \"flannel.alpha.coreos.com/backend-type\",\n\t\t},\n\t\t{\n\t\t\tprefix:              \"flannel.alpha.coreos.com/prefix\",\n\t\t\texpectedBackendType: \"flannel.alpha.coreos.com/prefix-backend-type\",\n\t\t},\n\t\t{\n\t\t\tprefix:              \"flannel.alpha.coreos.com/prefix-\",\n\t\t\texpectedBackendType: \"flannel.alpha.coreos.com/prefix-backend-type\",\n\t\t},\n\t\t{\n\t\t\tprefix:              \"org.com\",\n\t\t\texpectedBackendType: \"org.com/backend-type\",\n\t\t},\n\t\t{\n\t\t\tprefix:              \"org9.com\",\n\t\t\texpectedBackendType: \"org9.com/backend-type\",\n\t\t},\n\t\t{\n\t\t\tprefix:              \"org.com/9\",\n\t\t\texpectedBackendType: \"org.com/9-backend-type\",\n\t\t},\n\t\t{\n\t\t\t// Not a fqdn.\n\t\t\tprefix:   \"org\",\n\t\t\thasError: true,\n\t\t},\n\t\t{\n\t\t\t// Not a fqdn before /.\n\t\t\tprefix:   \"org/\",\n\t\t\thasError: true,\n\t\t},\n\t\t{\n\t\t\t// Not a fqdn before /.\n\t\t\tprefix:   \"org/prefix\",\n\t\t\thasError: true,\n\t\t},\n\t\t{\n\t\t\t// Uppercase letters.\n\t\t\tprefix:   \"org.COM\",\n\t\t\thasError: true,\n\t\t},\n\t\t{\n\t\t\t// Uppercase letters.\n\t\t\tprefix:   \"org.com/PREFIX\",\n\t\t\thasError: true,\n\t\t},\n\t}\n\n\tfor i, tc := range testCases {\n\t\tas, err := newAnnotations(tc.prefix)\n\n\t\tif err != nil && !tc.hasError {\n\t\t\tt.Errorf(\"#%d: error = %s, want nil\", i, err)\n\t\t\tcontinue\n\t\t}\n\t\tif err == nil && tc.hasError {\n\t\t\tt.Errorf(\"#%d: error = nil, want non-nil\", i)\n\t\t\tcontinue\n\t\t}\n\n\t\tif as.BackendType != tc.expectedBackendType {\n\t\t\tt.Errorf(\"#%d: BackendType = %s, want %s\", i, as.BackendType, tc.expectedBackendType)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/subnet/kube/kube.go",
    "content": "// Copyright 2016 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage kube\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/subnet\"\n\t\"golang.org/x/sync/semaphore\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/fields\"\n\t\"k8s.io/apimachinery/pkg/types\"\n\t\"k8s.io/apimachinery/pkg/util/strategicpatch\"\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\tclientset \"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\t\"k8s.io/client-go/tools/cache\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nvar (\n\tErrUnimplemented = errors.New(\"unimplemented\")\n)\n\nconst (\n\tresyncPeriod              = 5 * time.Minute\n\tnodeControllerSyncTimeout = 10 * time.Minute\n)\n\ntype subnetFileInfo struct {\n\tpath   string\n\tipMask bool\n\tsn     ip.IP4Net\n\tIPv6sn ip.IP6Net\n\tmtu    int\n}\n\ntype kubeSubnetManager struct {\n\tenableIPv4                bool\n\tenableIPv6                bool\n\tannotations               annotations\n\tannotationPrefix          string\n\tclient                    clientset.Interface\n\tnodeName                  string\n\tnodeStore                 cache.Store\n\tnodeController            cache.Controller\n\tsubnetConf                *subnet.Config\n\tevents                    chan lease.Event\n\tasyncSendSemaphore        *semaphore.Weighted\n\tclusterCIDRController     cache.Controller\n\tsetNodeNetworkUnavailable bool\n\tdisableNodeInformer       bool\n\tsnFileInfo                *subnetFileInfo\n}\n\nfunc NewSubnetManager(ctx context.Context, apiUrl, kubeconfig, prefix, netConfPath string, setNodeNetworkUnavailable bool) (subnet.Manager, error) {\n\tvar cfg *rest.Config\n\tvar err error\n\t// Try to build kubernetes config from a master url or a kubeconfig filepath. If neither masterUrl\n\t// or kubeconfigPath are passed in we fall back to inClusterConfig. If inClusterConfig fails,\n\t// we fallback to the default config.\n\tcfg, err = clientcmd.BuildConfigFromFlags(apiUrl, kubeconfig)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"fail to create kubernetes config: %v\", err)\n\t}\n\n\tc, err := clientset.NewForConfig(cfg)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"unable to initialize client: %v\", err)\n\t}\n\n\t// The kube subnet mgr needs to know the k8s node name that it's running on so it can annotate it.\n\t// If we're running as a pod then the POD_NAME and POD_NAMESPACE will be populated and can be used to find the node\n\t// name. Otherwise, the environment variable NODE_NAME can be passed in.\n\tnodeName := os.Getenv(\"NODE_NAME\")\n\tif nodeName == \"\" {\n\t\tpodName := os.Getenv(\"POD_NAME\")\n\t\tpodNamespace := os.Getenv(\"POD_NAMESPACE\")\n\t\tif podName == \"\" || podNamespace == \"\" {\n\t\t\treturn nil, fmt.Errorf(\"env variables POD_NAME and POD_NAMESPACE must be set\")\n\t\t}\n\n\t\tpod, err := c.CoreV1().Pods(podNamespace).Get(ctx, podName, metav1.GetOptions{})\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error retrieving pod spec for '%s/%s': %v\", podNamespace, podName, err)\n\t\t}\n\t\tnodeName = pod.Spec.NodeName\n\t\tif nodeName == \"\" {\n\t\t\treturn nil, fmt.Errorf(\"node name not present in pod spec '%s/%s'\", podNamespace, podName)\n\t\t}\n\t}\n\n\tnetConf, err := os.ReadFile(netConfPath)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to read net conf: %v\", err)\n\t}\n\n\tsc, err := subnet.ParseConfig(string(netConf))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error parsing subnet config: %s\", err)\n\t}\n\n\tsm, err := newKubeSubnetManager(ctx, c, sc, nodeName, prefix)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error creating network manager: %s\", err)\n\t}\n\tsm.setNodeNetworkUnavailable = setNodeNetworkUnavailable\n\n\tif sm.disableNodeInformer {\n\t\tlog.Infof(\"Node controller skips sync\")\n\t} else {\n\t\tgo sm.Run(ctx)\n\n\t\tlog.Infof(\"Waiting %s for node controller to sync\", nodeControllerSyncTimeout)\n\t\terr = wait.PollUntilContextTimeout(ctx, time.Second, nodeControllerSyncTimeout, true, func(context.Context) (bool, error) {\n\t\t\tch := make(chan bool, 1)\n\t\t\tgo func() {\n\t\t\t\t// HasSynced() is a blocking call waiting on\n\t\t\t\t// the DeltaFIFO mutex that's also used by other\n\t\t\t\t// cache controller callbacks calling wait or Update\n\t\t\t\t// use a channel/select to not block the main thread\n\t\t\t\tch <- sm.nodeController.HasSynced()\n\t\t\t}()\n\t\t\tselect {\n\t\t\tcase synced := <-ch:\n\t\t\t\treturn synced, nil\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn false, ctx.Err()\n\t\t\tcase <-time.After(time.Second):\n\t\t\t\treturn false, nil // or error to break\n\t\t\t}\n\t\t})\n\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Node controller sync not completed within 1s: %v\", err)\n\t\t\treturn sm, fmt.Errorf(\"error waiting for nodeController to sync state: %w\", err)\n\t\t}\n\t\tlog.Infof(\"Node controller sync successful\")\n\t}\n\n\treturn sm, nil\n}\n\n// newKubeSubnetManager fills the kubeSubnetManager. The most important part is the controller which will\n// watch for kubernetes node updates\nfunc newKubeSubnetManager(ctx context.Context, c clientset.Interface, sc *subnet.Config, nodeName, prefix string) (*kubeSubnetManager, error) {\n\tvar err error\n\tvar ksm kubeSubnetManager\n\tksm.annotationPrefix = prefix\n\tksm.annotations, err = newAnnotations(prefix)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tksm.enableIPv4 = sc.EnableIPv4\n\tksm.enableIPv6 = sc.EnableIPv6\n\tksm.client = c\n\tksm.nodeName = nodeName\n\tksm.subnetConf = sc\n\tscale := 5000\n\tscaleStr := os.Getenv(\"EVENT_QUEUE_DEPTH\")\n\tif scaleStr != \"\" {\n\t\tn, err := strconv.Atoi(scaleStr)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"env EVENT_QUEUE_DEPTH=%s format error: %v\", scaleStr, err)\n\t\t}\n\t\tif n > 0 {\n\t\t\tscale = n\n\t\t}\n\t}\n\tksm.events = make(chan lease.Event, scale)\n\tksm.asyncSendSemaphore = semaphore.NewWeighted(100)\n\t// when backend type is alloc, someone else (e.g. cloud-controller-managers) is taking care of the routing, thus we do not need informer\n\t// See https://github.com/flannel-io/flannel/issues/1617\n\tif sc.BackendType == \"alloc\" {\n\t\tksm.disableNodeInformer = true\n\t}\n\tif !ksm.disableNodeInformer {\n\t\tlisterWatcher := cache.NewListWatchFromClient(\n\t\t\tksm.client.CoreV1().RESTClient(),\n\t\t\t\"nodes\",\n\t\t\t\"\",\n\t\t\tfields.Everything())\n\n\t\thandler := cache.ResourceEventHandlerFuncs{\n\t\t\tAddFunc: func(obj interface{}) {\n\t\t\t\tksm.handleAddLeaseEvent(ctx, lease.EventAdded, obj)\n\t\t\t},\n\t\t\tUpdateFunc: func(oldObj, newObj interface{}) {\n\t\t\t\tksm.handleUpdateLeaseEvent(ctx, oldObj, newObj)\n\t\t\t},\n\t\t\tDeleteFunc: func(obj interface{}) {\n\t\t\t\t_, isNode := obj.(*v1.Node)\n\t\t\t\t// We can get DeletedFinalStateUnknown instead of *api.Node here and we need to handle that correctly.\n\t\t\t\tif !isNode {\n\t\t\t\t\tdeletedState, ok := obj.(cache.DeletedFinalStateUnknown)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tlog.Infof(\"Error received unexpected object: %v\", obj)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tnode, ok := deletedState.Obj.(*v1.Node)\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tlog.Infof(\"Error deletedFinalStateUnknown contained non-Node object: %v\", deletedState.Obj)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tobj = node\n\t\t\t\t}\n\t\t\t\tksm.handleAddLeaseEvent(ctx, lease.EventRemoved, obj)\n\t\t\t},\n\t\t}\n\t\tstore, controller := cache.NewInformerWithOptions(cache.InformerOptions{\n\t\t\tListerWatcher: listerWatcher,\n\t\t\tObjectType:    &v1.Node{},\n\t\t\tResyncPeriod:  resyncPeriod,\n\t\t\tHandler:       handler,\n\t\t\tIndexers:      cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},\n\t\t})\n\n\t\tksm.nodeController = controller\n\t\tksm.nodeStore = store\n\t}\n\n\treturn &ksm, nil\n}\n\nfunc (ksm *kubeSubnetManager) enqueueLeaseEvent(ctx context.Context, evt lease.Event, nodeName string) {\n\t// Try to send immediately\n\tselect {\n\tcase ksm.events <- evt:\n\t\treturn\n\tdefault:\n\t\tlog.Infof(\"Channel buffer full, add event asynchronously\")\n\t}\n\n\t// Instead of select with default, *block* until a slot is free\n\t// Use a context with no timeout to block until a slot is available\n\tif err := ksm.asyncSendSemaphore.Acquire(ctx, 1); err != nil {\n\t\tlog.Errorf(\"error in acquiring semaphore for async event send, dropping event: %v\", err)\n\t\treturn\n\t}\n\n\tgo func() {\n\t\tdefer func() { ksm.asyncSendSemaphore.Release(1) }() // release slot when done\n\n\t\tbackoff := 100 * time.Millisecond\n\t\tmaxBackoff := 5 * time.Second\n\n\t\tfor {\n\n\t\t\tticker := time.NewTicker(backoff)\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\tlog.Errorf(\"Context cancelled while retrying lease event for node %q\", nodeName)\n\t\t\t\tticker.Stop()\n\t\t\t\treturn\n\t\t\tcase ksm.events <- evt:\n\t\t\t\tlog.Infof(\"Async requeued lease event for node %q\", nodeName)\n\t\t\t\tticker.Stop()\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t\t// events channel still full, retry with exp backoff\n\t\t\t\tticker.Stop()\n\t\t\t\tbackoff *= 2\n\t\t\t\tif backoff > maxBackoff {\n\t\t\t\t\tbackoff = maxBackoff\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (ksm *kubeSubnetManager) handleAddLeaseEvent(ctx context.Context, et lease.EventType, obj interface{}) {\n\tn := obj.(*v1.Node)\n\tif s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != \"true\" {\n\t\treturn\n\t}\n\n\tl, err := ksm.nodeToLease(*n)\n\tif err != nil {\n\t\tlog.Infof(\"Error turning node %q to lease: %v\", n.Name, err)\n\t\treturn\n\t}\n\tksm.enqueueLeaseEvent(ctx, lease.Event{Type: et, Lease: l}, n.Name)\n}\n\n// handleUpdateLeaseEvent verifies if anything relevant changed in the node object: either\n// ksm.annotations.BackendData, ksm.annotations.BackendType or ksm.annotations.BackendPublicIP\nfunc (ksm *kubeSubnetManager) handleUpdateLeaseEvent(ctx context.Context, oldObj, newObj interface{}) {\n\to := oldObj.(*v1.Node)\n\tn := newObj.(*v1.Node)\n\tif s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != \"true\" {\n\t\treturn\n\t}\n\tvar changed = true\n\tif ksm.enableIPv4 && o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&\n\t\to.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&\n\t\to.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] {\n\t\tchanged = false\n\t}\n\n\tif ksm.enableIPv6 && o.Annotations[ksm.annotations.BackendV6Data] == n.Annotations[ksm.annotations.BackendV6Data] &&\n\t\to.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&\n\t\to.Annotations[ksm.annotations.BackendPublicIPv6] == n.Annotations[ksm.annotations.BackendPublicIPv6] {\n\t\tchanged = false\n\t}\n\n\tif !changed {\n\t\treturn // No change to lease\n\t}\n\n\tl, err := ksm.nodeToLease(*n)\n\tif err != nil {\n\t\tlog.Infof(\"Error turning node %q to lease: %v\", n.Name, err)\n\t\treturn\n\t}\n\tksm.enqueueLeaseEvent(ctx, lease.Event{Type: lease.EventAdded, Lease: l}, n.Name)\n}\n\nfunc (ksm *kubeSubnetManager) GetNetworkConfig(ctx context.Context) (*subnet.Config, error) {\n\treturn ksm.subnetConf, nil\n}\n\n// AcquireLease adds the flannel specific node annotations (defined in the struct LeaseAttrs) and returns a lease\n// with important information for the backend, such as the subnet. This function is called once by the backend when\n// registering\nfunc (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *lease.LeaseAttrs) (*lease.Lease, error) {\n\tvar cachedNode *v1.Node\n\twaitErr := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(context.Context) (done bool, err error) {\n\t\tif ksm.disableNodeInformer {\n\t\t\tcachedNode, err = ksm.client.CoreV1().Nodes().Get(ctx, ksm.nodeName, metav1.GetOptions{ResourceVersion: \"0\"})\n\t\t\tif err != nil {\n\t\t\t\tlog.V(2).Infof(\"Failed to get node %q: %v\", ksm.nodeName, err)\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t} else {\n\t\t\tnodeIface, exists, err := ksm.nodeStore.GetByKey(ksm.nodeName)\n\t\t\tif err != nil {\n\t\t\t\tlog.V(2).Infof(\"failed to get node %q: %v\", ksm.nodeName, err)\n\t\t\t\treturn false, nil\n\t\t\t} else if !exists {\n\t\t\t\tlog.V(2).Infof(\"node %q does not exist \", ksm.nodeName)\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t\tcachedNode = nodeIface.(*v1.Node)\n\t\t}\n\t\treturn true, nil\n\t})\n\tif waitErr != nil {\n\t\treturn nil, fmt.Errorf(\"timeout contacting kube-api, failed to patch node %q. Error: %v\", ksm.nodeName, waitErr)\n\t}\n\n\tn := cachedNode.DeepCopy()\n\tif n.Spec.PodCIDR == \"\" {\n\t\treturn nil, fmt.Errorf(\"node %q pod cidr not assigned\", ksm.nodeName)\n\t}\n\n\tvar bd, v6Bd []byte\n\tbd, err := attrs.BackendData.MarshalJSON()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tv6Bd, err = attrs.BackendV6Data.MarshalJSON()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar cidr, ipv6Cidr *net.IPNet\n\tswitch {\n\tcase len(n.Spec.PodCIDRs) == 0:\n\t\t_, parseCidr, err := net.ParseCIDR(n.Spec.PodCIDR)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(parseCidr.IP) == net.IPv4len {\n\t\t\tcidr = parseCidr\n\t\t} else if len(parseCidr.IP) == net.IPv6len {\n\t\t\tipv6Cidr = parseCidr\n\t\t}\n\tcase len(n.Spec.PodCIDRs) < 3:\n\t\tfor _, podCidr := range n.Spec.PodCIDRs {\n\t\t\t_, parseCidr, err := net.ParseCIDR(podCidr)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif len(parseCidr.IP) == net.IPv4len {\n\t\t\t\tcidr = parseCidr\n\t\t\t} else if len(parseCidr.IP) == net.IPv6len {\n\t\t\t\tipv6Cidr = parseCidr\n\t\t\t}\n\t\t}\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"node %q pod cidrs should be IPv4/IPv6 only or dualstack\", ksm.nodeName)\n\t}\n\n\tif (n.Annotations[ksm.annotations.BackendData] != string(bd) ||\n\t\tn.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||\n\t\tn.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||\n\t\tn.Annotations[ksm.annotations.SubnetKubeManaged] != \"true\" ||\n\t\t(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != \"\" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String())) ||\n\t\t(attrs.PublicIPv6 != nil &&\n\t\t\t(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||\n\t\t\t\tn.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||\n\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||\n\t\t\t\tn.Annotations[ksm.annotations.SubnetKubeManaged] != \"true\" ||\n\t\t\t\t(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != \"\" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String()))) {\n\t\tn.Annotations[ksm.annotations.BackendType] = attrs.BackendType\n\n\t\t//TODO -i only vxlan and host-gw backends support dual stack now.\n\t\tif (attrs.BackendType == \"vxlan\" && string(bd) != \"null\") || (attrs.BackendType == \"wireguard\" && string(bd) != \"null\") || attrs.BackendType != \"vxlan\" {\n\t\t\tn.Annotations[ksm.annotations.BackendData] = string(bd)\n\t\t\tif n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != \"\" {\n\t\t\t\tif n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {\n\t\t\t\t\tlog.Infof(\"Overriding public ip with '%s' from node annotation '%s'\",\n\t\t\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIPOverwrite],\n\t\t\t\t\t\tksm.annotations.BackendPublicIPOverwrite)\n\t\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()\n\t\t\t}\n\t\t}\n\n\t\tif (attrs.BackendType == \"vxlan\" && string(v6Bd) != \"null\") ||\n\t\t\t(attrs.BackendType == \"wireguard\" && string(v6Bd) != \"null\" && attrs.PublicIPv6 != nil) ||\n\t\t\t(attrs.BackendType == \"host-gw\" && attrs.PublicIPv6 != nil) ||\n\t\t\t(attrs.BackendType == \"extension\" && attrs.PublicIPv6 != nil) {\n\t\t\tn.Annotations[ksm.annotations.BackendV6Data] = string(v6Bd)\n\t\t\tif n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != \"\" {\n\t\t\t\tif n.Annotations[ksm.annotations.BackendPublicIPv6] != n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] {\n\t\t\t\t\tlog.Infof(\"Overriding public ipv6 with '%s' from node annotation '%s'\",\n\t\t\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIPv6Overwrite],\n\t\t\t\t\t\tksm.annotations.BackendPublicIPv6Overwrite)\n\t\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIPv6] = n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite]\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tn.Annotations[ksm.annotations.BackendPublicIPv6] = attrs.PublicIPv6.String()\n\t\t\t}\n\t\t}\n\t\tn.Annotations[ksm.annotations.SubnetKubeManaged] = \"true\"\n\n\t\toldData, err := json.Marshal(cachedNode)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tnewData, err := json.Marshal(n)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tpatchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to create patch for node %q: %v\", ksm.nodeName, err)\n\t\t}\n\n\t\twaitErr := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(context.Context) (done bool, err error) {\n\t\t\t_, err = ksm.client.CoreV1().Nodes().Patch(ctx, ksm.nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, \"status\")\n\t\t\tif err != nil {\n\t\t\t\tlog.V(2).Infof(\"Failed to patch node %q: %v\", ksm.nodeName, err)\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t\treturn true, nil\n\t\t})\n\t\tif waitErr != nil {\n\t\t\treturn nil, fmt.Errorf(\"timeout contacting kube-api, failed to patch node %q. Error: %v\", ksm.nodeName, waitErr)\n\t\t}\n\t}\n\n\tlease := &lease.Lease{\n\t\tAttrs:      *attrs,\n\t\tExpiration: time.Now().Add(24 * time.Hour),\n\t}\n\tif cidr != nil && ksm.enableIPv4 {\n\t\tif ksm.subnetConf.Network.Empty() || !containsCIDR(ksm.subnetConf.Network.ToIPNet(), cidr) {\n\t\t\treturn nil, fmt.Errorf(\"subnet %q specified in the flannel net config doesn't contain %q PodCIDR of the %q node\", ksm.subnetConf.Network, cidr, ksm.nodeName)\n\t\t}\n\n\t\tlease.Subnet = ip.FromIPNet(cidr)\n\t}\n\tif ipv6Cidr != nil && ksm.enableIPv6 {\n\t\tif ksm.subnetConf.IPv6Network.Empty() || !containsCIDR(ksm.subnetConf.IPv6Network.ToIPNet(), ipv6Cidr) {\n\t\t\treturn nil, fmt.Errorf(\"subnet %q specified in the flannel net config doesn't contain %q IPv6 PodCIDR of the %q node\", ksm.subnetConf.IPv6Network, ipv6Cidr, ksm.nodeName)\n\t\t}\n\n\t\tlease.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)\n\t}\n\t//TODO - only vxlan, host-gw and wireguard backends support dual stack now.\n\tif attrs.BackendType != \"vxlan\" && attrs.BackendType != \"host-gw\" && attrs.BackendType != \"wireguard\" {\n\t\tlease.EnableIPv4 = true\n\t\tlease.EnableIPv6 = false\n\t}\n\treturn lease, nil\n}\n\n// WatchLeases waits for the kubeSubnetManager to provide an event in case something relevant changed in the node data\nfunc (ksm *kubeSubnetManager) WatchLeases(ctx context.Context, receiver chan []lease.LeaseWatchResult) error {\n\tfor {\n\t\tselect {\n\t\tcase event := <-ksm.events:\n\t\t\treceiver <- []lease.LeaseWatchResult{\n\t\t\t\t{\n\t\t\t\t\tEvents: []lease.Event{event},\n\t\t\t\t}}\n\t\tcase <-ctx.Done():\n\t\t\tclose(receiver)\n\t\t\treturn ctx.Err()\n\t\t}\n\t}\n}\n\nfunc (ksm *kubeSubnetManager) Run(ctx context.Context) {\n\tlog.Infof(\"Starting kube subnet manager\")\n\tksm.nodeController.Run(ctx.Done())\n}\n\n// nodeToLease updates the lease with information fetch from the node, e.g. PodCIDR\nfunc (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l lease.Lease, err error) {\n\tif ksm.enableIPv4 {\n\t\tl.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])\n\t\tif err != nil {\n\t\t\treturn l, err\n\t\t}\n\t\tl.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])\n\n\t\tvar cidr *net.IPNet\n\t\tswitch {\n\t\tcase len(n.Spec.PodCIDRs) == 0:\n\t\t\t_, cidr, err = net.ParseCIDR(n.Spec.PodCIDR)\n\t\t\tif err != nil {\n\t\t\t\treturn l, err\n\t\t\t}\n\t\tcase len(n.Spec.PodCIDRs) < 3:\n\t\t\tlog.Infof(\"Creating the node lease for IPv4. This is the n.Spec.PodCIDRs: %v\", n.Spec.PodCIDRs)\n\t\t\tfor _, podCidr := range n.Spec.PodCIDRs {\n\t\t\t\t_, parseCidr, err := net.ParseCIDR(podCidr)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn l, err\n\t\t\t\t}\n\t\t\t\tif len(parseCidr.IP) == net.IPv4len {\n\t\t\t\t\tcidr = parseCidr\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\treturn l, fmt.Errorf(\"node %q pod cidrs should be IPv4/IPv6 only or dualstack\", ksm.nodeName)\n\t\t}\n\t\tif cidr == nil {\n\t\t\treturn l, fmt.Errorf(\"missing IPv4 address on n.Spec.PodCIDRs\")\n\t\t}\n\t\tl.Subnet = ip.FromIPNet(cidr)\n\t\tl.EnableIPv4 = ksm.enableIPv4\n\t}\n\n\tif ksm.enableIPv6 {\n\t\tl.Attrs.PublicIPv6, err = ip.ParseIP6(n.Annotations[ksm.annotations.BackendPublicIPv6])\n\t\tif err != nil {\n\t\t\treturn l, err\n\t\t}\n\t\tl.Attrs.BackendV6Data = json.RawMessage(n.Annotations[ksm.annotations.BackendV6Data])\n\n\t\tvar ipv6Cidr *net.IPNet\n\t\tswitch {\n\t\tcase len(n.Spec.PodCIDRs) == 0:\n\t\t\t_, ipv6Cidr, err = net.ParseCIDR(n.Spec.PodCIDR)\n\t\t\tif err != nil {\n\t\t\t\treturn l, err\n\t\t\t}\n\t\tcase len(n.Spec.PodCIDRs) < 3:\n\t\t\tlog.Infof(\"Creating the node lease for IPv6. This is the n.Spec.PodCIDRs: %v\", n.Spec.PodCIDRs)\n\t\t\tfor _, podCidr := range n.Spec.PodCIDRs {\n\t\t\t\t_, parseCidr, err := net.ParseCIDR(podCidr)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn l, err\n\t\t\t\t}\n\t\t\t\tif len(parseCidr.IP) == net.IPv6len {\n\t\t\t\t\tipv6Cidr = parseCidr\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\treturn l, fmt.Errorf(\"node %q pod cidrs should be IPv4/IPv6 only or dualstack\", ksm.nodeName)\n\t\t}\n\t\tif ipv6Cidr == nil {\n\t\t\treturn l, fmt.Errorf(\"missing IPv6 address on n.Spec.PodCIDRs\")\n\t\t}\n\t\tl.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)\n\t\tl.EnableIPv6 = ksm.enableIPv6\n\t}\n\tl.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]\n\treturn l, nil\n}\n\n// RenewLease: unimplemented\nfunc (ksm *kubeSubnetManager) RenewLease(ctx context.Context, lease *lease.Lease) error {\n\treturn ErrUnimplemented\n}\n\nfunc (ksm *kubeSubnetManager) WatchLease(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, receiver chan []lease.LeaseWatchResult) error {\n\treturn ErrUnimplemented\n}\n\nfunc (ksm *kubeSubnetManager) Name() string {\n\treturn fmt.Sprintf(\"Kubernetes Subnet Manager - %s\", ksm.nodeName)\n}\n\n// CompleteLease Set Kubernetes NodeNetworkUnavailable to false when starting\n// https://kubernetes.io/docs/concepts/architecture/nodes/#condition\nfunc (ksm *kubeSubnetManager) CompleteLease(ctx context.Context, lease *lease.Lease, wg *sync.WaitGroup) error {\n\tif ksm.clusterCIDRController != nil {\n\t\t//start clusterController after all subnet manager has been fully initialized\n\t\tlog.Info(\"starting clusterCIDR controller...\")\n\t\tgo ksm.clusterCIDRController.Run(ctx.Done())\n\n\t\tlog.Infof(\"Waiting %s for clusterCIDR controller to sync...\", nodeControllerSyncTimeout)\n\t\terr := wait.PollUntilContextTimeout(ctx, time.Second, nodeControllerSyncTimeout, true, func(context.Context) (bool, error) {\n\t\t\treturn ksm.clusterCIDRController.HasSynced(), nil\n\t\t})\n\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error waiting for clusterCIDR to sync state: %v\", err)\n\t\t}\n\t\tlog.Infof(\"clusterCIDR controller sync successful\")\n\t}\n\tif !ksm.setNodeNetworkUnavailable {\n\t\t// not set NodeNetworkUnavailable NodeCondition\n\t\treturn nil\n\t}\n\n\tcondition := v1.NodeCondition{\n\t\tType:               v1.NodeNetworkUnavailable,\n\t\tStatus:             v1.ConditionFalse,\n\t\tReason:             \"FlannelIsUp\",\n\t\tMessage:            \"Flannel is running on this node\",\n\t\tLastTransitionTime: metav1.Now(),\n\t\tLastHeartbeatTime:  metav1.Now(),\n\t}\n\traw, err := json.Marshal(&[]v1.NodeCondition{condition})\n\tif err != nil {\n\t\treturn err\n\t}\n\tpatch := []byte(fmt.Sprintf(`{\"status\":{\"conditions\":%s}}`, raw))\n\t_, err = ksm.client.CoreV1().Nodes().PatchStatus(ctx, ksm.nodeName, patch)\n\treturn err\n}\n\nfunc containsCIDR(ipnet1, ipnet2 *net.IPNet) bool {\n\tones1, _ := ipnet1.Mask.Size()\n\tones2, _ := ipnet2.Mask.Size()\n\treturn ones1 <= ones2 && ipnet1.Contains(ipnet2.IP)\n}\n\n// HandleSubnetFile writes the configuration file used by the CNI flannel plugin\n// and stores the immutable data in a dedicated struct of the subnet manager\n// so that we can update the file later when a clustercidr resource is created.\nfunc (m *kubeSubnetManager) HandleSubnetFile(path string, config *subnet.Config, ipMasq bool, sn ip.IP4Net, ipv6sn ip.IP6Net, mtu int) error {\n\tm.snFileInfo = &subnetFileInfo{\n\t\tpath:   path,\n\t\tipMask: ipMasq,\n\t\tsn:     sn,\n\t\tIPv6sn: ipv6sn,\n\t\tmtu:    mtu,\n\t}\n\treturn subnet.WriteSubnetFile(path, config, ipMasq, sn, ipv6sn, mtu)\n}\n\n// GetStoredMacAddresses reads MAC addresses from node annotations when flannel restarts\nfunc (ksm *kubeSubnetManager) GetStoredMacAddresses(ctx context.Context) (string, string) {\n\tvar macv4, macv6 string\n\t// get mac info from Name func.\n\tnode, err := ksm.client.CoreV1().Nodes().Get(ctx, ksm.nodeName, metav1.GetOptions{})\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to get node for backend data: %v\", err)\n\t\treturn \"\", \"\"\n\t}\n\n\t// node backend data format: `{\"VNI\":1,\"VtepMAC\":\"12:c6:65:89:b4:e3\"}`\n\t// and we will return only mac addr str like 12:c6:65:89:b4:e3\n\tif node != nil && node.Annotations != nil {\n\t\tlog.Infof(\"List of node(%s) annotations: %#+v\", ksm.nodeName, node.Annotations)\n\t\tbackendData, ok := node.Annotations[fmt.Sprintf(\"%s/backend-data\", ksm.annotationPrefix)]\n\t\tif ok {\n\t\t\tmacStr := strings.Trim(backendData, \"\\\"}\")\n\t\t\tmacInfoSlice := strings.Split(macStr, \":\\\"\")\n\t\t\tif len(macInfoSlice) == 2 {\n\t\t\t\tmacv4 = macInfoSlice[1]\n\t\t\t}\n\t\t}\n\t\tbackendDatav6, okv6 := node.Annotations[fmt.Sprintf(\"%s/backend-v6-data\", ksm.annotationPrefix)]\n\t\tif okv6 {\n\t\t\tmacStr := strings.Trim(backendDatav6, \"\\\"}\")\n\t\t\tmacInfoSlice := strings.Split(macStr, \":\\\"\")\n\t\t\tif len(macInfoSlice) == 2 {\n\t\t\t\tmacv6 = macInfoSlice[1]\n\t\t\t}\n\t\t}\n\t\treturn macv4, macv6\n\t}\n\n\treturn \"\", \"\"\n}\n\n// GetStoredPublicIP reads if there are any public IP configured as annotation when flannel starts\nfunc (ksm *kubeSubnetManager) GetStoredPublicIP(ctx context.Context) (string, string) {\n\t// get mac info from Name func.\n\tnode, err := ksm.client.CoreV1().Nodes().Get(ctx, ksm.nodeName, metav1.GetOptions{})\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to get node for backend data: %v\", err)\n\t\treturn \"\", \"\"\n\t}\n\n\tif node != nil && node.Annotations != nil {\n\t\tlog.Infof(\"List of node(%s) annotations: %#+v\", ksm.nodeName, node.Annotations)\n\t\tpublicIP := node.Annotations[ksm.annotations.BackendNodePublicIP]\n\t\tpublicIPv6 := node.Annotations[ksm.annotations.BackendNodePublicIPv6]\n\t\treturn publicIP, publicIPv6\n\t}\n\n\treturn \"\", \"\"\n}\n"
  },
  {
    "path": "pkg/subnet/kube/kube_test.go",
    "content": "// Copyright 2018 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage kube\n\nimport (\n\t\"net\"\n\n\t\"testing\"\n)\n\nfunc TestContainsCIDR(t *testing.T) {\n\ttestCases := []struct {\n\t\tcidr1          string\n\t\tcidr2          string\n\t\texpectedResult bool\n\t}{\n\t\t{\"10.244.0.0/16\", \"10.244.0.0/16\", true},\n\t\t{\"10.244.0.0/16\", \"10.244.0.0/24\", true},\n\t\t{\"10.244.0.0/16\", \"10.244.255.0/24\", true},\n\t\t{\"10.244.0.0/16\", \"10.244.0.0/15\", false},\n\t\t{\"10.244.0.0/16\", \"192.168.0.0/24\", false},\n\n\t\t{\"2001:0db8:1234::/48\", \"2001:0db8:1234::/48\", true},\n\t\t{\"2001:0db8:1234::/48\", \"2001:0db8:1234::/64\", true},\n\t\t{\"2001:0db8:1234::/48\", \"2001:0db8:1234:ffff::/64\", true},\n\t\t{\"2001:0db8:1234::/48\", \"2001:0db8:1234::/47\", false},\n\t\t{\"2001:0db8:1234::/48\", \"fe02::/32\", false},\n\t}\n\n\tfor i, tc := range testCases {\n\t\t_, ipnet1, _ := net.ParseCIDR(tc.cidr1)\n\t\t_, ipnet2, _ := net.ParseCIDR(tc.cidr2)\n\n\t\tactualResult := containsCIDR(ipnet1, ipnet2)\n\n\t\tif actualResult != tc.expectedResult {\n\t\t\tt.Errorf(\"#%d: Expected %t, but was %t.\", i, tc.expectedResult, actualResult)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/subnet/subnet.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage subnet\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"sync\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nvar (\n\tsubnetRegex = regexp.MustCompile(`(\\d+\\.\\d+.\\d+.\\d+)-(\\d+)(?:&([a-f\\d:]+)-(\\d+))?$`)\n)\n\nfunc ParseSubnetKey(s string) (*ip.IP4Net, *ip.IP6Net) {\n\tif parts := subnetRegex.FindStringSubmatch(s); len(parts) == 5 {\n\t\tsnIp := net.ParseIP(parts[1]).To4()\n\t\tprefixLen, err := strconv.ParseUint(parts[2], 10, 5)\n\n\t\tif snIp == nil || err != nil {\n\t\t\treturn nil, nil\n\t\t}\n\t\tsn4 := &ip.IP4Net{IP: ip.FromIP(snIp), PrefixLen: uint(prefixLen)}\n\n\t\tvar sn6 *ip.IP6Net\n\t\tif parts[3] != \"\" {\n\t\t\tsnIp6 := net.ParseIP(parts[3]).To16()\n\t\t\tprefixLen, err = strconv.ParseUint(parts[4], 10, 7)\n\t\t\tif snIp6 == nil || err != nil {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t\tsn6 = &ip.IP6Net{IP: ip.FromIP6(snIp6), PrefixLen: uint(prefixLen)}\n\t\t}\n\n\t\treturn sn4, sn6\n\t}\n\n\treturn nil, nil\n}\n\nfunc MakeSubnetKey(sn ip.IP4Net, sn6 ip.IP6Net) string {\n\tif sn6.Empty() {\n\t\treturn sn.StringSep(\".\", \"-\")\n\t} else {\n\t\treturn sn.StringSep(\".\", \"-\") + \"&\" + sn6.StringSep(\":\", \"-\")\n\t}\n}\n\nfunc WriteSubnetFile(path string, config *Config, ipMasq bool, sn ip.IP4Net, ipv6sn ip.IP6Net, mtu int) error {\n\tdir, name := filepath.Split(path)\n\terr := os.MkdirAll(dir, 0755)\n\tif err != nil {\n\t\treturn err\n\t}\n\ttempFile := filepath.Join(dir, \".\"+name)\n\tvar b bytes.Buffer\n\tif config.EnableIPv4 {\n\t\tfmt.Fprintf(&b, \"FLANNEL_NETWORK=%s\\n\", config.Network)\n\t\t// Write out the first usable IP by incrementing sn.IP by one\n\t\tsn.IncrementIP()\n\n\t\tfmt.Fprintf(&b, \"FLANNEL_SUBNET=%s\\n\", sn)\n\t}\n\tif config.EnableIPv6 {\n\t\tfmt.Fprintf(&b, \"FLANNEL_IPV6_NETWORK=%s\\n\", config.IPv6Network)\n\t\t// Write out the first usable IP by incrementing ip6Sn.IP by one\n\t\tipv6sn.IncrementIP()\n\t\tfmt.Fprintf(&b, \"FLANNEL_IPV6_SUBNET=%s\\n\", ipv6sn)\n\t}\n\n\tfmt.Fprintf(&b, \"FLANNEL_MTU=%d\\n\", mtu)\n\tfmt.Fprintf(&b, \"FLANNEL_IPMASQ=%v\\n\", ipMasq)\n\n\tif err := os.WriteFile(tempFile, b.Bytes(), 0644); err != nil {\n\t\treturn err\n\t}\n\n\t// rename(2) the temporary file to the desired location so that it becomes\n\t// atomically visible with the contents\n\treturn os.Rename(tempFile, path)\n\t// TODO - is this safe? What if it's not on the same FS?\n}\n\ntype Manager interface {\n\tGetNetworkConfig(ctx context.Context) (*Config, error)\n\tHandleSubnetFile(path string, config *Config, ipMasq bool, sn ip.IP4Net, ipv6sn ip.IP6Net, mtu int) error\n\tAcquireLease(ctx context.Context, attrs *lease.LeaseAttrs) (*lease.Lease, error)\n\tRenewLease(ctx context.Context, lease *lease.Lease) error\n\tWatchLease(ctx context.Context, sn ip.IP4Net, sn6 ip.IP6Net, receiver chan []lease.LeaseWatchResult) error\n\tWatchLeases(ctx context.Context, receiver chan []lease.LeaseWatchResult) error\n\tCompleteLease(ctx context.Context, lease *lease.Lease, wg *sync.WaitGroup) error\n\n\tGetStoredMacAddresses(ctx context.Context) (string, string)\n\tGetStoredPublicIP(ctx context.Context) (string, string)\n\tName() string\n}\n\n// WatchLeases performs a long term watch of the given network's subnet leases\n// and communicates addition/deletion events on receiver channel. It takes care\n// of handling \"fall-behind\" logic where the history window has advanced too far\n// and it needs to diff the latest snapshot with its saved state and generate events\nfunc WatchLeases(ctx context.Context, sm Manager, ownLease *lease.Lease, receiver chan []lease.Event) {\n\n\t// LeaseWatcher is initiated with the Lease of the local node\n\tlw := &lease.LeaseWatcher{\n\t\tOwnLease: ownLease,\n\t}\n\n\tleaseWatchChan := make(chan []lease.LeaseWatchResult)\n\tgo func() {\n\t\terr := sm.WatchLeases(ctx, leaseWatchChan)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"could not watch leases: %s\", err)\n\t\t\treturn\n\t\t}\n\t}()\n\tfor watchResults := range leaseWatchChan {\n\t\tfor _, wr := range watchResults {\n\t\t\tvar batch []lease.Event\n\n\t\t\tif len(wr.Events) > 0 {\n\t\t\t\tbatch = lw.Update(wr.Events)\n\t\t\t} else {\n\t\t\t\tbatch = lw.Reset(wr.Snapshot)\n\t\t\t}\n\n\t\t\tfor i := range batch {\n\t\t\t\tlog.Infof(\"Batch elem [%d] is { %#v }\", i, batch[i])\n\t\t\t}\n\t\t\tif len(batch) > 0 {\n\t\t\t\treceiver <- batch\n\t\t\t}\n\t\t}\n\t}\n\n\tclose(receiver)\n}\n\n// WatchLease performs a long term watch of the given network's subnet lease\n// and communicates addition/deletion events on receiver channel. It takes care\n// of handling \"fall-behind\" logic where the history window has advanced too far\n// and it needs to diff the latest snapshot with its saved state and generate events\nfunc WatchLease(ctx context.Context, sm Manager, sn ip.IP4Net, sn6 ip.IP6Net, receiver chan lease.Event) {\n\tleaseWatchChan := make(chan []lease.LeaseWatchResult)\n\tvar closeOnce sync.Once\n\n\t// Helper function to close the receiver channel safely\n\tcloseReceiver := func() {\n\t\tcloseOnce.Do(func() {\n\t\t\tclose(receiver)\n\t\t})\n\t}\n\n\tgo func() {\n\t\terr := sm.WatchLease(ctx, sn, sn6, leaseWatchChan)\n\t\tif err != nil {\n\t\t\tif err == context.Canceled || err == context.DeadlineExceeded {\n\t\t\t\tlog.Infof(\"%v, close receiver chan\", err)\n\t\t\t\tcloseReceiver()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlog.Errorf(\"Subnet watch failed: %v\", err)\n\t\t\tcloseReceiver()\n\t\t\treturn\n\t\t}\n\n\t}()\n\n\tfor watchResults := range leaseWatchChan {\n\t\tfor _, wr := range watchResults {\n\t\t\tif len(wr.Snapshot) > 0 {\n\t\t\t\treceiver <- lease.Event{\n\t\t\t\t\tType:  lease.EventAdded,\n\t\t\t\t\tLease: wr.Snapshot[0],\n\t\t\t\t}\n\t\t\t} else if len(wr.Events) > 0 {\n\t\t\t\treceiver <- wr.Events[0]\n\t\t\t} else {\n\t\t\t\tlog.V(2).Info(\"WatchLease: empty event received\")\n\t\t\t}\n\t\t}\n\n\t}\n\tlog.Info(\"leaseWatchChan channel closed\")\n\tcloseReceiver()\n}\n"
  },
  {
    "path": "pkg/subnet/subnet_test.go",
    "content": "// Copyright 2022 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage subnet\n\nimport (\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"testing\"\n)\n\nfunc TestSubnetNodev4(t *testing.T) {\n\tkey := \"10.12.13.0-24\"\n\tsn, sn6 := ParseSubnetKey(key)\n\n\tif sn == nil {\n\t\tt.Errorf(\"Failed to parse ipv4 address\")\n\t\treturn\n\t}\n\n\tif sn.ToIPNet() == nil {\n\t\tt.Errorf(\"Failed to transform sn into IPNet\")\n\t\treturn\n\t}\n\n\tif sn.ToIPNet().String() != \"10.12.13.0/24\" {\n\t\tt.Errorf(\"Unexpected ipv4 network\")\n\t}\n\n\tif sn6 != nil {\n\t\tt.Errorf(\"Not expecting ipv6 address\")\n\t}\n\n\tif MakeSubnetKey(*sn, ip.IP6Net{}) != key {\n\t\tt.Errorf(\"MakeSubnetKey doesn't match parsed key\")\n\t}\n}\n\nfunc TestSubnetNodev6(t *testing.T) {\n\tkey := \"10.12.13.0-24&fd00:12:13::-56\"\n\tsn, sn6 := ParseSubnetKey(key)\n\n\tif sn == nil {\n\t\tt.Errorf(\"Failed to parse ipv4 address\")\n\t\treturn\n\t}\n\n\tif sn.ToIPNet() == nil {\n\t\tt.Errorf(\"Failed to transform sn into IPNet\")\n\t\treturn\n\t}\n\n\tif sn.ToIPNet().String() != \"10.12.13.0/24\" {\n\t\tt.Errorf(\"Unexpected ipv4 network\")\n\t}\n\n\tif sn6 == nil {\n\t\tt.Errorf(\"Failed to parse ipv6 address\")\n\t\treturn\n\t}\n\n\tif sn6.ToIPNet().String() != \"fd00:12:13::/56\" {\n\t\tt.Errorf(\"Unexpected ipv6 network\")\n\t}\n\n\tif MakeSubnetKey(*sn, *sn6) != key {\n\t\tt.Errorf(\"MakeSubnetKey doesn't match parsed key\")\n\t}\n}\n\nfunc TestSubnetNodeInvalid(t *testing.T) {\n\tkeys := []string{\n\t\t\"10\",\n\t\t\"10.12.13.0\",\n\t\t\"10.12.13-24\",\n\t\t\"10.12.13.300-24\",\n\t\t\"10.12.13.0-24hi\",\n\t\t\"&2001::-56\",\n\t\t\"10.12.13.0-24&:12:13:-56\",\n\t\t\"10.12.13.0-24&20011::-56\",\n\t\t\"10.12.13.0-24&2001-56\",\n\t\t\"10.12.13.0-24&2001::\",\n\t\t\"10.12.13.0-24&2001::-56hi\",\n\t}\n\tfor _, key := range keys {\n\t\tsn, sn6 := ParseSubnetKey(key)\n\n\t\tif sn != nil || sn6 != nil {\n\t\t\tt.Errorf(\"Unexpectedly parsed %v - %v %v\", key, sn, sn6)\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "pkg/trafficmngr/iptables/iptables.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage iptables\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/coreos/go-iptables/iptables\"\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr\"\n\tlog \"k8s.io/klog/v2\"\n)\n\ntype IPTables interface {\n\tAppendUnique(table string, chain string, rulespec ...string) error\n\tChainExists(table, chain string) (bool, error)\n\tClearChain(table, chain string) error\n\tDelete(table string, chain string, rulespec ...string) error\n\tExists(table string, chain string, rulespec ...string) (bool, error)\n}\n\ntype IPTablesError interface {\n\tIsNotExist() bool\n\tError() string\n}\n\ntype IPTablesManager struct {\n\tipv4Rules []trafficmngr.IPTablesRule\n\tipv6Rules []trafficmngr.IPTablesRule\n}\n\nfunc (iptm *IPTablesManager) Init(ctx context.Context) error {\n\tlog.Info(\"Starting flannel in iptables mode...\")\n\n\tiptm.ipv4Rules = make([]trafficmngr.IPTablesRule, 0, 10)\n\tiptm.ipv6Rules = make([]trafficmngr.IPTablesRule, 0, 10)\n\n\treturn nil\n}\n\nfunc (iptm *IPTablesManager) CleanUp(ctx context.Context) error {\n\tlog.Info(\"Cleaning-up iptables rules...\")\n\t//IPv4\n\tipt, err := iptables.New()\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\treturn fmt.Errorf(\"failed to setup IPTables. iptables binary was not found: %v\", err)\n\t}\n\terr = ipt.ClearAndDeleteChain(\"nat\", \"FLANNEL-POSTRTG\")\n\tif err != nil {\n\t\tlog.V(2).Infof(\"could not clean-up FLANNEL-POSTRTG (IPv4): %v\", err)\n\t}\n\terr = ipt.ClearAndDeleteChain(\"nat\", \"FLANNEL-FWD\")\n\tif err != nil {\n\t\tlog.V(2).Infof(\"could not clean-up FLANNEL-FWD (IPv4): %v\", err)\n\t}\n\n\t//IPv6\n\tipt6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\treturn fmt.Errorf(\"failed to setup IPTables. ip6tables binary was not found: %v\", err)\n\t}\n\terr = ipt6.ClearAndDeleteChain(\"nat\", \"FLANNEL-POSTRTG\")\n\tif err != nil {\n\t\tlog.V(2).Infof(\"could not clean-up FLANNEL-POSTRTG (IPv6): %v\", err)\n\t}\n\terr = ipt6.ClearAndDeleteChain(\"nat\", \"FLANNEL-FWD\")\n\tif err != nil {\n\t\tlog.V(2).Infof(\"could not clean-up FLANNEL-FWD (IPv6): %v\", err)\n\t}\n\treturn nil\n}\n\nfunc (iptm *IPTablesManager) SetupAndEnsureMasqRules(ctx context.Context, flannelIPv4Net, prevSubnet, prevNetwork ip.IP4Net,\n\tflannelIPv6Net, prevIPv6Subnet, prevIPv6Network ip.IP6Net,\n\tcurrentlease *lease.Lease,\n\tresyncPeriod int,\n\tipMasqRandomFullyDisable bool) error {\n\n\tif !flannelIPv4Net.Empty() {\n\t\t// recycle iptables rules only when network configured or subnet leased is not equal to current one.\n\t\tif !flannelIPv4Net.Equal(prevNetwork) || !prevSubnet.Equal(currentlease.Subnet) {\n\t\t\tlog.Infof(\"Current network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old iptables rules\",\n\t\t\t\tflannelIPv4Net, currentlease.Subnet, prevNetwork, prevSubnet)\n\t\t\tnewLease := &lease.Lease{\n\t\t\t\tSubnet: prevSubnet,\n\t\t\t}\n\t\t\tif err := iptm.deleteIP4Tables(iptm.masqRules(prevNetwork, newLease, ipMasqRandomFullyDisable)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tlog.Infof(\"Setting up masking rules\")\n\t\tiptm.CreateIP4Chain(\"nat\", \"FLANNEL-POSTRTG\")\n\t\tgo iptm.setupAndEnsureIP4Tables(ctx, iptm.masqRules(flannelIPv4Net, currentlease, ipMasqRandomFullyDisable), resyncPeriod)\n\t}\n\tif !flannelIPv6Net.Empty() {\n\t\t// recycle iptables rules only when network configured or subnet leased is not equal to current one.\n\t\tif !flannelIPv6Net.Equal(prevIPv6Network) || !prevIPv6Subnet.Equal(currentlease.IPv6Subnet) {\n\t\t\tlog.Infof(\"Current network or subnet (%v, %v) is not equal to previous one (%v, %v), trying to recycle old iptables rules\",\n\t\t\t\tflannelIPv6Net, currentlease.IPv6Subnet, prevIPv6Network, prevIPv6Subnet)\n\t\t\tnewLease := &lease.Lease{\n\t\t\t\tIPv6Subnet: prevIPv6Subnet,\n\t\t\t}\n\t\t\tif err := iptm.deleteIP6Tables(iptm.masqIP6Rules(prevIPv6Network, newLease, ipMasqRandomFullyDisable)); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tlog.Infof(\"Setting up masking rules for IPv6\")\n\t\tiptm.CreateIP6Chain(\"nat\", \"FLANNEL-POSTRTG\")\n\t\tgo iptm.setupAndEnsureIP6Tables(ctx, iptm.masqIP6Rules(flannelIPv6Net, currentlease, ipMasqRandomFullyDisable), resyncPeriod)\n\t}\n\treturn nil\n}\n\nfunc (iptm *IPTablesManager) masqRules(ccidr ip.IP4Net, lease *lease.Lease, ipMasqRandomFullyDisable bool) []trafficmngr.IPTablesRule {\n\tcluster_cidr := ccidr.String()\n\n\tpod_cidr := lease.Subnet.String()\n\tipt, err := iptables.New()\n\tsupports_random_fully := false\n\tif err == nil {\n\t\tsupports_random_fully = ipt.HasRandomFully()\n\t}\n\trules := make([]trafficmngr.IPTablesRule, 2)\n\t// This rule ensure that the flannel iptables rules are executed before other rules on the node\n\trules[0] = trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"FLANNEL-POSTRTG\"}}\n\t// This rule will not masquerade traffic marked by the kube-proxy to avoid double NAT bug on some kernel version\n\trules[1] = trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-m\", \"mark\", \"--mark\", trafficmngr.KubeProxyMark, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}}\n\t// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0), for any of the cluster_cidrs\n\trules = append(rules,\n\t\ttrafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", pod_cidr, \"-d\", cluster_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}},\n\t\ttrafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", cluster_cidr, \"-d\", pod_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}},\n\t)\n\t// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address\n\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"!\", \"-s\", cluster_cidr, \"-d\", pod_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}})\n\t// NAT if it's not multicast traffic\n\tif supports_random_fully && !ipMasqRandomFullyDisable {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", cluster_cidr, \"!\", \"-d\", \"224.0.0.0/4\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"}})\n\t} else {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", cluster_cidr, \"!\", \"-d\", \"224.0.0.0/4\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\"}})\n\t}\n\t// Masquerade anything headed towards flannel from the host\n\tif supports_random_fully && !ipMasqRandomFullyDisable {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"!\", \"-s\", cluster_cidr, \"-d\", cluster_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"}})\n\t} else {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"!\", \"-s\", cluster_cidr, \"-d\", cluster_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\"}})\n\t}\n\treturn rules\n}\n\nfunc (iptm *IPTablesManager) masqIP6Rules(ccidr ip.IP6Net, lease *lease.Lease, ipMasqRandomFullyDisable bool) []trafficmngr.IPTablesRule {\n\tcluster_cidr := ccidr.String()\n\tpod_cidr := lease.IPv6Subnet.String()\n\tipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)\n\tsupports_random_fully := false\n\tif err == nil {\n\t\tsupports_random_fully = ipt.HasRandomFully()\n\t}\n\trules := make([]trafficmngr.IPTablesRule, 2)\n\n\t// This rule ensure that the flannel iptables rules are executed before other rules on the node\n\trules[0] = trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"FLANNEL-POSTRTG\"}}\n\t// This rule will not masquerade traffic marked by the kube-proxy to avoid double NAT bug on some kernel version\n\trules[1] = trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-m\", \"mark\", \"--mark\", trafficmngr.KubeProxyMark, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}}\n\n\t// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0), for any of the cluster_cidrs\n\trules = append(rules,\n\t\ttrafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", pod_cidr, \"-d\", cluster_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}},\n\t\ttrafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", cluster_cidr, \"-d\", pod_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}},\n\t)\n\t// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address\n\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"!\", \"-s\", cluster_cidr, \"-d\", pod_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}})\n\t// NAT if it's not multicast traffic\n\tif supports_random_fully && !ipMasqRandomFullyDisable {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", cluster_cidr, \"!\", \"-d\", \"ff00::/8\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"}})\n\t} else {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"-s\", cluster_cidr, \"!\", \"-d\", \"ff00::/8\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\"}})\n\t}\n\n\t// Masquerade anything headed towards flannel from the host\n\tif supports_random_fully && !ipMasqRandomFullyDisable {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"!\", \"-s\", cluster_cidr, \"-d\", cluster_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"}})\n\t} else {\n\t\trules = append(rules, trafficmngr.IPTablesRule{Table: \"nat\", Action: \"-A\", Chain: \"FLANNEL-POSTRTG\", Rulespec: []string{\"!\", \"-s\", cluster_cidr, \"-d\", cluster_cidr, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\"}})\n\t}\n\n\treturn rules\n}\n\nfunc (iptm *IPTablesManager) SetupAndEnsureForwardRules(ctx context.Context, flannelIPv4Network ip.IP4Net, flannelIPv6Network ip.IP6Net, resyncPeriod int) {\n\tif !flannelIPv4Network.Empty() {\n\t\tlog.Infof(\"Changing default FORWARD chain policy to ACCEPT\")\n\t\tiptm.CreateIP4Chain(\"filter\", \"FLANNEL-FWD\")\n\t\tgo iptm.setupAndEnsureIP4Tables(ctx, iptm.forwardRules(flannelIPv4Network.String()), resyncPeriod)\n\t}\n\tif !flannelIPv6Network.Empty() {\n\t\tlog.Infof(\"IPv6: Changing default FORWARD chain policy to ACCEPT\")\n\t\tiptm.CreateIP6Chain(\"filter\", \"FLANNEL-FWD\")\n\t\tgo iptm.setupAndEnsureIP6Tables(ctx, iptm.forwardRules(flannelIPv6Network.String()), resyncPeriod)\n\t}\n}\n\nfunc (iptm *IPTablesManager) forwardRules(flannelNetwork string) []trafficmngr.IPTablesRule {\n\treturn []trafficmngr.IPTablesRule{\n\t\t// This rule ensure that the flannel iptables rules are executed before other rules on the node\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"FORWARD\", Rulespec: []string{\"-m\", \"comment\", \"--comment\", \"flanneld forward\", \"-j\", \"FLANNEL-FWD\"}},\n\t\t// These rules allow traffic to be forwarded if it is to or from the flannel network range.\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"FLANNEL-FWD\", Rulespec: []string{\"-s\", flannelNetwork, \"-m\", \"comment\", \"--comment\", \"flanneld forward\", \"-j\", \"ACCEPT\"}},\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"FLANNEL-FWD\", Rulespec: []string{\"-d\", flannelNetwork, \"-m\", \"comment\", \"--comment\", \"flanneld forward\", \"-j\", \"ACCEPT\"}},\n\t}\n}\n\nfunc (iptm *IPTablesManager) CreateIP4Chain(table, chain string) {\n\tipt, err := iptables.New()\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IPTables. iptables binary was not found: %v\", err)\n\t\treturn\n\t}\n\terr = ipt.ClearChain(table, chain)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IPTables. Error on creating the chain: %v\", err)\n\t\treturn\n\t}\n}\n\nfunc (iptm *IPTablesManager) CreateIP6Chain(table, chain string) {\n\tipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IP6Tables. iptables binary was not found: %v\", err)\n\t\treturn\n\t}\n\terr = ipt.ClearChain(table, chain)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IP6Tables. Error on creating the chain: %v\", err)\n\t\treturn\n\t}\n}\n\nfunc ipTablesRulesExist(ipt IPTables, rules []trafficmngr.IPTablesRule) (bool, error) {\n\tfor _, rule := range rules {\n\t\tif rule.Chain == \"FLANNEL-FWD\" || rule.Rulespec[len(rule.Rulespec)-1] == \"FLANNEL-FWD\" {\n\t\t\tchainExist, err := ipt.ChainExists(rule.Table, \"FLANNEL-FWD\")\n\t\t\tif err != nil {\n\t\t\t\treturn false, fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t\t}\n\t\t\tif !chainExist {\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t} else if rule.Chain == \"FLANNEL-POSTRTG\" || rule.Rulespec[len(rule.Rulespec)-1] == \"FLANNEL-POSTRTG\" {\n\t\t\tchainExist, err := ipt.ChainExists(rule.Table, \"FLANNEL-POSTRTG\")\n\t\t\tif err != nil {\n\t\t\t\treturn false, fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t\t}\n\t\t\tif !chainExist {\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t}\n\t\texists, err := ipt.Exists(rule.Table, rule.Chain, rule.Rulespec...)\n\t\tif err != nil {\n\t\t\t// this shouldn't ever happen\n\t\t\treturn false, fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t}\n\t\tif !exists {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\n// ipTablesCleanAndBuild create from a list of iptables rules a transaction (as string) for iptables-restore for ordering the rules that effectively running\nfunc ipTablesCleanAndBuild(ipt IPTables, rules []trafficmngr.IPTablesRule) (IPTablesRestoreRules, error) {\n\ttablesRules := IPTablesRestoreRules{}\n\n\t// Build append and delete rules\n\tfor _, rule := range rules {\n\t\tif rule.Chain == \"FLANNEL-FWD\" || rule.Rulespec[len(rule.Rulespec)-1] == \"FLANNEL-FWD\" {\n\t\t\tchainExist, err := ipt.ChainExists(rule.Table, \"FLANNEL-FWD\")\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t\t}\n\t\t\tif !chainExist {\n\t\t\t\terr = ipt.ClearChain(rule.Table, \"FLANNEL-FWD\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"failed to create rule chain: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t} else if rule.Chain == \"FLANNEL-POSTRTG\" || rule.Rulespec[len(rule.Rulespec)-1] == \"FLANNEL-POSTRTG\" {\n\t\t\tchainExist, err := ipt.ChainExists(rule.Table, \"FLANNEL-POSTRTG\")\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t\t}\n\t\t\tif !chainExist {\n\t\t\t\terr = ipt.ClearChain(rule.Table, \"FLANNEL-POSTRTG\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"failed to create rule chain: %v\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\texists, err := ipt.Exists(rule.Table, rule.Chain, rule.Rulespec...)\n\t\tif err != nil {\n\t\t\t// this shouldn't ever happen\n\t\t\treturn nil, fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t}\n\t\tif exists {\n\t\t\tif _, ok := tablesRules[rule.Table]; !ok {\n\t\t\t\ttablesRules[rule.Table] = []IPTablesRestoreRuleSpec{}\n\t\t\t}\n\t\t\t// if the rule exists it's safer to delete it and then create them\n\t\t\ttablesRules[rule.Table] = append(tablesRules[rule.Table], append(IPTablesRestoreRuleSpec{\"-D\", rule.Chain}, rule.Rulespec...))\n\t\t}\n\t\t// with iptables-restore we can ensure that all rules created are in good order and have no external rule between them\n\t\ttablesRules[rule.Table] = append(tablesRules[rule.Table], append(IPTablesRestoreRuleSpec{rule.Action, rule.Chain}, rule.Rulespec...))\n\t}\n\n\treturn tablesRules, nil\n}\n\n// ipTablesBootstrap init iptables rules using iptables-restore (with some cleaning if some rules already exists)\nfunc ipTablesBootstrap(ipt IPTables, iptRestore IPTablesRestore, rules []trafficmngr.IPTablesRule) error {\n\ttablesRules, err := ipTablesCleanAndBuild(ipt, rules)\n\tif err != nil {\n\t\t// if we can't find iptables or if we can check existing rules, give up and return\n\t\treturn fmt.Errorf(\"failed to setup iptables-restore payload: %v\", err)\n\t}\n\n\tlog.V(6).Infof(\"trying to run iptables-restore < %+v\", tablesRules)\n\n\terr = iptRestore.ApplyWithoutFlush(tablesRules)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to apply partial iptables-restore %v\", err)\n\t}\n\n\tlog.Infof(\"bootstrap done\")\n\n\treturn nil\n}\n\nfunc (iptm *IPTablesManager) setupAndEnsureIP4Tables(ctx context.Context, rules []trafficmngr.IPTablesRule, resyncPeriod int) {\n\tipt, err := iptables.New()\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IPTables. iptables binary was not found: %v\", err)\n\t\treturn\n\t}\n\tiptRestore, err := NewIPTablesRestoreWithProtocol(iptables.ProtocolIPv4)\n\tif err != nil {\n\t\t// if we can't find iptables-restore, give up and return\n\t\tlog.Errorf(\"Failed to setup IPTables. iptables-restore binary was not found: %v\", err)\n\t\treturn\n\t}\n\n\terr = ipTablesBootstrap(ipt, iptRestore, rules)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to bootstrap IPTables: %v\", err)\n\t}\n\n\tiptm.ipv4Rules = append(iptm.ipv4Rules, rules...)\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\t//clean-up is setup in Init\n\t\t\treturn\n\t\tcase <-time.After(time.Duration(resyncPeriod) * time.Second):\n\t\t\t// Ensure that all the iptables rules exist every 5 seconds\n\t\t\tif err := ensureIPTables(ipt, iptRestore, rules); err != nil {\n\t\t\t\tlog.Errorf(\"Failed to ensure iptables rules: %v\", err)\n\t\t\t}\n\t\t}\n\n\t}\n}\n\nfunc (iptm *IPTablesManager) setupAndEnsureIP6Tables(ctx context.Context, rules []trafficmngr.IPTablesRule, resyncPeriod int) {\n\tipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IP6Tables. iptables binary was not found: %v\", err)\n\t\treturn\n\t}\n\tiptRestore, err := NewIPTablesRestoreWithProtocol(iptables.ProtocolIPv6)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup iptables-restore: %v\", err)\n\t\treturn\n\t}\n\n\terr = ipTablesBootstrap(ipt, iptRestore, rules)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to bootstrap IPTables: %v\", err)\n\t}\n\tiptm.ipv6Rules = append(iptm.ipv6Rules, rules...)\n\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\t//clean-up is setup in Init\n\t\t\treturn\n\t\tcase <-time.After(time.Duration(resyncPeriod) * time.Second):\n\t\t\t// Ensure that all the iptables rules exist every 5 seconds\n\t\t\tif err := ensureIPTables(ipt, iptRestore, rules); err != nil {\n\t\t\t\tlog.Errorf(\"Failed to ensure iptables rules: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// deleteIP4Tables delete specified iptables rules\nfunc (iptm *IPTablesManager) deleteIP4Tables(rules []trafficmngr.IPTablesRule) error {\n\tipt, err := iptables.New()\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IPTables. iptables binary was not found: %v\", err)\n\t\treturn err\n\t}\n\tiptRestore, err := NewIPTablesRestoreWithProtocol(iptables.ProtocolIPv4)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup iptables-restore: %v\", err)\n\t\treturn err\n\t}\n\terr = teardownIPTables(ipt, iptRestore, rules)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to teardown iptables: %v\", err)\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// deleteIP6Tables delete specified iptables rules\nfunc (iptm *IPTablesManager) deleteIP6Tables(rules []trafficmngr.IPTablesRule) error {\n\tipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup IP6Tables. iptables binary was not found: %v\", err)\n\t\treturn err\n\t}\n\n\tiptRestore, err := NewIPTablesRestoreWithProtocol(iptables.ProtocolIPv6)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\tlog.Errorf(\"Failed to setup iptables-restore: %v\", err)\n\t\treturn err\n\t}\n\terr = teardownIPTables(ipt, iptRestore, rules)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to teardown iptables: %v\", err)\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc ensureIPTables(ipt IPTables, iptRestore IPTablesRestore, rules []trafficmngr.IPTablesRule) error {\n\texists, err := ipTablesRulesExist(ipt, rules)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error checking rule existence: %v\", err)\n\t}\n\tif exists {\n\t\t// if all the rules already exist, no need to do anything\n\t\treturn nil\n\t}\n\t// Otherwise, teardown all the rules and set them up again\n\t// We do this because the order of the rules is important\n\tlog.Info(\"Some iptables rules are missing; deleting and recreating rules\")\n\terr = ipTablesBootstrap(ipt, iptRestore, rules)\n\tif err != nil {\n\t\t// if we can't find iptables, give up and return\n\t\treturn fmt.Errorf(\"error setting up rules: %v\", err)\n\t}\n\treturn nil\n}\n\nfunc teardownIPTables(ipt IPTables, iptr IPTablesRestore, rules []trafficmngr.IPTablesRule) error {\n\ttablesRules := IPTablesRestoreRules{}\n\n\t// Build delete rules to a transaction for iptables restore\n\tfor _, rule := range rules {\n\t\tif rule.Chain == \"FLANNEL-FWD\" || rule.Rulespec[len(rule.Rulespec)-1] == \"FLANNEL-FWD\" {\n\t\t\tchainExists, err := ipt.ChainExists(rule.Table, \"FLANNEL-FWD\")\n\t\t\tif err != nil {\n\t\t\t\t// this shouldn't ever happen\n\t\t\t\treturn fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t\t}\n\t\t\tif !chainExists {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else if rule.Chain == \"FLANNEL-POSTRTG\" || rule.Rulespec[len(rule.Rulespec)-1] == \"FLANNEL-POSTRTG\" {\n\t\t\tchainExists, err := ipt.ChainExists(rule.Table, \"FLANNEL-POSTRTG\")\n\t\t\tif err != nil {\n\t\t\t\t// this shouldn't ever happen\n\t\t\t\treturn fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t\t}\n\t\t\tif !chainExists {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\texists, err := ipt.Exists(rule.Table, rule.Chain, rule.Rulespec...)\n\t\tif err != nil {\n\t\t\t// this shouldn't ever happen\n\t\t\treturn fmt.Errorf(\"failed to check rule existence: %v\", err)\n\t\t}\n\n\t\tif exists {\n\t\t\tif _, ok := tablesRules[rule.Table]; !ok {\n\t\t\t\ttablesRules[rule.Table] = []IPTablesRestoreRuleSpec{}\n\t\t\t}\n\t\t\ttablesRules[rule.Table] = append(tablesRules[rule.Table], append(IPTablesRestoreRuleSpec{\"-D\", rule.Chain}, rule.Rulespec...))\n\t\t}\n\t}\n\terr := iptr.ApplyWithoutFlush(tablesRules) // ApplyWithoutFlush make a diff, Apply make a replace (desired state)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unable to teardown iptables: %v\", err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/trafficmngr/iptables/iptables_restore.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage iptables\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"os/exec\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"sync\"\n\n\t\"github.com/coreos/go-iptables/iptables\"\n\tlog \"k8s.io/klog/v2\"\n)\n\nconst (\n\tipTablesRestoreCmd  string = \"iptables-restore\"\n\tip6TablesRestoreCmd string = \"ip6tables-restore\"\n\tipTablesCmd         string = \"iptables\"\n\tip6TablesCmd        string = \"ip6tables\"\n)\n\n// IPTablesRestore wrapper for iptables-restore\ntype IPTablesRestore interface {\n\t// ApplyWithoutFlush apply without flush chains\n\tApplyWithoutFlush(rules IPTablesRestoreRules) error\n}\n\n// ipTablesRestore internal type\ntype ipTablesRestore struct {\n\tpath    string\n\tproto   iptables.Protocol\n\thasWait bool\n\t// ipTablesRestore needs a mutex to ensure that two avoid\n\t// collisions between two goroutines calling ApplyWithoutFlush in parallel.\n\t// This could result in the second call accidentally restoring a rule removed by the first\n\tmu sync.Mutex\n}\n\n// IPTablesRestoreRules represents iptables-restore table block\ntype IPTablesRestoreRules map[string][]IPTablesRestoreRuleSpec\n\n// IPTablesRestoreRuleSpec represents one rule spec delimited by space\ntype IPTablesRestoreRuleSpec []string\n\n// NewIPTablesRestoreWithProtocol build new IPTablesRestore for supplied proto\nfunc NewIPTablesRestoreWithProtocol(protocol iptables.Protocol) (IPTablesRestore, error) {\n\tcmd := getIptablesRestoreCommand(protocol)\n\tpath, err := exec.LookPath(cmd)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcmdIptables := getIptablesCommand(protocol)\n\tpathIptables, err := exec.LookPath(cmdIptables)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\thasWait, err := getIptablesRestoreSupport(pathIptables)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tipt := ipTablesRestore{\n\t\tpath:    path,\n\t\tproto:   protocol,\n\t\thasWait: hasWait,\n\t\tmu:      sync.Mutex{},\n\t}\n\treturn &ipt, nil\n}\n\n// ApplyWithoutFlush apply without flush chains\nfunc (iptr *ipTablesRestore) ApplyWithoutFlush(rules IPTablesRestoreRules) error {\n\tiptr.mu.Lock()\n\tdefer iptr.mu.Unlock()\n\tpayload := buildIPTablesRestorePayload(rules)\n\n\tlog.V(6).Infof(\"trying to run with payload %s\", payload)\n\tstdout, stderr, err := iptr.runWithOutput([]string{\"--noflush\"}, bytes.NewBuffer([]byte(payload)))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unable to run iptables-restore (%s, %s): %v\", stdout, stderr, err)\n\t}\n\treturn nil\n}\n\n// runWithOutput runs an iptables command with the given arguments,\n// writing any stdout output to the given writer\nfunc (iptr *ipTablesRestore) runWithOutput(args []string, stdin io.Reader) (string, string, error) {\n\tvar stdout bytes.Buffer\n\tvar stderr bytes.Buffer\n\tif iptr.hasWait {\n\t\targs = append(args, \"--wait\")\n\t}\n\n\tcmd := exec.Command(iptr.path, args...)\n\tcmd.Stdout = &stdout\n\tcmd.Stderr = &stderr\n\tcmd.Stdin = stdin\n\n\tif err := cmd.Run(); err != nil {\n\t\treturn stdout.String(), stderr.String(), err\n\t}\n\n\treturn stdout.String(), stderr.String(), nil\n}\n\n// buildIPTablesRestorePayload build table/COMMIT payload\nfunc buildIPTablesRestorePayload(tableRules IPTablesRestoreRules) string {\n\tiptablesRestorePayload := \"\"\n\tfor table, rules := range tableRules {\n\t\tiptablesRestorePayload += \"*\" + table + \"\\n\"\n\n\t\tfor _, lineRule := range rules {\n\t\t\t// as iptables-restore use stdin if \"--comment\" then protect \"the comment\"\n\t\t\tsize := len(lineRule)\n\t\t\tfor i, rule := range lineRule {\n\t\t\t\tif i > 0 && lineRule[i-1] == \"--comment\" {\n\t\t\t\t\tiptablesRestorePayload += \"\\\"\" + rule + \"\\\"\"\n\t\t\t\t} else {\n\t\t\t\t\tiptablesRestorePayload += rule\n\t\t\t\t}\n\n\t\t\t\tif i < size-1 {\n\t\t\t\t\tiptablesRestorePayload += \" \"\n\t\t\t\t} else {\n\t\t\t\t\tiptablesRestorePayload += \"\\n\"\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tiptablesRestorePayload += \"COMMIT\\n\"\n\t}\n\treturn iptablesRestorePayload\n}\n\n// getIptablesRestoreSupport get current iptables-restore support\nfunc getIptablesRestoreSupport(path string) (hasWait bool, err error) {\n\tversion, err := getIptablesRestoreVersionString(path)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tv1, v2, v3, err := extractIptablesRestoreVersion(version)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\treturn ipTablesHasWaitSupport(v1, v2, v3), nil\n}\n\n// Checks if an iptables-restore version is after 1.6.2, when --wait was added\nfunc ipTablesHasWaitSupport(v1, v2, v3 int) bool {\n\tif v1 > 1 {\n\t\treturn true\n\t}\n\tif v1 == 1 && v2 > 6 {\n\t\treturn true\n\t}\n\tif v1 == 1 && v2 == 6 && v3 >= 2 {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// extractIptablesRestoreVersion returns the first three components of the iptables-restore version\n// e.g. \"iptables-restore v1.3.66\" would return (1, 3, 66, nil)\nfunc extractIptablesRestoreVersion(str string) (int, int, int, error) {\n\tversionMatcher := regexp.MustCompile(`v([0-9]+)\\.([0-9]+)\\.([0-9]+)`)\n\tresult := versionMatcher.FindStringSubmatch(str)\n\tif result == nil {\n\t\treturn 0, 0, 0, fmt.Errorf(\"no iptables-restore version found in string: %s\", str)\n\t}\n\n\tv1, err := strconv.Atoi(result[1])\n\tif err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\n\tv2, err := strconv.Atoi(result[2])\n\tif err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\n\tv3, err := strconv.Atoi(result[3])\n\tif err != nil {\n\t\treturn 0, 0, 0, err\n\t}\n\treturn v1, v2, v3, nil\n}\n\n// getIptablesRestoreVersionString run iptables-restore --version\nfunc getIptablesRestoreVersionString(path string) (string, error) {\n\tcmd := exec.Command(path, \"--version\")\n\tvar out bytes.Buffer\n\tcmd.Stdout = &out\n\terr := cmd.Run()\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"unable to find iptables-restore version: %v\", err)\n\t}\n\treturn out.String(), nil\n}\n\n// getIptablesRestoreCommand returns the correct command for the given proto, either \"iptables-restore\" or \"ip6tables-restore\".\nfunc getIptablesRestoreCommand(proto iptables.Protocol) string {\n\tif proto == iptables.ProtocolIPv6 {\n\t\treturn ip6TablesRestoreCmd\n\t}\n\treturn ipTablesRestoreCmd\n}\n\n// getIptablesCommand returns the correct command for the given proto, either \"iptables\" or \"ip6tables\".\nfunc getIptablesCommand(proto iptables.Protocol) string {\n\tif proto == iptables.ProtocolIPv6 {\n\t\treturn ip6TablesCmd\n\t}\n\treturn ipTablesCmd\n}\n"
  },
  {
    "path": "pkg/trafficmngr/iptables/iptables_restore_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage iptables\n\nimport \"testing\"\n\nfunc TestRules(t *testing.T) {\n\tbaseRules := IPTablesRestoreRules{\n\t\t\"filter\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n\texpectedFilterPayload := `*filter\n-A INPUT -s 127.0.0.1 -d 127.0.0.1 -j RETURN\n-A INPUT -s 127.0.0.1 ! -d 224.0.0.0/4 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\nCOMMIT\n`\n\texpectedNATPayload := `*nat\n-A INPUT -s 127.0.0.1 -d 127.0.0.1 -j RETURN\n-A INPUT -s 127.0.0.1 ! -d 224.0.0.0/4 -m comment --comment \"flanneld masq\" -j MASQUERADE --random-fully\nCOMMIT\n`\n\tpayload := buildIPTablesRestorePayload(baseRules)\n\tif payload != expectedFilterPayload+expectedNATPayload && payload != expectedNATPayload+expectedFilterPayload {\n\t\tt.Errorf(\"iptables-restore payload not as expected. Expected: %#v, Actual: %#v\", expectedFilterPayload+expectedNATPayload, payload)\n\t}\n}\n"
  },
  {
    "path": "pkg/trafficmngr/iptables/iptables_test.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage iptables\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr\"\n)\n\nfunc testingLease() *lease.Lease {\n\t_, ipv6Net, _ := net.ParseCIDR(\"fc00::/48\")\n\t_, net, _ := net.ParseCIDR(\"192.168.0.0/16\")\n\treturn &lease.Lease{\n\t\tSubnet:     ip.FromIPNet(net),\n\t\tIPv6Subnet: ip.FromIP6Net(ipv6Net),\n\t}\n}\n\ntype MockIPTables struct {\n\trules    []trafficmngr.IPTablesRule\n\tt        *testing.T\n\tfailures map[string]*MockIPTablesError\n}\n\ntype MockIPTablesRestore struct {\n\tt     *testing.T\n\trules []IPTablesRestoreRules\n}\n\ntype MockIPTablesError struct {\n\tnotExist bool\n}\n\nfunc (mock *MockIPTablesError) IsNotExist() bool {\n\treturn mock.notExist\n}\n\nfunc (mock *MockIPTablesError) Error() string {\n\treturn fmt.Sprintf(\"IsNotExist: %v\", !mock.notExist)\n}\n\nfunc (mock *MockIPTablesRestore) ApplyFully(rules IPTablesRestoreRules) error {\n\tmock.rules = []IPTablesRestoreRules{rules}\n\treturn nil\n}\n\nfunc (mock *MockIPTablesRestore) ApplyWithoutFlush(rules IPTablesRestoreRules) error {\n\tmock.rules = append(mock.rules, rules)\n\treturn nil\n}\n\nfunc (mock *MockIPTables) ruleIndex(table string, chain string, rulespec []string) int {\n\tfor i, rule := range mock.rules {\n\t\tif rule.Table == table && rule.Chain == chain && reflect.DeepEqual(rule.Rulespec, rulespec) {\n\t\t\treturn i\n\t\t}\n\t}\n\treturn -1\n}\n\nfunc (mock *MockIPTables) ChainExists(table, chain string) (bool, error) {\n\treturn true, nil\n}\n\nfunc (mock *MockIPTables) ClearChain(table, chain string) error {\n\treturn nil\n}\n\nfunc (mock *MockIPTables) Delete(table string, chain string, rulespec ...string) error {\n\tvar ruleIndex = mock.ruleIndex(table, chain, rulespec)\n\tkey := table + chain + strings.Join(rulespec, \"\")\n\treason := mock.failures[key]\n\tif reason != nil {\n\t\treturn reason\n\t}\n\n\tif ruleIndex != -1 {\n\t\tmock.rules = append(mock.rules[:ruleIndex], mock.rules[ruleIndex+1:]...)\n\t}\n\treturn nil\n}\n\nfunc (mock *MockIPTables) Exists(table string, chain string, rulespec ...string) (bool, error) {\n\tvar ruleIndex = mock.ruleIndex(table, chain, rulespec)\n\tif ruleIndex != -1 {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc (mock *MockIPTables) AppendUnique(table string, chain string, rulespec ...string) error {\n\tvar ruleIndex = mock.ruleIndex(table, chain, rulespec)\n\tif ruleIndex == -1 {\n\t\tmock.rules = append(mock.rules, trafficmngr.IPTablesRule{Table: table, Chain: chain, Rulespec: rulespec})\n\t}\n\treturn nil\n}\n\nfunc TestDeleteRules(t *testing.T) {\n\tipt := &MockIPTables{t: t}\n\tiptr := &MockIPTablesRestore{t: t}\n\tiptm := IPTablesManager{}\n\tbaseRules := iptm.masqRules(\n\t\tip.IP4Net{\n\t\t\tIP:        ip.MustParseIP4(\"10.0.1.0\"),\n\t\t\tPrefixLen: 16,\n\t\t}, testingLease(), false)\n\texpectedRules := expectedTearDownIPTablesRestoreRules(baseRules)\n\n\terr := ipTablesBootstrap(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error bootstrapping up iptables\")\n\t}\n\terr = setupIPTables(ipt, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\tif len(ipt.rules) != 7 {\n\t\tt.Errorf(\"Should be 7 masqRules, there are actually %d: %#v\", len(ipt.rules), ipt.rules)\n\t}\n\n\tiptr.rules = []IPTablesRestoreRules{}\n\terr = teardownIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error tearing down iptables\")\n\t}\n\tif !reflect.DeepEqual(expectedRules, iptr.rules) {\n\t\tt.Errorf(\"Incorrect restores rules, Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n\n}\n\nfunc TestDeleteMoreRules(t *testing.T) {\n\tipt := &MockIPTables{}\n\tiptr := &MockIPTablesRestore{}\n\n\tbaseRules := []trafficmngr.IPTablesRule{\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"INPUT\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"INPUT\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t}\n\n\texpectedRules := IPTablesRestoreRules{\n\t\t\"filter\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-D\", \"INPUT\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-D\", \"INPUT\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-D\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-D\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n\n\terr := ipTablesBootstrap(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error bootstrapping up iptables\")\n\t}\n\terr = setupIPTables(ipt, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\tif len(ipt.rules) != 4 {\n\t\tt.Errorf(\"Should be 4 masqRules, there are actually %d: %#v\", len(ipt.rules), ipt.rules)\n\t}\n\n\tiptr.rules = []IPTablesRestoreRules{}\n\terr = teardownIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error tearing down iptables\")\n\t}\n\tif !reflect.DeepEqual(iptr.rules, []IPTablesRestoreRules{expectedRules}) {\n\t\tt.Errorf(\"Incorrect restores rules, Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n}\n\nfunc TestBootstrapRules(t *testing.T) {\n\tiptr := &MockIPTablesRestore{}\n\tipt := &MockIPTables{}\n\n\tbaseRules := []trafficmngr.IPTablesRule{\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"INPUT\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"filter\", Action: \"-A\", Chain: \"INPUT\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t}\n\n\terr := ipTablesBootstrap(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error bootstrapping up iptables\")\n\t}\n\t// Ensure iptable mock has rules too\n\terr = setupIPTables(ipt, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\n\texpectedRules := IPTablesRestoreRules{\n\t\t\"filter\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n\n\tif !reflect.DeepEqual(iptr.rules, []IPTablesRestoreRules{expectedRules}) {\n\t\tt.Errorf(\"iptables masqRules after ensureIPTables are incorrected. Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n\n\tiptr.rules = []IPTablesRestoreRules{}\n\n\texpectedRules = IPTablesRestoreRules{\n\t\t\"filter\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-D\", \"INPUT\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-D\", \"INPUT\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t\t{\"-A\", \"INPUT\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-D\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-D\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n\t// Re-run ensure has new operations\n\terr = ipTablesBootstrap(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error bootstrapping up iptables\")\n\t}\n\tif !reflect.DeepEqual(iptr.rules, []IPTablesRestoreRules{expectedRules}) {\n\t\tt.Errorf(\"iptables masqRules after ensureIPTables are incorrected. Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n}\n\nfunc TestDeleteIP6Rules(t *testing.T) {\n\tipt := &MockIPTables{}\n\tiptr := &MockIPTablesRestore{}\n\n\tbaseRules := IP6Rules(ip.IP6Net{}, testingLease())\n\n\t// expect to have the same DELETE rules\n\texpectedRules := IP6RestoreDeleteRules(ip.IP6Net{}, testingLease())\n\n\terr := ipTablesBootstrap(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error bootstrapping up iptables\")\n\t}\n\terr = setupIPTables(ipt, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\tif len(ipt.rules) != 4 {\n\t\tt.Errorf(\"Should be 4 masqRules, there are actually %d: %#v\", len(ipt.rules), ipt.rules)\n\t}\n\tiptr.rules = []IPTablesRestoreRules{}\n\terr = teardownIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error tearing down iptables\")\n\t}\n\tif !reflect.DeepEqual(iptr.rules, []IPTablesRestoreRules{expectedRules}) {\n\t\tt.Errorf(\"Should be 4 deleted iptables rules, there are actually. Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n}\n\nfunc TestEnsureRules(t *testing.T) {\n\tiptr := &MockIPTablesRestore{}\n\tipt := &MockIPTables{}\n\n\t// Ensure iptable mock has other rules\n\totherRules := []trafficmngr.IPTablesRule{\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-A\", \"POSTROUTING\", \"-j\", \"KUBE-POSTROUTING\"}},\n\t}\n\terr := setupIPTables(ipt, otherRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\n\tbaseRules := []trafficmngr.IPTablesRule{\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t}\n\n\terr = ensureIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Errorf(\"ensureIPTables should have completed without errors\")\n\t}\n\t// Ensure iptable mock has rules too\n\terr = setupIPTables(ipt, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\n\texpectedRules := IPTablesRestoreRules{\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"-d\", \"127.0.0.1\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", \"127.0.0.1\", \"!\", \"-d\", \"224.0.0.0/4\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n\n\tif !reflect.DeepEqual(iptr.rules, []IPTablesRestoreRules{expectedRules}) {\n\t\tt.Errorf(\"iptables masqRules after ensureIPTables are incorrected. Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n\n\tiptr.rules = []IPTablesRestoreRules{}\n\t// Re-run ensure no new operations\n\terr = ensureIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Errorf(\"ensureIPTables should have completed without errors\")\n\t}\n\tif len(iptr.rules) > 0 {\n\t\tt.Errorf(\"iptables masqRules after ensureIPTables are incorrected. Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n}\n\nfunc TestEnsureIP6Rules(t *testing.T) {\n\tiptr := &MockIPTablesRestore{}\n\tipt := &MockIPTables{}\n\n\t// Ensure iptable mock has other rules\n\totherRules := []trafficmngr.IPTablesRule{\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-A\", \"POSTROUTING\", \"-j\", \"KUBE-POSTROUTING\"}},\n\t}\n\terr := setupIPTables(ipt, otherRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\n\tbaseRules := IP6Rules(ip.IP6Net{}, testingLease())\n\n\terr = ensureIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Errorf(\"ensureIPTables should have completed without errors\")\n\t}\n\t// Ensure iptable mock has rules too\n\terr = setupIPTables(ipt, baseRules)\n\tif err != nil {\n\t\tt.Error(\"Error setting up iptables\")\n\t}\n\n\texpectedRules := IP6RestoreRules(ip.IP6Net{}, testingLease())\n\n\tif !reflect.DeepEqual(iptr.rules, []IPTablesRestoreRules{expectedRules}) {\n\t\tt.Errorf(\"iptables rules after ensureIPTables are incorrected. Expected: %#v, Actual: %#v\", expectedRules, iptr.rules)\n\t}\n\n\tiptr.rules = []IPTablesRestoreRules{}\n\t// Re-run ensure no new operations\n\terr = ensureIPTables(ipt, iptr, baseRules)\n\tif err != nil {\n\t\tt.Errorf(\"ensureIPTables should have completed without errors\")\n\t}\n\tif len(iptr.rules) > 0 {\n\t\tt.Errorf(\"rules masqRules after ensureIPTables are incorrected. Expected empty, Actual: %#v\", iptr.rules)\n\t}\n\n}\n\nfunc setupIPTables(ipt IPTables, rules []trafficmngr.IPTablesRule) error {\n\tfor _, rule := range rules {\n\t\terr := ipt.AppendUnique(rule.Table, rule.Chain, rule.Rulespec...)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to insert IPTables rule: %v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc expectedTearDownIPTablesRestoreRules(rules []trafficmngr.IPTablesRule) []IPTablesRestoreRules {\n\ttablesRules := IPTablesRestoreRules{}\n\tfor _, rule := range rules {\n\t\tif _, ok := tablesRules[rule.Table]; !ok {\n\t\t\ttablesRules[rule.Table] = []IPTablesRestoreRuleSpec{}\n\t\t}\n\t\ttablesRules[rule.Table] = append(tablesRules[rule.Table], append(IPTablesRestoreRuleSpec{\"-D\", rule.Chain}, rule.Rulespec...))\n\t}\n\n\treturn []IPTablesRestoreRules{tablesRules}\n}\n\nfunc IP6Rules(ipn ip.IP6Net, lease *lease.Lease) []trafficmngr.IPTablesRule {\n\tn := ipn.String()\n\tsn := lease.IPv6Subnet.String()\n\n\treturn []trafficmngr.IPTablesRule{\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", n, \"-d\", n, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"-s\", n, \"!\", \"-d\", \"ff00::/8\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"!\", \"-s\", n, \"-d\", sn, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"}},\n\t\t{Table: \"nat\", Action: \"-A\", Chain: \"POSTROUTING\", Rulespec: []string{\"!\", \"-s\", n, \"-d\", n, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"}},\n\t}\n}\n\nfunc IP6RestoreRules(ipn ip.IP6Net, lease *lease.Lease) IPTablesRestoreRules {\n\tn := ipn.String()\n\tsn := lease.IPv6Subnet.String()\n\treturn IPTablesRestoreRules{\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", n, \"-d\", n, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"-s\", n, \"!\", \"-d\", \"ff00::/8\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"!\", \"-s\", n, \"-d\", sn, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"},\n\t\t\t{\"-A\", \"POSTROUTING\", \"!\", \"-s\", n, \"-d\", n, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n}\n\nfunc IP6RestoreDeleteRules(ipn ip.IP6Net, lease *lease.Lease) IPTablesRestoreRules {\n\tn := ipn.String()\n\tsn := lease.IPv6Subnet.String()\n\treturn IPTablesRestoreRules{\n\t\t\"nat\": []IPTablesRestoreRuleSpec{\n\t\t\t{\"-D\", \"POSTROUTING\", \"-s\", n, \"-d\", n, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"},\n\t\t\t{\"-D\", \"POSTROUTING\", \"-s\", n, \"!\", \"-d\", \"ff00::/8\", \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t\t{\"-D\", \"POSTROUTING\", \"!\", \"-s\", n, \"-d\", sn, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"RETURN\"},\n\t\t\t{\"-D\", \"POSTROUTING\", \"!\", \"-s\", n, \"-d\", n, \"-m\", \"comment\", \"--comment\", \"flanneld masq\", \"-j\", \"MASQUERADE\", \"--random-fully\"},\n\t\t},\n\t}\n}\n"
  },
  {
    "path": "pkg/trafficmngr/iptables/iptables_windows.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage iptables\n\nimport (\n\t\"context\"\n\n\tlog \"k8s.io/klog/v2\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr\"\n)\n\ntype IPTablesManager struct{}\n\ntype IPTables interface {\n\tAppendUnique(table string, chain string, rulespec ...string) error\n\tChainExists(table, chain string) (bool, error)\n\tClearChain(table, chain string) error\n\tDelete(table string, chain string, rulespec ...string) error\n\tExists(table string, chain string, rulespec ...string) (bool, error)\n}\n\nfunc (iptm IPTablesManager) Init(ctx context.Context) error {\n\tlog.Info(\"Starting flannel in windows mode...\")\n\treturn nil\n}\n\nfunc (iptm *IPTablesManager) CleanUp(ctx context.Context) error {\n\treturn nil\n}\n\nfunc (iptm *IPTablesManager) SetupAndEnsureForwardRules(ctx context.Context, flannelIPv4Network ip.IP4Net, flannelIPv6Network ip.IP6Net, resyncPeriod int) {\n}\n\nfunc (iptm *IPTablesManager) SetupAndEnsureMasqRules(ctx context.Context, flannelIPv4Net, prevSubnet, prevNetwork ip.IP4Net,\n\tflannelIPv6Net, prevIPv6Subnet, prevIPv6Network ip.IP6Net,\n\tcurrentlease *lease.Lease,\n\tresyncPeriod int,\n\tipMasqRandomFullyDisable bool) error {\n\tlog.Warning(trafficmngr.ErrUnimplemented)\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/trafficmngr/nftables/nftables.go",
    "content": "// Copyright 2024 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage nftables\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\tlog \"k8s.io/klog/v2\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"sigs.k8s.io/knftables\"\n)\n\nconst (\n\tipv4Table    = \"flannel-ipv4\"\n\tipv6Table    = \"flannel-ipv6\"\n\tforwardChain = \"forward\"\n\tpostrtgChain = \"postrtg\"\n)\n\ntype NFTablesManager struct {\n\tnftv4 knftables.Interface\n\tnftv6 knftables.Interface\n}\n\nfunc (nftm *NFTablesManager) Init(ctx context.Context) error {\n\tlog.Info(\"Starting flannel in nftables mode...\")\n\tvar err error\n\tnftm.nftv4, err = initTable(ctx, knftables.IPv4Family, ipv4Table)\n\tif err != nil {\n\t\treturn err\n\t}\n\tnftm.nftv6, err = initTable(ctx, knftables.IPv6Family, ipv6Table)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// create a new table and returns the interface to interact with it\nfunc initTable(ctx context.Context, ipFamily knftables.Family, name string) (knftables.Interface, error) {\n\tnft, err := knftables.New(ipFamily, name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"no nftables support: %v\", err)\n\t}\n\ttx := nft.NewTransaction()\n\n\ttx.Add(&knftables.Table{\n\t\tComment: knftables.PtrTo(\"rules for \" + name),\n\t})\n\terr = nft.Run(ctx, tx)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"nftables: couldn't initialise table %s: %v\", name, err)\n\t}\n\treturn nft, nil\n}\n\n// It is needed when using nftables? accept seems to be the default\n// warning: never add a default 'drop' policy on the forwardChain as it breaks connectivity to the node\nfunc (nftm *NFTablesManager) SetupAndEnsureForwardRules(ctx context.Context,\n\tflannelIPv4Network ip.IP4Net, flannelIPv6Network ip.IP6Net, resyncPeriod int) {\n\tif !flannelIPv4Network.Empty() {\n\t\tlog.Infof(\"Changing default FORWARD chain policy to ACCEPT\")\n\t\ttx := nftm.nftv4.NewTransaction()\n\n\t\ttx.Add(&knftables.Chain{\n\t\t\tName:     forwardChain,\n\t\t\tComment:  knftables.PtrTo(\"chain to accept flannel traffic\"),\n\t\t\tType:     knftables.PtrTo(knftables.FilterType),\n\t\t\tHook:     knftables.PtrTo(knftables.ForwardHook),\n\t\t\tPriority: knftables.PtrTo(knftables.FilterPriority),\n\t\t})\n\t\ttx.Flush(&knftables.Chain{\n\t\t\tName: forwardChain,\n\t\t})\n\n\t\ttx.Add(&knftables.Rule{\n\t\t\tChain: forwardChain,\n\t\t\tRule: knftables.Concat(\n\t\t\t\t\"ip saddr\", flannelIPv4Network.String(),\n\t\t\t\t\"accept\",\n\t\t\t),\n\t\t})\n\t\ttx.Add(&knftables.Rule{\n\t\t\tChain: forwardChain,\n\t\t\tRule: knftables.Concat(\n\t\t\t\t\"ip daddr\", flannelIPv4Network.String(),\n\t\t\t\t\"accept\",\n\t\t\t),\n\t\t})\n\t\terr := nftm.nftv4.Run(ctx, tx)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"nftables: couldn't setup forward rules: %v\", err)\n\t\t}\n\t}\n\tif !flannelIPv6Network.Empty() {\n\t\tlog.Infof(\"Changing default FORWARD chain policy to ACCEPT (ipv6)\")\n\t\ttx := nftm.nftv6.NewTransaction()\n\n\t\ttx.Add(&knftables.Chain{\n\t\t\tName:     forwardChain,\n\t\t\tComment:  knftables.PtrTo(\"chain to accept flannel traffic\"),\n\t\t\tType:     knftables.PtrTo(knftables.FilterType),\n\t\t\tHook:     knftables.PtrTo(knftables.ForwardHook),\n\t\t\tPriority: knftables.PtrTo(knftables.FilterPriority),\n\t\t})\n\t\ttx.Flush(&knftables.Chain{\n\t\t\tName: forwardChain,\n\t\t})\n\n\t\ttx.Add(&knftables.Rule{\n\t\t\tChain: forwardChain,\n\t\t\tRule: knftables.Concat(\n\t\t\t\t\"ip6 saddr\", flannelIPv6Network.String(),\n\t\t\t\t\"accept\",\n\t\t\t),\n\t\t})\n\t\ttx.Add(&knftables.Rule{\n\t\t\tChain: forwardChain,\n\t\t\tRule: knftables.Concat(\n\t\t\t\t\"ip6 daddr\", flannelIPv6Network.String(),\n\t\t\t\t\"accept\",\n\t\t\t),\n\t\t})\n\t\terr := nftm.nftv6.Run(ctx, tx)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"nftables: couldn't setup forward rules (ipv6): %v\", err)\n\t\t}\n\t}\n}\n\nfunc (nftm *NFTablesManager) SetupAndEnsureMasqRules(ctx context.Context, flannelIPv4Net, prevSubnet, prevNetwork ip.IP4Net,\n\tflannelIPv6Net, prevIPv6Subnet, prevIPv6Network ip.IP6Net,\n\tcurrentlease *lease.Lease,\n\tresyncPeriod int,\n\tipMasqRandomFullyDisable bool) error {\n\tif !flannelIPv4Net.Empty() {\n\t\tlog.Infof(\"nftables: setting up masking rules (ipv4)\")\n\t\ttx := nftm.nftv4.NewTransaction()\n\n\t\ttx.Add(&knftables.Chain{\n\t\t\tName:     postrtgChain,\n\t\t\tComment:  knftables.PtrTo(\"chain to manage traffic masquerading by flannel\"),\n\t\t\tType:     knftables.PtrTo(knftables.NATType),\n\t\t\tHook:     knftables.PtrTo(knftables.PostroutingHook),\n\t\t\tPriority: knftables.PtrTo(knftables.SNATPriority),\n\t\t})\n\t\t// make sure that the chain is empty before adding our rules\n\t\t// => no need for the check and recycle part of iptables.go\n\t\ttx.Flush(&knftables.Chain{\n\t\t\tName: postrtgChain,\n\t\t})\n\t\terr := nftm.addMasqRules(ctx, tx, flannelIPv4Net.String(), currentlease.Subnet.String(), knftables.IPv4Family, ipMasqRandomFullyDisable)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"nftables: couldn't setup masq rules: %v\", err)\n\t\t}\n\t\terr = nftm.nftv4.Run(ctx, tx)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"nftables: couldn't setup masq rules: %v\", err)\n\t\t}\n\t}\n\tif !flannelIPv6Net.Empty() {\n\t\tlog.Infof(\"nftables: setting up masking rules (ipv6)\")\n\t\ttx := nftm.nftv6.NewTransaction()\n\n\t\ttx.Add(&knftables.Chain{\n\t\t\tName:     postrtgChain,\n\t\t\tComment:  knftables.PtrTo(\"chain to manage traffic masquerading by flannel\"),\n\t\t\tType:     knftables.PtrTo(knftables.NATType),\n\t\t\tHook:     knftables.PtrTo(knftables.PostroutingHook),\n\t\t\tPriority: knftables.PtrTo(knftables.SNATPriority),\n\t\t})\n\t\t// make sure that the chain is empty before adding our rules\n\t\t// => no need for the check and recycle part of iptables.go\n\t\ttx.Flush(&knftables.Chain{\n\t\t\tName: postrtgChain,\n\t\t})\n\t\terr := nftm.addMasqRules(ctx, tx, flannelIPv6Net.String(), currentlease.IPv6Subnet.String(), knftables.IPv6Family, ipMasqRandomFullyDisable)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"nftables: couldn't setup masq rules: %v\", err)\n\t\t}\n\t\terr = nftm.nftv6.Run(ctx, tx)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"nftables: couldn't setup masq rules: %v\", err)\n\t\t}\n\t}\n\treturn nil\n}\n\n// add required masking rules to transaction tx\nfunc (nftm *NFTablesManager) addMasqRules(ctx context.Context,\n\ttx *knftables.Transaction,\n\tclusterCidr, podCidr string,\n\tfamily knftables.Family,\n\tipMasqRandomFullyDisable bool) error {\n\tmasquerade := \"masquerade fully-random\"\n\tif !nftm.checkRandomfully(ctx) || ipMasqRandomFullyDisable {\n\t\tmasquerade = \"masquerade\"\n\t}\n\n\tmulticastCidr := \"224.0.0.0/4\"\n\tif family == knftables.IPv6Family {\n\t\tmulticastCidr = \"ff00::/8\"\n\t}\n\t// This rule will not masquerade traffic marked\n\t// by the kube-proxy to avoid double NAT bug on some kernel version\n\ttx.Add(&knftables.Rule{\n\t\tChain: postrtgChain,\n\t\tRule: knftables.Concat(\n\t\t\t\"meta mark\", \"0x4000\", //TODO_TF: check the correct value when deploying kube-proxy\n\t\t\t\"return\",\n\t\t),\n\t})\n\t// don't NAT traffic within overlay network\n\ttx.Add(&knftables.Rule{\n\t\tChain: postrtgChain,\n\t\tRule: knftables.Concat(\n\t\t\tfamily, \"saddr\", podCidr,\n\t\t\tfamily, \"daddr\", clusterCidr,\n\t\t\t\"return\",\n\t\t),\n\t})\n\ttx.Add(&knftables.Rule{\n\t\tChain: postrtgChain,\n\t\tRule: knftables.Concat(\n\t\t\tfamily, \"saddr\", clusterCidr,\n\t\t\tfamily, \"daddr\", podCidr,\n\t\t\t\"return\",\n\t\t),\n\t})\n\t// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address\n\ttx.Add(&knftables.Rule{\n\t\tChain: postrtgChain,\n\t\tRule: knftables.Concat(\n\t\t\tfamily, \"saddr\", \"!=\", podCidr,\n\t\t\tfamily, \"daddr\", clusterCidr,\n\t\t\t\"return\",\n\t\t),\n\t})\n\t// NAT if it's not multicast traffic\n\ttx.Add(&knftables.Rule{\n\t\tChain: postrtgChain,\n\t\tRule: knftables.Concat(\n\t\t\tfamily, \"saddr\", clusterCidr,\n\t\t\tfamily, \"daddr\", \"!=\", multicastCidr,\n\t\t\tmasquerade,\n\t\t),\n\t})\n\t// Masquerade anything headed towards flannel from the host\n\ttx.Add(&knftables.Rule{\n\t\tChain: postrtgChain,\n\t\tRule: knftables.Concat(\n\t\t\tfamily, \"saddr\", \"!=\", clusterCidr,\n\t\t\tfamily, \"daddr\", clusterCidr,\n\t\t\tmasquerade,\n\t\t),\n\t})\n\treturn nil\n}\n\n// clean-up all nftables states created by flannel by deleting all related tables\nfunc (nftm *NFTablesManager) CleanUp(ctx context.Context) error {\n\tlog.Info(\"Cleaning-up nftables rules...\")\n\tnft, err := knftables.New(knftables.IPv4Family, ipv4Table)\n\tif err == nil {\n\t\ttx := nft.NewTransaction()\n\t\ttx.Delete(&knftables.Table{})\n\t\terr = nft.Run(ctx, tx)\n\t}\n\tif err != nil {\n\t\tlog.V(2).Infof(\"nftables: couldn't delete table: %v\", err)\n\t}\n\n\tnft, err = knftables.New(knftables.IPv6Family, ipv6Table)\n\tif err == nil {\n\t\ttx := nft.NewTransaction()\n\t\ttx.Delete(&knftables.Table{})\n\t\terr = nft.Run(ctx, tx)\n\t}\n\tif err != nil {\n\t\tlog.V(2).Infof(\"nftables (ipv6): couldn't delete table: %v\", err)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/trafficmngr/nftables/nftables_windows.go",
    "content": "// Copyright 2024 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nftables\n\nimport (\n\t\"context\"\n\n\tlog \"k8s.io/klog/v2\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n\t\"github.com/flannel-io/flannel/pkg/trafficmngr\"\n)\n\ntype NFTablesManager struct {\n}\n\nfunc (nftm *NFTablesManager) Init(ctx context.Context) error {\n\tlog.Info(\"Starting flannel in windows mode...\")\n\treturn nil\n}\n\nfunc (nftm *NFTablesManager) CleanUp(ctx context.Context) error {\n\treturn nil\n}\n\nfunc (nftm *NFTablesManager) SetupAndEnsureForwardRules(ctx context.Context,\n\tflannelIPv4Network ip.IP4Net, flannelIPv6Network ip.IP6Net, resyncPeriod int) {\n}\n\nfunc (nftm *NFTablesManager) SetupAndEnsureMasqRules(ctx context.Context, flannelIPv4Net, prevSubnet, prevNetwork ip.IP4Net,\n\tflannelIPv6Net, prevIPv6Subnet, prevIPv6Network ip.IP6Net,\n\tcurrentlease *lease.Lease,\n\tresyncPeriod int,\n\tipMasqRandomFullyDisable bool) error {\n\tlog.Warning(trafficmngr.ErrUnimplemented)\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/trafficmngr/nftables/utils.go",
    "content": "// Copyright 2024 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//go:build !windows\n// +build !windows\n\npackage nftables\n\nimport (\n\t\"context\"\n\n\tlog \"k8s.io/klog/v2\"\n\t\"sigs.k8s.io/knftables\"\n)\n\nconst (\n\tmasqueradeTestTable = \"masqueradeTest\"\n)\n\n// check whether masquerade fully-random is supported by the kernel\nfunc (nftm *NFTablesManager) checkRandomfully(ctx context.Context) bool {\n\tresult := true\n\ttx := nftm.nftv4.NewTransaction()\n\ttx.Add(&knftables.Chain{\n\t\tName:     masqueradeTestTable,\n\t\tComment:  knftables.PtrTo(\"chain to test if masquerade random fully is supported\"),\n\t\tType:     knftables.PtrTo(knftables.NATType),\n\t\tHook:     knftables.PtrTo(knftables.PostroutingHook),\n\t\tPriority: knftables.PtrTo(knftables.SNATPriority),\n\t})\n\ttx.Flush(&knftables.Chain{\n\t\tName: masqueradeTestTable,\n\t})\n\t// Masquerade anything headed towards flannel from the host\n\ttx.Add(&knftables.Rule{\n\t\tChain: masqueradeTestTable,\n\t\tRule: knftables.Concat(\n\t\t\t\"ip saddr\", \"!=\", \"127.0.0.1\",\n\t\t\t\"masquerade fully-random\",\n\t\t),\n\t})\n\terr := nftm.nftv4.Check(ctx, tx)\n\tif err != nil {\n\t\tlog.Warningf(\"nftables: random fully unsupported\")\n\t\tresult = false\n\t}\n\treturn result\n}\n"
  },
  {
    "path": "pkg/trafficmngr/trafficmngr.go",
    "content": "// Copyright 2024 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage trafficmngr\n\nimport (\n\t\"context\"\n\t\"errors\"\n\n\t\"github.com/flannel-io/flannel/pkg/ip\"\n\t\"github.com/flannel-io/flannel/pkg/lease\"\n)\n\ntype IPTablesRule struct {\n\tTable    string\n\tAction   string\n\tChain    string\n\tRulespec []string\n}\n\nvar (\n\tErrUnimplemented = errors.New(\"unimplemented\")\n)\n\nconst KubeProxyMark string = \"0x4000/0x4000\"\n\ntype TrafficManager interface {\n\t// Initialize the TrafficManager\n\tInit(ctx context.Context) error\n\t// Clean-up existing tables and rules\n\tCleanUp(ctx context.Context) error\n\t// Install kernel rules to forward the traffic to and from the flannel network range.\n\t// This is done for IPv4 and/or IPv6 based on whether flannelIPv4Network and flannelIPv6Network are set.\n\t// SetupAndEnsureForwardRules starts a go routine that\n\t// rewrites these rules every resyncPeriod seconds if needed\n\tSetupAndEnsureForwardRules(ctx context.Context, flannelIPv4Network ip.IP4Net, flannelIPv6Network ip.IP6Net, resyncPeriod int)\n\t// Install kernel rules to setup NATing of packets sent to the flannel interface\n\t// This is done for IPv4 and/or IPv6 based on whether flannelIPv4Network and flannelIPv6Network are set.\n\t// prevSubnet,prevNetworks, prevIPv6Subnet, prevIPv6Networks are used\n\t// to determine whether the existing rules need to be replaced.\n\t// SetupAndEnsureMasqRules starts a go routine that\n\t// rewrites these rules every resyncPeriod seconds if needed\n\tSetupAndEnsureMasqRules(ctx context.Context,\n\t\tflannelIPv4Net, prevSubnet, prevNetwork ip.IP4Net,\n\t\tflannelIPv6Net, prevIPv6Subnet, prevIPv6Network ip.IP6Net,\n\t\tcurrentlease *lease.Lease,\n\t\tresyncPeriod int,\n\t\tipMasqRandomFullyDisable bool) error\n}\n"
  },
  {
    "path": "pkg/version/version.go",
    "content": "// Copyright 2015 flannel authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage version\n\nvar Version = \"dev\"\n"
  }
]