Repository: vmware/purser Branch: master Commit: fe4654999969 Files: 249 Total size: 600.1 KB Directory structure: gitextract_khhzg63f/ ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ ├── custom.md │ │ └── feature_request.md │ ├── ISSUE_TEMPLATE.md │ └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .make/ │ ├── Makefile.deploy.controller │ └── Makefile.deploy.purser ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile.in ├── Gopkg.toml ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── build/ │ ├── build.sh │ ├── purser-binary-install.sh │ ├── purser-binary-uninstall.sh │ ├── purser-minimal-setup.sh │ └── purser-setup.sh ├── cluster/ │ ├── artifacts/ │ │ ├── example-group.yaml │ │ ├── example-subscriber.yaml │ │ ├── group-template.json │ │ ├── purser-group-crd.yaml │ │ └── purser-subscriber-crd.yaml │ ├── helm/ │ │ └── chart/ │ │ └── purser/ │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── README.md │ │ ├── templates/ │ │ │ ├── NOTES.txt │ │ │ ├── _helpers.tpl │ │ │ ├── purser-controller-deployment.yaml │ │ │ ├── purser-controller-rbac.yaml │ │ │ ├── purser-controller-serviceaccount.yaml │ │ │ ├── purser-controller-svc.yaml │ │ │ ├── purser-database-statefulset.yaml │ │ │ ├── purser-database-svc.yaml │ │ │ ├── purser-ui-configmap.yaml │ │ │ ├── purser-ui-deployment.yaml │ │ │ ├── purser-ui-ingress.yaml │ │ │ └── purser-ui-svc.yaml │ │ └── values.yaml │ ├── minimal/ │ │ ├── purser-controller-setup.yaml │ │ ├── purser-database-setup.yaml │ │ └── purser-ui-setup.yaml │ ├── purser-controller-setup.yaml │ ├── purser-database-setup.yaml │ └── purser-ui-setup.yaml ├── cmd/ │ ├── controller/ │ │ ├── api/ │ │ │ ├── api.go │ │ │ ├── apiHandlers/ │ │ │ │ ├── authenticationHandlers.go │ │ │ │ ├── customGroupHandlers.go │ │ │ │ ├── helpers.go │ │ │ │ └── hierarchyAndMetricAPIHandlers.go │ │ │ ├── logger.go │ │ │ ├── router.go │ │ │ └── routes.go │ │ ├── config/ │ │ │ └── config.go │ │ └── purserctrl.go │ └── plugin/ │ ├── purser.go │ └── types.go ├── docs/ │ ├── architecture.md │ ├── custom-group-installation-and-usage.md │ ├── design/ │ │ └── pricing.md │ ├── developers-guide.md │ ├── manual-installation.md │ ├── plugin-installation.md │ ├── plugin-usage.md │ ├── purser-deployment.md │ └── sourcecode-installation.md ├── openapi.yaml ├── pkg/ │ ├── apis/ │ │ ├── groups/ │ │ │ └── v1/ │ │ │ ├── deepcopy.go │ │ │ ├── docs.go │ │ │ ├── register.go │ │ │ └── types.go │ │ └── subscriber/ │ │ └── v1/ │ │ ├── deepcopy.go │ │ ├── docs.go │ │ ├── register.go │ │ └── types.go │ ├── client/ │ │ ├── clientset/ │ │ │ └── typed/ │ │ │ ├── groups/ │ │ │ │ └── v1/ │ │ │ │ ├── group.go │ │ │ │ └── group_client.go │ │ │ └── subscriber/ │ │ │ └── v1/ │ │ │ ├── subsciber_client.go │ │ │ └── subscriber.go │ │ └── clientset.go │ ├── controller/ │ │ ├── buffering/ │ │ │ └── ring_buffer.go │ │ ├── controller.go │ │ ├── controller_test.go │ │ ├── dgraph/ │ │ │ ├── dgraph.go │ │ │ ├── login.go │ │ │ ├── models/ │ │ │ │ ├── constants.go │ │ │ │ ├── container.go │ │ │ │ ├── daemonset.go │ │ │ │ ├── deployment.go │ │ │ │ ├── group.go │ │ │ │ ├── job.go │ │ │ │ ├── label.go │ │ │ │ ├── namespace.go │ │ │ │ ├── node.go │ │ │ │ ├── pod.go │ │ │ │ ├── pod_test.go │ │ │ │ ├── process.go │ │ │ │ ├── pv.go │ │ │ │ ├── pvc.go │ │ │ │ ├── query/ │ │ │ │ │ ├── cluster.go │ │ │ │ │ ├── cluster_test.go │ │ │ │ │ ├── constants_test.go │ │ │ │ │ ├── group.go │ │ │ │ │ ├── group_test.go │ │ │ │ │ ├── helpers.go │ │ │ │ │ ├── helpers_test.go │ │ │ │ │ ├── label.go │ │ │ │ │ ├── label_test.go │ │ │ │ │ ├── login.go │ │ │ │ │ ├── pod.go │ │ │ │ │ ├── pod_test.go │ │ │ │ │ ├── queries.go │ │ │ │ │ ├── resource.go │ │ │ │ │ ├── resource_test.go │ │ │ │ │ ├── subscriber.go │ │ │ │ │ ├── subscriber_test.go │ │ │ │ │ └── types.go │ │ │ │ ├── rateCard.go │ │ │ │ ├── replicaset.go │ │ │ │ ├── service.go │ │ │ │ ├── statefulset.go │ │ │ │ └── subscriber.go │ │ │ └── purge.go │ │ ├── discovery/ │ │ │ ├── executer/ │ │ │ │ └── exec.go │ │ │ ├── generator/ │ │ │ │ └── graph.go │ │ │ ├── linker/ │ │ │ │ ├── podlinks.go │ │ │ │ ├── processlinks.go │ │ │ │ └── servicelinks.go │ │ │ └── processor/ │ │ │ ├── container.go │ │ │ ├── pod.go │ │ │ └── svc.go │ │ ├── eventprocessor/ │ │ │ ├── notifier.go │ │ │ ├── processor.go │ │ │ ├── sync.go │ │ │ └── updater.go │ │ ├── metrics/ │ │ │ └── metrics.go │ │ ├── payload.go │ │ ├── persistentVolume.go │ │ ├── types.go │ │ └── utils/ │ │ ├── jsonutils.go │ │ ├── k8sUtils.go │ │ ├── purge.go │ │ ├── purge_test.go │ │ ├── timeUtils.go │ │ ├── unitConversions.go │ │ └── unitConversions_test.go │ ├── plugin/ │ │ ├── costing.go │ │ ├── grouping.go │ │ ├── metrics/ │ │ │ └── metrics.go │ │ ├── node.go │ │ ├── pod.go │ │ ├── pricing.go │ │ ├── utils.go │ │ └── volume.go │ ├── pricing/ │ │ ├── aws/ │ │ │ ├── aws.go │ │ │ └── convert.go │ │ └── cloud.go │ └── utils/ │ ├── fileutils.go │ ├── k8sutil.go │ └── logutil.go ├── plugin.yaml ├── test/ │ ├── controller/ │ │ └── buffering/ │ │ └── ring_buffer_test.go │ ├── pricing/ │ │ └── pricing_aws_test.go │ └── utils/ │ └── checkUtil.go └── ui/ ├── Dockerfile.deploy.purser ├── README.md ├── angular.json ├── e2e/ │ ├── protractor.conf.js │ ├── src/ │ │ ├── app.e2e-spec.ts │ │ └── app.po.ts │ └── tsconfig.e2e.json ├── nginx.conf ├── package.json ├── proxy.conf.json ├── src/ │ ├── app/ │ │ ├── app.component.html │ │ ├── app.component.scss │ │ ├── app.component.spec.ts │ │ ├── app.component.ts │ │ ├── app.constants.ts │ │ ├── app.module.ts │ │ ├── app.routing.ts │ │ ├── common/ │ │ │ └── messages/ │ │ │ ├── common.messages.ts │ │ │ └── left-navigation.messages.ts │ │ └── modules/ │ │ ├── capacity-graph/ │ │ │ ├── capacity-graph.module.ts │ │ │ ├── components/ │ │ │ │ ├── capactiy-graph.component.html │ │ │ │ ├── capactiy-graph.component.scss │ │ │ │ ├── capactiy-graph.component.spec.ts │ │ │ │ └── capactiy-graph.component.ts │ │ │ └── services/ │ │ │ └── capacity-graph.service.ts │ │ ├── changepassword/ │ │ │ ├── changepassword.module.ts │ │ │ ├── components/ │ │ │ │ ├── changepassword.component.html │ │ │ │ ├── changepassword.component.scss │ │ │ │ ├── changepassword.component.spec.ts │ │ │ │ └── changepassword.component.ts │ │ │ └── services/ │ │ │ └── changepassword.service.ts │ │ ├── logical-group/ │ │ │ ├── components/ │ │ │ │ ├── logical-group.component.css │ │ │ │ ├── logical-group.component.html │ │ │ │ ├── logical-group.component.spec.ts │ │ │ │ └── logical-group.component.ts │ │ │ ├── logical-group.module.ts │ │ │ └── services/ │ │ │ └── logical-group.service.ts │ │ ├── login/ │ │ │ ├── components/ │ │ │ │ ├── login.component.html │ │ │ │ ├── login.component.scss │ │ │ │ ├── login.component.spec.ts │ │ │ │ └── login.component.ts │ │ │ ├── login.module.ts │ │ │ └── services/ │ │ │ └── login.service.ts │ │ ├── logout/ │ │ │ ├── components/ │ │ │ │ ├── logout.component.html │ │ │ │ ├── logout.component.scss │ │ │ │ ├── logout.component.spec.ts │ │ │ │ └── logout.component.ts │ │ │ └── logout.module.ts │ │ ├── options/ │ │ │ ├── components/ │ │ │ │ ├── options.component.html │ │ │ │ ├── options.component.scss │ │ │ │ ├── options.component.spec.ts │ │ │ │ └── options.component.ts │ │ │ └── options.module.ts │ │ ├── topo-graph/ │ │ │ ├── components/ │ │ │ │ ├── topo-graph.component.html │ │ │ │ ├── topo-graph.component.scss │ │ │ │ ├── topo-graph.component.spec.ts │ │ │ │ └── topo-graph.component.ts │ │ │ ├── modules.ts │ │ │ └── services/ │ │ │ └── topo-graph.service.ts │ │ └── topologyGraph/ │ │ ├── components/ │ │ │ ├── index.ts │ │ │ ├── topologyGraph.component.html │ │ │ ├── topologyGraph.component.scss │ │ │ └── topologyGraph.component.ts │ │ ├── modules.ts │ │ └── services/ │ │ └── topologyGraph.service.ts │ ├── assets/ │ │ └── .gitkeep │ ├── browserslist │ ├── environments/ │ │ ├── environment.prod.ts │ │ └── environment.ts │ ├── index.html │ ├── json/ │ │ └── logicalGroup.json │ ├── karma.conf.js │ ├── main.ts │ ├── polyfills.ts │ ├── styles.css │ ├── test.ts │ ├── tsconfig.app.json │ ├── tsconfig.spec.json │ └── tslint.json ├── tsconfig.json └── tslint.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: '' labels: '' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Environment (please complete the following information):** - OS: [e.g. iOS] - Browser [e.g. chrome, safari] - Version [e.g. 22] **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/custom.md ================================================ --- name: Custom issue template about: Describe this issue template's purpose here. title: '' labels: '' assignees: '' --- ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: '' labels: '' assignees: '' --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ **Is this a BUG REPORT or FEATURE REQUEST?**: > Uncomment only one, leave it on its own line: > > /kind bug > /kind feature **What happened**: **What you expected to happen**: **How to reproduce it (as minimally and precisely as possible)**: **Anything else we need to know?**: **Environment**: - golang version: - Kubernetes version (use `kubectl version`): - Cloud provider or hardware configuration: - OS (e.g. from /etc/os-release): - Kernel (e.g. `uname -a`): - Install tools: - Others: ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ **What this PR does / why we need it**: **Which issue(s) this PR fixes** *(optional, in `fixes #(, fixes #, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: **Release note**: ```release-note ``` ================================================ FILE: .gitignore ================================================ .idea .vscode .go/ bin/ vendor/ *.log # compiled output /dist /tmp /out-tsc tmp/ # dependencies node_modules/ ================================================ FILE: .make/Makefile.deploy.controller ================================================ IMAGE := $(DOCKER_REPO)/purser OUTPUT_DIR := tmp DOCKER_OUT := $(OUTPUT_DIR)/docker .PHONY: build build: $(DOCKER_OUT)/bin/$(ARCH)/$(BIN) $(DOCKER_OUT)/bin/$(ARCH)/$(BIN): build-dirs @echo "building: $@" @docker run \ -ti \ -u $$(id -u):$$(id -g) \ -v $$(pwd)/$(DOCKER_OUT)/.go:/go:$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(BUILD):/go/src/$(PRO)/$(BUILD):$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(CMD):/go/src/$(PRO)/$(CMD):$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(PKG):/go/src/$(PRO)/$(PKG):$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(DEP):/go/src/$(PRO)/$(DEP):$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(DOCKER_OUT)/bin/$(ARCH):/go/bin:$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(DOCKER_OUT)/bin/$(ARCH):/go/bin/linux_$(ARCH):$(DOCKER_MOUNT_MODE) \ -v $$(pwd)/$(DOCKER_OUT)/.go/std/$(ARCH):/usr/local/go/pkg/linux_$(ARCH)_static:$(DOCKER_MOUNT_MODE) \ -w /go/src \ $(BUILD_IMAGE) \ /bin/sh -c " \ ARCH=$(ARCH) \ VERSION=$(VERSION) \ PKG=$(PKG) \ ./$(PRO)/$(BUILD)/build.sh \ " DOTFILE_IMAGE = $(subst :,_,$(subst /,_,$(IMAGE))-$(VERSION)) .PHONY: container container: $(DOCKER_OUT)/.container-$(DOTFILE_IMAGE) container-name $(DOCKER_OUT)/.container-$(DOTFILE_IMAGE): $(DOCKER_OUT)/bin/$(ARCH)/$(BIN) Dockerfile.in @sed \ -e 's|ARG_DOCK|$(DOCKER_OUT)|g' \ -e 's|ARG_BIN|$(BIN)|g' \ -e 's|ARG_ARCH|$(ARCH)|g' \ -e 's|ARG_FROM|$(BASEIMAGE)|g' \ Dockerfile.in > $(DOCKER_OUT)/.dockerfile-$(ARCH) @docker build -t $(IMAGE):$(VERSION) -f $(DOCKER_OUT)/.dockerfile-$(ARCH) . @docker images -q $(IMAGE):$(VERSION) > $@ .PHONY: container-name container-name: @echo "container: $(IMAGE):$(VERSION)" .PHONY: push push: $(DOCKER_OUT)/.push-$(DOTFILE_IMAGE) push-name $(DOCKER_OUT)/.push-$(DOTFILE_IMAGE): $(DOCKER_OUT)/.container-$(DOTFILE_IMAGE) ifeq ($(findstring gcr.io,$(DOCKER_REPO)),gcr.io) @gcloud docker -- push $(IMAGE):$(VERSION) else @docker push $(IMAGE):$(VERSION) endif @docker images -q $(IMAGE):$(VERSION) > $@ .PHONY: push-name push-name: @echo "pushed: $(IMAGE):$(VERSION)" .PHONY: build-dirs build-dirs: @mkdir -p $(DOCKER_OUT) @mkdir -p $(DOCKER_OUT)/bin/$(ARCH) @mkdir -p $(DOCKER_OUT)/.go/src/$(PKG) $(DOCKER_OUT)/.go/pkg $(DOCKER_OUT)/.go/bin $(DOCKER_OUT)/.go/std/$(ARCH) .PHONY: clean clean: container-clean bin-clean .PHONY: container-clean container-clean: rm -rf $(DOCKER_OUT)/.container-* $(DOCKER_OUT)/.dockerfile-* $(DOCKER_OUT)/.push-* .PHONY: bin-clean bin-clean: rm -rf $(DOCKER_OUT)/ ================================================ FILE: .make/Makefile.deploy.purser ================================================ DEPLOY_DOCKERFILE?=ui/Dockerfile.deploy.purser CLUSTER_DIR?=${PWD}/cluster COMMIT:=$(shell git rev-parse --short HEAD) TIMESTAMP:=$(shell date +%s) TAG?=$(COMMIT)-$(TIMESTAMP) .PHONY: deploy-purser deploy-purser: kubectl-deploy-purser-db kubectl-deploy-purser-ui .PHONY: kubectl-deploy-purser-ui kubectl-deploy-purser-ui: @echo "Deploys purser-ui service" @kubectl create -f $(CLUSTER_DIR)/purser-ui.yaml .PHONY: deploy-purser-ui deploy-purser-ui: build-purser-ui-image push-purser-ui-image .PHONY: build-purser-ui-image build-purser-ui-image: @docker build --build-arg BINARY=purser-ui -t $(REGISTRY)/$(DOCKER_REPO)/purser-ui -f $(DEPLOY_DOCKERFILE) . @docker tag $(REGISTRY)/$(DOCKER_REPO)/purser-ui $(REGISTRY)/$(DOCKER_REPO)/purser-ui:$(TAG) .PHONY: push-purser-ui-image push-purser-ui-image: build-purser-ui-image @docker push $(REGISTRY)/$(DOCKER_REPO)/purser-ui .PHONY: clean-purser-ui-image clean-purser-ui-image: @docker rmi -f $(REGISTRY)/$(DOCKER_REPO)/purser-ui .PHONY: kubectl-deploy-purser-db kubectl-deploy-purser-db: @echo "Deploys purser purser-db service" @kubectl create -f $(CLUSTER_DIR)/purser-db.yaml ================================================ FILE: .travis.yml ================================================ services: - docker language: go os: - linux go: - "1.10" script: - make tools - make deps - make install - make travis-build - make check notifications: email: false ================================================ FILE: CODE_OF_CONDUCT.md ================================================ Contributor Code of Conduct ====================== As contributors and maintainers of this project, we pledge to respect everyone who contributes by posting issues, updating documentation, submitting pull requests, providing feedback in comments, and any other activities. Communication through any project channels (GitHub, mailing lists, Twitter, and so on) must be constructive and never resort to personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct. We promise to extend courtesy and respect to everyone involved in this project, regardless of gender, gender identity, sexual orientation, disability, age, race, ethnicity, religious beliefs, or level of experience. We expect anyone contributing to this project to do the same. If any member of the community violates this code of conduct, the maintainers of this project may take action, including removing issues, comments, and PRs or blocking accounts, as deemed appropriate. If you are subjected to or witness unacceptable behavior, or have any other concerns, please communicate with us. If you have suggestions to improve the code of conduct, please submit an issue or PR. **Attribution** This Code of Conduct is adapted from the VMware Clarity project, available at this page: https://github.com/vmware/clarity/blob/master/CODE_OF_CONDUCT.md ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to Purser Welcome! We gladly accept contributions from the community. If you wish to contribute code and you have not signed our contributor license agreement (CLA), our bot will update the issue when you open a pull request. For any questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq). ## Logging Bugs Anyone can log a bug using the GitHub 'New Issue' button. Please use a short title and give as much information as you can about what the problem is, relevant software versions, and how to reproduce it. If you know the fix or a workaround include that too. ## Install dependencies - Install [git](https://git-scm.com/downloads) - Install [Go](https://golang.org/dl/) version at least 1.7 - Set GOPATH environment variable. [https://github.com/golang/go/wiki/SettingGOPATH](https://github.com/golang/go/wiki/SettingGOPATH) - Add GOPATH/bin in system PATH variable ## Code Contribution Flow We use GitHub pull requests to incorporate code changes from external contributors. Typical contribution flow steps are: - Fork the Purser repo into a new repo on GitHub - Clone the forked repo locally and set the original Purser repo as the upstream repo - Make changes in a topic branch and commit - Fetch changes from upstream and resolve any merge conflicts so that your topic branch is up-to-date - Push all commits to the topic branch in your forked repo - Submit a pull request to merge topic branch commits to upstream master If this process sounds unfamiliar have a look at the excellent [overview of collaboration via pull requests on GitHub](https://help.github.com/categories/collaborating-with-issues-and-pull-requests) for more information. ## Coding Style Our standard for Golang contributions is to match the format of the [standard Go package library](https://golang.org/pkg). - Run `go fmt` on all code. - All public interfaces, functions, and structs must have complete, grammatically correct Godoc comments that explain their purpose and proper usage. - Use self-explanatory names for all variables, functions, and interfaces. - Add comments for non-obvious features of internal implementations but otherwise let the code explain itself. - Include unit tests for new features and update tests for old ones. Go is pretty readable so if you follow these rules most functions will not need additional comments. ### Commit Message Format We follow the conventions on [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/). Be sure to include any related GitHub issue references in the commit message. See [GFM syntax](https://guides.github.com/features/mastering-markdown/#GitHub-flavored-markdown) for referencing issues. ### Sign the Contributor License Agreement (CLA) VMware Apache-licensed projects require all contributors to sign a CLA. Visit https://cla.vmware.com and follow steps presented there. ### Fork the Repo Navigate to the [Purser repo on GitHub](https://github.com/vmware/purser) and use the 'Fork' button to create a forked repository under your GitHub account. This gives you a copy of the repo for pull requests back to purser in https://github.com/your-github-id/purser ### Clone and Set Upstream Remote Make a local clone of the forked repo and add the base purser repo as the upstream remote repository. ``` shell # (go to directory $GOPATH/src/github.com/vmware) cd $GOPATH/src/github.com/vmware # (clone the forked repository) git clone https://github.com//purser.git # (go to purser directory) cd $GOPATH/src/github.com/vmware/purser # (add upstream repository as the original purser repo) git remote add upstream https://github.com/vmware/purser.git ``` The last git command prepares your clone to pull changes from the upstream repo and push them into the fork, which enables you to keep the fork up to date. More on that shortly. ### Download dependencies Run the following commands to download dependencies. ``` shell make tools make deps make install ``` ### Make Changes and Commit Start a new topic branch(say branch-name: `foo-api-fix-22`) from the current HEAD position on master and commit your feature changes into that branch. ``` shell git checkout -b foo-api-fix-22 master # (Make feature changes) ``` Ensure that you run the following commands to ensure new dependencies are recorded and to fix formatting of the code. ``` shell make update make format make check ``` If there is an error while running `make check`, fix them and re run the above commands. ``` shell git commit -a --signoff git push origin foo-api-fix-22 ``` The --signoff puts your signature in the commit. It's required by our CLA bot. It is a git best practice to put work for each new feature in a separate topic branch and use git checkout commands to jump between them. This makes it possible to have multiple active pull requests. We can accept pull requests from any branch, so it's up to you how to manage them. ### Stay in Sync with Upstream From time to time you'll need to merge changes from the upstream repo so your topic branch stays in sync with other checkins. To do so switch to your topic branch, pull from the upstream repo, and push into the fork. If there are conflicts you'll need to [merge them now](https://stackoverflow.com/questions/161813/how-to-resolve-merge-conflicts-in-git). ``` shell git checkout foo-api-fix-22 git fetch -a git pull --rebase upstream master --tags git push --force-with-lease origin foo-api-fix-22 ``` The git pull and push options are important. Here are some details if you need deeper understanding. - 'pull --rebase' eliminates unnecessary merges by replaying your commit(s) into the log as if they happened after the upstream changes. Check out [What is a "merge bubble"?](https://stackoverflow.com/questions/26239379/what-is-a-merge-bubble) for why this is important. - --tags ensures that object tags are also pulled - Depending on your git configuration push --force-with-lease is required to make git update your fork with commits from the upstream repo. ### Create a Pull Request Github docs on creating a pull request from a fork: [Pull request from a fork](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) To contribute your feature, create a pull request by going to the [purser upstream repo on GitHub](https://github.com/vmware/purser) and pressing the 'New pull request' button. Select 'compare across forks' and select your-github-id/purser as 'head fork' and foo-api-fix-22 as the 'compare' branch. Leave the base fork as vmware/purser and master. ### Wait... A committer will look the request over and do one of three things: - accept it - send back comments about things you need to fix - or close the request without merging if we don't think it's a good addition. ### Updating Pull Requests with New Changes If your pull request needs changes based on code review, you'll most likely want to squash the fixes into existing commits. If your pull request contains a single commit or your changes are related to the most recent commit, you can simply amend the commit. ``` shell git add . git commit --amend git push --force-with-lease origin foo-api-fix-22 ``` If you need to squash changes into an earlier commit, you can use: ``` shell git add . git commit --fixup git rebase -i --autosquash master git push --force-with-lease origin foo-api-fix-22 ``` Be sure to add a comment to the pull request indicating your new changes are ready to review, as GitHub does not generate a notification when you git push. ## Final Words Thanks for helping us make the project better! ================================================ FILE: Dockerfile.in ================================================ FROM ARG_FROM LABEL maintainer = "VMware " LABEL author = "Krishna Karthik " ADD ARG_DOCK/bin/ARG_ARCH/ARG_BIN /ARG_BIN ================================================ FILE: Gopkg.toml ================================================ # Gopkg.toml example # # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html # for detailed Gopkg.toml documentation. # # required = ["github.com/user/thing/cmd/thing"] # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] # # [[constraint]] # name = "github.com/user/project" # version = "1.0.0" # # [[constraint]] # name = "github.com/user/project2" # branch = "dev" # source = "github.com/myfork/project2" # # [[override]] # name = "github.com/x/y" # version = "2.4.0" # # [prune] # non-go = false # go-tests = true # unused-packages = true [[constraint]] name = "k8s.io/api" version = "kubernetes-1.9.0" [[constraint]] name = "k8s.io/apiextensions-apiserver" version = "kubernetes-1.9.0" [[constraint]] name = "k8s.io/apimachinery" version = "kubernetes-1.9.0" [[constraint]] name = "k8s.io/client-go" version = "6.0.0" [[constraint]] name = "google.golang.org/grpc" version = "1.15.0" [[constraint]] name = "github.com/dgraph-io/dgo" branch = "master" [[override]] name = "github.com/tidwall/gjson" version = "1.1.2" [prune] go-tests = true unused-packages = true ================================================ FILE: LICENSE ================================================ Purser Copyright (c) 2018 VMware, Inc. All rights reserved. The Apache 2.0 license (the License) set forth below applies to all parts of the Purser project. You may not use this file except in compliance with the License. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty- free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: a. You must give any other recipients of the Work or Derivative Works a copy of this License; and b. You must cause any modified files to carry prominent notices stating that You changed the files; and c. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and d. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: Makefile ================================================ # Copyright (c) 2018 VMware Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # The binary to build (just the basename). BIN := controller # This repo's root import path (under GOPATH) PRO := github.com/vmware/purser DEP := vendor BUILD := build PKG := pkg CMD := cmd/controller # Where to push the docker image. REGISTRY?=docker.io DOCKER_REPO?=kreddyj # Which architecture to build - see $(ALL_ARCH) for options. ARCH?= amd64 # This version-strategy uses a manual value to set the version string VERSION := controller-1.0.2 ### ### These variables should not need tweaking. ### ALL_ARCH := amd64 arm arm64 ppc64le BASEIMAGE?=photon BUILD_IMAGE?=golang:1.11 DOCKER_MOUNT_MODE=delegated # Set dep management tool parameters VENDOR_DIR := vendor DEP_BIN_NAME := dep DEP_BIN_DIR := ./tmp/bin DEP_BIN := $(DEP_BIN_DIR)/$(DEP_BIN_NAME) DEP_VERSION := v0.5.0 # Define and get the vakue for UNAME_S variable from shell UNAME_S := $(shell uname -s) .PHONY: travis-build travis-build: install-plugin install-controller travis-success .PHONY: install-plugin install-plugin: go install github.com/vmware/purser/cmd/plugin .PHONY: install-controller install-controller: build container .PHONY: travis-success travis-success: @echo "travis build success" # If you want to build all binaries, see the 'all-build' rule. # If you want to build all containers, see the 'all-container' rule. # If you want to build AND push all containers, see the 'all-push' rule. .PHONY: all all: deps build check build-%: @$(MAKE) --no-print-directory ARCH=$* build container-%: @$(MAKE) --no-print-directory ARCH=$* container push-%: @$(MAKE) --no-print-directory ARCH=$* push .PHONY: all-build all-build: $(addprefix build-, $(ALL_ARCH)) .PHONY: all-container all-container: $(addprefix container-, $(ALL_ARCH)) .PHONY: all-push all-push: $(addprefix push-, $(ALL_ARCH)) .PHONY: deps ## Download build dependencies. deps: $(DEP_BIN) $(VENDOR_DIR) # install dep in a the tmp/bin dir of the repo $(DEP_BIN): @echo "Installing 'dep' $(DEP_VERSION) at '$(DEP_BIN_DIR)'..." mkdir -p $(DEP_BIN_DIR) ifeq ($(UNAME_S),Darwin) @curl -L -s https://github.com/golang/dep/releases/download/$(DEP_VERSION)/dep-darwin-amd64 -o $(DEP_BIN) @cd $(DEP_BIN_DIR) && \ echo "1a7bdb0d6c31ecba8b3fd213a1170adf707657123e89dff234871af9e0498be2 dep" > dep-darwin-amd64.sha256 && \ shasum -a 256 --check dep-darwin-amd64.sha256 else @curl -L -s https://github.com/golang/dep/releases/download/$(DEP_VERSION)/dep-linux-amd64 -o $(DEP_BIN) @cd $(DEP_BIN_DIR) && \ echo "287b08291e14f1fae8ba44374b26a2b12eb941af3497ed0ca649253e21ba2f83 dep" > dep-linux-amd64.sha256 && \ sha256sum -c dep-linux-amd64.sha256 endif @chmod +x $(DEP_BIN) $(VENDOR_DIR): Gopkg.toml Gopkg.lock @echo "checking dependencies..." @$(DEP_BIN) ensure -v .PHONY: install install: ## Fetches all dependencies using dep @$(DEP_BIN) ensure -v .PHONY: update update: ## Updates all dependencies defined for dep @$(DEP_BIN) ensure -update -v include ./.make/Makefile.deploy.controller include ./.make/Makefile.deploy.purser .PHONY: version version: @echo $(VERSION) .PHONY: clean-vendor ## Removes the ./vendor directory. clean-vendor: -rm -rf $(VENDOR_DIR) GOFORMAT_FILES := $(shell find . -name '*.go' | grep -v /vendor/) .PHONY: format ## Formats any go file that differs from gofmt's style and removes unused imports format: @gofmt -s -l -w ${GOFORMAT_FILES} @goimports -l -w ${GOFORMAT_FILES} .PHONY: tools tools: ## Installs required go tools @go get -u github.com/alecthomas/gometalinter && gometalinter --install @go get -u golang.org/x/tools/cmd/goimports .PHONY: check check: ## Concurrently runs a whole bunch of static analysis tools gometalinter --enable=misspell --enable-gc --vendor --deadline 300s ./... ================================================ FILE: NOTICE ================================================ Purser Copyright (c) 2018 VMware, Inc. All Rights Reserved. This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in compliance with the Apache 2.0 License. This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. ================================================ FILE: README.md ================================================ ![logo](https://user-images.githubusercontent.com/42761785/53145168-2f4e4980-35c5-11e9-867b-8d637671ec23.png) # K8s Extension for Application Visibility [![Build Status](https://travis-ci.org/vmware/purser.svg?branch=master)](https://travis-ci.org/vmware/purser) [![Go Report Card](https://goreportcard.com/badge/github.com/vmware/purser)](https://goreportcard.com/report/github.com/vmware/purser) - [What is Purser?](#overview) - [Features](#features) - [Setup and Installation](#setup-and-installation) - [Uninstalling](#uninstalling) - [API Documentation](#api-documentation) - [Additional Documentation](#additional-documentation) - [Community, Discussion, Contribution and Support](#community-discussion-contribution-and-support) ## Overview Purser is an extension to Kubernetes tasked at providing an insight into *cluster topology*, *costing*, *capacity allocations* and *resource interactions* along with the provision of *logical grouping of resources* for Kubernetes based cloud native applications in a cloud neutral manner, with the focus on catering to a multitude of users ranging from Sys Admins, to DevOps to Developers. It comprises of three components: a controller, a plugin and a UI dashboard. The controller component deployed inside the cluster watches for K8s native and custom resources associated with the application, thereby, periodically building not just an inventory but also performing discovery by generating and storing the interactions among the resources such as containers, pods and services. The plugin component is a CLI tool interfacing with the `kubectl` that helps query costs, savings defined at a level of control of the application level components rather than at the infrastructure level. The UI dashboard is a robust application that renders the Purser UI for providing visual representation to the complete cluster metrics in a single pane of glass. ## Features Purser with its robust CLI and UI capabilities provides a set of features including, but not limited to the list below. - Capability to provide visibility into the following aspects of the K8s cluster - workload cost associated with the native/custom resources - savings opportunities associated with storage and compute requirements - single pane view of the complete cluster hierarchy - capacity allocations for CPU, memory, disk space and other resources - interactions among associated resources such as pods and services - Capability of user defined logical grouping of resources based on `K8s CRD` implementation for enhanced filtering. - A plugin extension to `kubectl` along with the UI for developer centric usage. - Capability to subscribe to inventory changes via web-hook implementation. ### UI Demo ![demo](https://user-images.githubusercontent.com/42461220/54865566-35367680-4d8d-11e9-9e07-e9aa77d7c6ec.gif) ### CLI Demo ![demo](/docs/img/purser-cli.gif) ## Setup and Installation ### Prerequisites - Kubernetes version 1.9 or greater. - `kubectl` installed and configured. For details see [here](https://kubernetes.io/docs/tasks/tools/install-kubectl/). - `curl` installed. Download it [here](https://curl.haxx.se/download.html) ### Linux/Mac Users: ```bash curl https://raw.githubusercontent.com/vmware/purser/master/build/purser-setup.sh -O && sh purser-setup.sh ``` _NOTE: If you want to try out purser on minikube, you can do the following steps instead._ ```bash curl https://raw.githubusercontent.com/vmware/purser/master/build/purser-minimal-setup.sh -O && sh purser-minimal-setup.sh # Wait for containers to start, around 30s # Open Purser in browser minikube service purser-ui -n purser ``` ### Windows/Other Users: For detailed installation steps follow the instructions in the [manual installation guide](./docs/manual-installation.md). ### Purser Plugin Setup (Optional) _NOTE: This Plugin installation is optional. This feature is not actively maintained and will be deprecated soon._ If you want to install and use Purser's command line interface - [Plugin installation guide](./docs/plugin-installation.md). - [Plugin Usage](./docs/plugin-usage.md). ### Other Installation Methods For other installation methods such as **manual installation** or **installation from source code** refer guides in [docs](./docs). ### Uninstalling ``` bash kubectl delete ns purser ``` _**NOTE:** Use flag `--kubeconfig=` if your cluster configuration is not at the [default location](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable)._ ## API Documentation The project uses Swagger to document API's endpoints. The documentation is available at [Swagger Hub](https://app.swaggerhub.com/apis/hemani19/purser/1.0.0). ## Additional Documentation Additional documentation can be found below: - [Manual Installation Guide](./docs/manual-installation.md) - [Source Code Installation Guide](./docs/sourcecode-installation.md) - [Purser Architecture and Workflow](./docs/architecture.md) - [Purser Plugin Usage](./docs/plugin-usage.md) - [Developers Guide](./docs/developers-guide.md) - [Purser Deployment Guide](./docs/purser-deployment.md) - [Purser UI Development Guide](./ui/README.md) ## Community, Discussion, Contribution and Support **Issues:** Have an issue with Purser, please [log it](https://github.com/vmware/purser/issues). **Contributing:** Would you like to contribute to our project, refer [How to contribute](./CONTRIBUTING.md), [Developers Guide](./docs/developers-guide.md) and [Code of Conduct](./CODE_OF_CONDUCT.md) docs. ================================================ FILE: build/build.sh ================================================ # Copyright (c) 2018 VMware Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset if [ -z "${PKG}" ]; then echo "PKG must be set" exit 1 fi if [ -z "${ARCH}" ]; then echo "ARCH must be set" exit 1 fi if [ -z "${VERSION}" ]; then echo "VERSION must be set" exit 1 fi export CGO_ENABLED=0 export GOARCH="${ARCH}" go install \ -installsuffix "static" \ -ldflags "-X ${PKG}/version.VERSION=${VERSION}" \ ./... ================================================ FILE: build/purser-binary-install.sh ================================================ # Copyright (c) 2018 VMware Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Realease Version releaseVersion=v1.0.0 # === Purser Plugin === # Detecting os type unameOut="$(uname -s)" case "${unameOut}" in Linux*) machine=Linux;; Darwin*) machine=Mac;; CYGWIN*) machine=Cygwin;; MINGW*) machine=MinGw;; *) machine="UNKNOWN:${unameOut}" esac echo "Detecting your Operating System: ${machine}" echo "Downloading files for plugin..." # Download purser plugin yaml pluginYamlUrl=https://github.com/vmware/purser/releases/download/$releaseVersion/plugin.yaml wget -q --show-progress -O plugin.yaml $pluginYamlUrl # Downloading purser plugin binary based on os type if [ $machine = Linux ] then pluginUrl=https://github.com/vmware/purser/releases/download/$releaseVersion/purser_plugin_linux_amd64 elif [ $machine = Mac ] then pluginUrl=https://github.com/vmware/purser/releases/download/$releaseVersion/purser_plugin_darwin_amd64 else echo "No match found for your os: $machine" echo "Install the plugin from source code: https://github.com/vmware/purser/blob/master/README.md" exit 3 # unsuccessful shell script fi wget -q --show-progress -O purser_plugin $pluginUrl # Move th plugin yaml to one of the location specified in # https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/ if [ ! -d $HOME/.kube/plugins ] then mkdir $HOME/.kube/plugins fi echo "Moving plugin.yaml to $HOME/.kube/plugins/" mv plugin.yaml $HOME/.kube/plugins/ # Change execution permissions for the binary chmod +x purser_plugin # Move the binary to a location which is in environment PATH variable echo "Moving the binary to /usr/local/bin" sudo mv purser_plugin /usr/local/bin echo "Purser plugin installation Completed" echo "" echo "Purser Installation Completed" ================================================ FILE: build/purser-binary-uninstall.sh ================================================ # Copyright (c) 2018 VMware Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. echo "Removing plugin.yaml from $HOME/.kube/plugins/" rm $HOME/.kube/plugins/plugin.yaml echo "Removing the binary from /usr/local/bin" sudo rm /usr/local/bin/purser_plugin ================================================ FILE: build/purser-minimal-setup.sh ================================================ # Copyright (c) 2018 VMware Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Realease Version releaseVersion=1.0.2 echo "Installing Purser (minimal setup) version: ${releaseVersion}" # Namespace setup echo "Creating namespace purser" kubectl create ns purser # DB setup echo "Setting up database for Purser" curl https://raw.githubusercontent.com/vmware/purser/master/cluster/minimal/purser-database-setup.yaml -O kubectl --namespace=purser create -f purser-database-setup.yaml echo "Waiting for database containers to be in running state... (30s)" sleep 30s # Purser controller setup echo "Setting up controller for Purser" curl https://raw.githubusercontent.com/vmware/purser/master/cluster/minimal/purser-controller-setup.yaml -O kubectl --namespace=purser create -f purser-controller-setup.yaml # Purser UI setup echo "Setting up UI for Purser" curl https://raw.githubusercontent.com/vmware/purser/master/cluster/minimal/purser-ui-setup.yaml -O kubectl --namespace=purser create -f purser-ui-setup.yaml echo "Purser setup is completed" ================================================ FILE: build/purser-setup.sh ================================================ # Copyright (c) 2018 VMware Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Kubeconfig location read -p "Location for cluster's configuration (Press 'Enter' to take default $HOME/.kube/config): " readConfig if [ -z "$readConfig" ]; then kubeConfig="$HOME/.kube/config" else kubeConfig=$readConfig fi # Realease Version releaseVersion=1.0.2 echo "Installing Purser version: ${releaseVersion}" # Namespace setup echo "Creating namespace purser" kubectl --kubeconfig=$kubeConfig create ns purser # DB setup echo "Setting up database for Purser" curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-database-setup.yaml -O kubectl --kubeconfig=$kubeConfig --namespace=purser create -f purser-database-setup.yaml echo "Waiting for database containers to be in running state... (1 minute)" sleep 60s # Purser controller setup echo "Setting up controller for Purser" curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-controller-setup.yaml -O kubectl --kubeconfig=$kubeConfig --namespace=purser create -f purser-controller-setup.yaml # Purser UI setup echo "Setting up UI for Purser" curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-ui-setup.yaml -O kubectl --kubeconfig=$kubeConfig --namespace=purser create -f purser-ui-setup.yaml echo "Purser setup is completed" ================================================ FILE: cluster/artifacts/example-group.yaml ================================================ apiVersion: vmware.purser.com/v1 kind: Group metadata: name: example-group spec: name: example-group labels: expr1: app: - sample-app - sample-app2 env: - dev expr2: namespace: - ns1 - ns2 expr3: key1: - val1 key2: - val2 ================================================ FILE: cluster/artifacts/example-subscriber.yaml ================================================ apiVersion: vmware.purser.com/v1 kind: Subscriber metadata: name: example-subscriber spec: name: example-subscriber headers: authorization: "Bearer " cluster: "" url: ================================================ FILE: cluster/artifacts/group-template.json ================================================ { "apiVersion": "vmware.purser.com/v1", "kind": "Group", "metadata": { "name": "" }, "spec": { "name": "", "labels": { "expr1": { "": [ "", "" ], "": [ "" ] }, "expr2": { "": [ "" ], "": [ "" ] } } } } ================================================ FILE: cluster/artifacts/purser-group-crd.yaml ================================================ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: groups.vmware.purser.com spec: group: vmware.purser.com names: kind: Group listKind: GroupList plural: groups singular: group scope: Namespaced version: v1 status: acceptedNames: kind: Group listKind: GroupList plural: groups singular: group ================================================ FILE: cluster/artifacts/purser-subscriber-crd.yaml ================================================ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: subscribers.vmware.purser.com spec: group: vmware.purser.com names: kind: Subscriber listKind: SubscriberList plural: subscribers singular: subscriber scope: Namespaced version: v1 status: acceptedNames: kind: Subscriber listKind: SubscriberList plural: subscribers singular: subscriber ================================================ FILE: cluster/helm/chart/purser/.helmignore ================================================ # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. .DS_Store # Common VCS dirs .git/ .gitignore .bzr/ .bzrignore .hg/ .hgignore .svn/ # Common backup files *.swp *.bak *.tmp *~ # Various IDEs .project .idea/ *.tmproj .vscode/ ================================================ FILE: cluster/helm/chart/purser/Chart.yaml ================================================ apiVersion: v1 appVersion: "1.0" description: A Helm chart for Purser name: purser version: 0.1.0 ================================================ FILE: cluster/helm/chart/purser/README.md ================================================ # [Purser](https://github.com/vmware/purser) Purser is an extension to Kubernetes tasked at providing an insight into cluster topology, costing, capacity allocations and resource interactions along with the provision of logical grouping of resources for Kubernetes based cloud native applications in a cloud neutral manner, with the focus on catering to a multitude of users ranging from Sys Admins, to DevOps to Developers. It comprises of three components: a controller, a plugin and a UI dashboard. The controller component deployed inside the cluster watches for K8s native and custom resources associated with the application, thereby, periodically building not just an inventory but also performing discovery by generating and storing the interactions among the resources such as containers, pods and services. The plugin component is a CLI tool interfacing with the kubectl that helps query costs, savings defined at a level of control of the application level components rather than at the infrastructure level. The UI dashboard is a robust application that renders the Purser UI for providing visual representation to the complete cluster metrics in a single pane of glass. > Taken from main [README](https://github.com/vmware/purser/blob/master/README.md) > [Plugin installation guide](https://github.com/vmware/purser/blob/master/README.md#purser-plugin-setup) ## Chart Configuration *See `values.yaml` for configuration notes. Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, ```console $ helm install --name purser \ --set database.storage=10Gi \ purser ``` Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, ```console $ helm install --name purser -f values.yaml ``` > **Tip**: You can use the default [values.yaml](values.yaml) ================================================ FILE: cluster/helm/chart/purser/templates/NOTES.txt ================================================ 1. Get the application URL by running these commands: {{- if .Values.ui.ingress.enabled }} {{- range $host := .Values.ui.ingress.hosts }} {{- range .paths }} http{{ if $.Values.ui.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} {{- end }} {{- end }} {{- else if contains "NodePort" .Values.ui.service.type }} export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "purser.fullname" . }}-ui) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.ui.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "purser.fullname" . }}-ui' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "purser.fullname" . }}-ui -o jsonpath='{.status.loadBalancer.ingress[0].ip}') echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.ui.service.type }} export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "purser.name" . }}-ui,app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl port-forward $POD_NAME 8080:80 {{- end }} ================================================ FILE: cluster/helm/chart/purser/templates/_helpers.tpl ================================================ {{/* vim: set filetype=mustache: */}} {{/* Expand the name of the chart. */}} {{- define "purser.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} {{- define "purser.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} {{- $name := default .Chart.Name .Values.nameOverride -}} {{- if contains $name .Release.Name -}} {{- .Release.Name | trunc 63 | trimSuffix "-" -}} {{- else -}} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{- end -}} {{- end -}} {{/* Create chart name and version as used by the chart label. */}} {{- define "purser.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} ================================================ FILE: cluster/helm/chart/purser/templates/purser-controller-deployment.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "purser.fullname" . }}-controller namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }}-controller helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: replicas: {{ .Values.controller.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "purser.name" . }}-controller app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: labels: app.kubernetes.io/name: {{ include "purser.name" . }}-controller app.kubernetes.io/instance: {{ .Release.Name }} spec: serviceAccountName: {{ include "purser.fullname" . }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" imagePullPolicy: {{ .Values.controller.image.pullPolicy }} command: - "/controller" args: - "--cookieKey=purser-super-secret-key" - "--cookieName=purser-session-token" - "--log=info" {{- if .Values.controller.interactions }} - "--interactions=enable" {{- else }} - "--interactions=disable" {{- end }} - "--dgraphURL={{ include "purser.fullname" . }}-database" - "--dgraphPort=9080" ports: - name: http containerPort: 3030 protocol: TCP resources: {{- toYaml .Values.controller.resources | nindent 12 }} initContainers: - name: init-sleep image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" command: ["/usr/bin/bash", "-c", "sleep 60"] {{- with .Values.controller.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.controller.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.controller.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-controller-rbac.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: {{ include "purser.fullname" . }} labels: app.kubernetes.io/name: {{ include "purser.name" . }} helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "watch", "list", "update", "create", "delete"] - apiGroups: ["vmware.purser.com"] resources: ["groups", "subscribers"] verbs: ["get", "watch", "list", "update", "create", "delete"] - apiGroups: ["*"] resources: ["*"] verbs: ["get", "watch", "list"] {{- if .Values.controller.interaction }} - apiGroups: ["*"] resources: ["pods/exec"] verbs: ["create"] {{- end }} --- # ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: {{ include "purser.fullname" . }} labels: app.kubernetes.io/name: {{ include "purser.name" . }} helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ include "purser.fullname" . }} subjects: - kind: ServiceAccount name: {{ include "purser.fullname" . }} namespace: {{ .Release.Namespace }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-controller-serviceaccount.yaml ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "purser.fullname" . }} namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }} helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-controller-svc.yaml ================================================ apiVersion: v1 kind: Service metadata: name: {{ include "purser.fullname" . }}-controller namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }} helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: type: {{ .Values.controller.service.type }} ports: - port: 3030 targetPort: http protocol: TCP selector: app.kubernetes.io/name: {{ include "purser.name" . }}-controller app.kubernetes.io/instance: {{ .Release.Name }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-database-statefulset.yaml ================================================ apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ include "purser.fullname" . }}-database namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }}-database helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: serviceName: "dgraph" replicas: 1 selector: matchLabels: app.kubernetes.io/name: {{ include "purser.name" . }}-database app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: labels: app.kubernetes.io/name: {{ include "purser.name" . }}-database app.kubernetes.io/instance: {{ .Release.Name }} spec: containers: - name: {{ .Chart.Name }}-zero image: "{{ .Values.database.zero.image.repository }}:{{ .Values.database.zero.image.tag }}" imagePullPolicy: {{ .Values.database.zero.image.pullPolicy }} ports: - containerPort: 5080 name: zero-grpc - containerPort: 6080 name: zero-http volumeMounts: - name: datadir mountPath: /dgraph command: - bash - "-c" - | set -ex dgraph zero --my=0.0.0.0:5080 resources: {{- toYaml .Values.database.zero.resources | nindent 12 }} - name: {{ .Chart.Name }}-server image: "{{ .Values.database.zero.image.repository }}:{{ .Values.database.zero.image.tag }}" imagePullPolicy: {{ .Values.database.zero.image.pullPolicy }} ports: - containerPort: 8080 name: server-http - containerPort: 9080 name: server-grpc volumeMounts: - name: datadir mountPath: /dgraph command: - bash - "-c" - | set -ex dgraph server --my=0.0.0.0:7080 --lru_mb 2048 --zero 0.0.0.0:5080 resources: {{- toYaml .Values.database.server.resources | nindent 12 }} terminationGracePeriodSeconds: 60 volumes: - name: datadir persistentVolumeClaim: claimName: datadir {{- with .Values.database.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.database.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.database.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} volumeClaimTemplates: - metadata: name: datadir annotations: volume.alpha.kubernetes.io/storage-class: anything spec: accessModes: - "ReadWriteOnce" resources: requests: storage: {{ .Values.database.storage | default "10Gi" }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-database-svc.yaml ================================================ apiVersion: v1 kind: Service metadata: name: {{ include "purser.fullname" . }}-database namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }} helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: type: {{ .Values.database.service.type }} ports: - port: 5080 targetPort: 5080 name: zero-grpc - port: 6080 targetPort: 6080 name: zero-http - port: 8080 targetPort: 8080 name: server-http - port: 9080 targetPort: 9080 name: server-grpc selector: app.kubernetes.io/name: {{ include "purser.name" . }}-database app.kubernetes.io/instance: {{ .Release.Name }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-ui-configmap.yaml ================================================ apiVersion: v1 kind: ConfigMap metadata: name: {{ include "purser.fullname" . }}-ui namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }}-ui helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: nginx.conf: | upstream purser { server {{ include "purser.fullname" . }}-controller:3030; } server { listen 4200; location /auth { proxy_pass http://purser; } location /api { proxy_pass http://purser; } location / { root /usr/share/nginx/html/purser; index index.html index.htm; try_files $uri $uri/ /index.html =404; } } ================================================ FILE: cluster/helm/chart/purser/templates/purser-ui-deployment.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "purser.fullname" . }}-ui namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }}-ui helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: replicas: {{ .Values.ui.replicaCount }} selector: matchLabels: app.kubernetes.io/name: {{ include "purser.name" . }}-ui app.kubernetes.io/instance: {{ .Release.Name }} template: metadata: labels: app.kubernetes.io/name: {{ include "purser.name" . }}-ui app.kubernetes.io/instance: {{ .Release.Name }} spec: volumes: - configMap: defaultMode: 420 name: {{ include "purser.fullname" . }}-ui name: nginx containers: - name: {{ .Chart.Name }} image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag }}" imagePullPolicy: {{ .Values.ui.image.pullPolicy }} ports: - name: http containerPort: 4200 protocol: TCP volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx livenessProbe: httpGet: path: / port: http readinessProbe: httpGet: path: / port: http resources: {{- toYaml .Values.ui.resources | nindent 12 }} {{- with .Values.ui.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.ui.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.ui.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-ui-ingress.yaml ================================================ {{- if .Values.ui.ingress.enabled -}} {{- $fullName := include "purser.fullname" . -}} apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ $fullName }} namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }} helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- with .Values.ui.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: {{- if .Values.ui.ingress.tls }} tls: {{- range .Values.ui.ingress.tls }} - hosts: {{- range .hosts }} - {{ . | quote }} {{- end }} secretName: {{ .secretName }} {{- end }} {{- end }} rules: {{- range .Values.ui.ingress.hosts }} - host: {{ .host | quote }} http: paths: {{- range .paths }} - path: {{ . }} backend: serviceName: {{ $fullName }}-ui servicePort: http {{- end }} {{- end }} {{- end }} ================================================ FILE: cluster/helm/chart/purser/templates/purser-ui-svc.yaml ================================================ apiVersion: v1 kind: Service metadata: name: {{ include "purser.fullname" . }}-ui namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "purser.name" . }}-ui helm.sh/chart: {{ include "purser.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: type: {{ .Values.ui.service.type }} ports: - port: {{ .Values.ui.service.port }} targetPort: http protocol: TCP name: http selector: app.kubernetes.io/name: {{ include "purser.name" . }}-ui app.kubernetes.io/instance: {{ .Release.Name }} ================================================ FILE: cluster/helm/chart/purser/values.yaml ================================================ # Default values for purser. # This is a YAML-formatted file. # Declare variables to be passed into your templates. nameOverride: "" fullnameOverride: "" controller: replicaCount: 1 interaction: false image: repository: kreddyj/controller-amd64 tag: 1.0.2 pullPolicy: Always service: type: ClusterIP port: 80 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi nodeSelector: {} tolerations: [] affinity: {} database: replicaCount: 1 # Storage space given to dgraph storage: 10Gi service: type: ClusterIP zero: image: repository: dgraph/dgraph tag: v1.0.9 pullPolicy: IfNotPresent resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi server: image: repository: dgraph/dgraph tag: v1.0.9 pullPolicy: IfNotPresent resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi nodeSelector: {} tolerations: [] affinity: {} ui: replicaCount: 1 image: repository: kreddyj/purser-ui tag: 1.0.2 pullPolicy: Always service: type: ClusterIP port: 80 ingress: enabled: false annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" hosts: - host: chart-example.local paths: [] tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi nodeSelector: {} tolerations: [] affinity: {} ================================================ FILE: cluster/minimal/purser-controller-setup.yaml ================================================ # Service account apiVersion: v1 kind: ServiceAccount metadata: name: purser-service-account --- # RBAC apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: purser-permissions rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "watch", "list", "update", "create", "delete"] - apiGroups: ["vmware.purser.com"] resources: ["groups", "subscribers"] verbs: ["get", "watch", "list", "update", "create", "delete"] - apiGroups: ["*"] resources: ["*"] verbs: ["get", "watch", "list"] # Uncomment next three lines to enable interactions feature. # - apiGroups: ["*"] # resources: ["pods/exec"] # verbs: ["create"] --- # ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: purser-cluster-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: purser-permissions subjects: - kind: ServiceAccount name: purser-service-account namespace: purser --- apiVersion: v1 kind: Service metadata: name: purser spec: selector: app: purser ports: - protocol: TCP port: 3030 targetPort: http --- apiVersion: apps/v1 kind: Deployment metadata: name: purser spec: selector: matchLabels: app: purser replicas: 1 template: metadata: labels: app: purser spec: serviceAccountName: purser-service-account containers: - name: purser-controller image: kreddyj/purser:controller-1.0.2 imagePullPolicy: Always ports: - name: http containerPort: 3030 command: ["/controller"] args: ["--log=info", "--interactions=disable", "--dgraphURL=purser-db", "--dgraphPort=9080"] ================================================ FILE: cluster/minimal/purser-database-setup.yaml ================================================ # Service Dgraph - This is the service that should be used by the clients of Dgraph to talk to the server. apiVersion: v1 kind: Service metadata: name: purser-db labels: app: purser-db spec: type: ClusterIP ports: - port: 5080 targetPort: 5080 name: zero-grpc - port: 6080 targetPort: 6080 name: zero-http - port: 8080 targetPort: 8080 name: server-http - port: 9080 targetPort: 9080 name: server-grpc selector: app: purser-db --- # Dgraph StatefulSet runs 1 pod with one Zero and one Server containers. apiVersion: apps/v1 kind: StatefulSet metadata: name: purser-dgraph spec: serviceName: "dgraph" replicas: 1 selector: matchLabels: app: purser-db template: metadata: labels: app: purser-db spec: containers: - name: zero image: dgraph/dgraph:v1.0.9 imagePullPolicy: IfNotPresent ports: - containerPort: 5080 name: zero-grpc - containerPort: 6080 name: zero-http volumeMounts: - name: datadir mountPath: /dgraph command: - bash - "-c" - | set -ex dgraph zero --my=$(hostname -f):5080 - name: server image: dgraph/dgraph:v1.0.9 imagePullPolicy: IfNotPresent ports: - containerPort: 8080 name: server-http - containerPort: 9080 name: server-grpc volumeMounts: - name: datadir mountPath: /dgraph command: - bash - "-c" - | set -ex dgraph server --my=$(hostname -f):7080 --lru_mb 2048 --zero $(hostname -f):5080 terminationGracePeriodSeconds: 60 volumes: - name: datadir persistentVolumeClaim: claimName: datadir updateStrategy: type: RollingUpdate volumeClaimTemplates: - metadata: name: datadir annotations: volume.alpha.kubernetes.io/storage-class: anything spec: accessModes: - "ReadWriteOnce" resources: requests: storage: 5Gi ================================================ FILE: cluster/minimal/purser-ui-setup.yaml ================================================ apiVersion: v1 kind: Service metadata: name: purser-ui labels: run: purser-ui app: purser spec: selector: app: purser run: purser-ui ports: - protocol: TCP port: 80 targetPort: 4200 type: LoadBalancer --- apiVersion: apps/v1 kind: Deployment metadata: name: purser-ui spec: selector: matchLabels: app: purser run: purser-ui replicas: 1 template: metadata: labels: app: purser run: purser-ui spec: containers: - name: purser-ui image: kreddyj/purser:ui-1.0.2 imagePullPolicy: Always ports: - containerPort: 4200 ================================================ FILE: cluster/purser-controller-setup.yaml ================================================ # Service account apiVersion: v1 kind: ServiceAccount metadata: name: purser-service-account --- # RBAC apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: purser-permissions rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "watch", "list", "update", "create", "delete"] - apiGroups: ["vmware.purser.com"] resources: ["groups", "subscribers"] verbs: ["get", "watch", "list", "update", "create", "delete"] - apiGroups: ["*"] resources: ["*"] verbs: ["get", "watch", "list"] # Uncomment next three lines to enable interactions feature. # - apiGroups: ["*"] # resources: ["pods/exec"] # verbs: ["create"] --- # ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: purser-cluster-role roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: purser-permissions subjects: - kind: ServiceAccount name: purser-service-account namespace: purser --- apiVersion: v1 kind: Service metadata: name: purser spec: selector: app: purser ports: - protocol: TCP port: 3030 targetPort: http --- apiVersion: apps/v1 kind: Deployment metadata: name: purser spec: selector: matchLabels: app: purser replicas: 1 template: metadata: labels: app: purser spec: serviceAccountName: purser-service-account containers: - name: purser-controller image: kreddyj/purser:controller-1.0.2 imagePullPolicy: Always resources: limits: memory: 1000Mi cpu: 300m requests: memory: 1000Mi cpu: 300m ports: - name: http containerPort: 3030 command: ["/controller"] args: ["--log=info", "--interactions=disable", "--dgraphURL=purser-db", "--dgraphPort=9080"] ================================================ FILE: cluster/purser-database-setup.yaml ================================================ # Service Dgraph - This is the service that should be used by the clients of Dgraph to talk to the server. apiVersion: v1 kind: Service metadata: name: purser-db labels: app: purser-db spec: type: ClusterIP ports: - port: 5080 targetPort: 5080 name: zero-grpc - port: 6080 targetPort: 6080 name: zero-http - port: 8080 targetPort: 8080 name: server-http - port: 9080 targetPort: 9080 name: server-grpc selector: app: purser-db --- # Dgraph StatefulSet runs 1 pod with one Zero and one Server containers. apiVersion: apps/v1 kind: StatefulSet metadata: name: purser-dgraph spec: serviceName: "dgraph" replicas: 1 selector: matchLabels: app: purser-db template: metadata: labels: app: purser-db spec: containers: - name: zero image: dgraph/dgraph:v1.0.9 imagePullPolicy: IfNotPresent resources: limits: memory: 1000Mi cpu: 300m requests: memory: 1000Mi cpu: 300m ports: - containerPort: 5080 name: zero-grpc - containerPort: 6080 name: zero-http volumeMounts: - name: datadir mountPath: /dgraph command: - bash - "-c" - | set -ex dgraph zero --my=$(hostname -f):5080 - name: server image: dgraph/dgraph:v1.0.9 imagePullPolicy: IfNotPresent resources: limits: memory: 1500Mi cpu: 500m requests: memory: 1500Mi cpu: 500m ports: - containerPort: 8080 name: server-http - containerPort: 9080 name: server-grpc volumeMounts: - name: datadir mountPath: /dgraph command: - bash - "-c" - | set -ex dgraph server --my=$(hostname -f):7080 --lru_mb 2048 --zero $(hostname -f):5080 terminationGracePeriodSeconds: 60 volumes: - name: datadir persistentVolumeClaim: claimName: datadir updateStrategy: type: RollingUpdate volumeClaimTemplates: - metadata: name: datadir annotations: volume.alpha.kubernetes.io/storage-class: anything spec: accessModes: - "ReadWriteOnce" resources: requests: storage: 10Gi ================================================ FILE: cluster/purser-ui-setup.yaml ================================================ apiVersion: v1 kind: Service metadata: name: purser-ui labels: run: purser-ui app: purser spec: selector: app: purser run: purser-ui ports: - protocol: TCP port: 80 targetPort: 4200 type: LoadBalancer --- apiVersion: apps/v1 kind: Deployment metadata: name: purser-ui spec: selector: matchLabels: app: purser run: purser-ui replicas: 1 template: metadata: labels: app: purser run: purser-ui spec: containers: - name: purser-ui image: kreddyj/purser:ui-1.0.2 imagePullPolicy: Always resources: limits: memory: 1200Mi cpu: 500m requests: memory: 1200Mi cpu: 500m ports: - containerPort: 4200 ================================================ FILE: cmd/controller/api/api.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package api import ( "net/http" "github.com/Sirupsen/logrus" "github.com/gorilla/handlers" "github.com/vmware/purser/cmd/controller/api/apiHandlers" "github.com/vmware/purser/pkg/controller" ) // StartServer starts api server func StartServer(conf controller.Config) { apiHandlers.SetKubeClientAndGroupClient(conf) allowedOrigins := handlers.AllowedOrigins([]string{"*"}) allowedCredentials := handlers.AllowCredentials() router := NewRouter() logrus.Info("Purser server started on port `localhost:3030`") logrus.Fatal(http.ListenAndServe(":3030", handlers.CORS(allowedOrigins, allowedCredentials)(router))) } ================================================ FILE: cmd/controller/api/apiHandlers/authenticationHandlers.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package apiHandlers import ( "encoding/json" "encoding/gob" "net/http" "github.com/Sirupsen/logrus" "github.com/gorilla/sessions" "github.com/gorilla/securecookie" "github.com/vmware/purser/pkg/controller/dgraph/models/query" ) // Credentials structure type Credentials struct { Password string `json:"password"` Username string `json:"username"` NewPassword string `json:"newPassword"` } // User structure type User struct { Username string Authenticated bool } var cookieName = "purser-session-token" var store *sessions.CookieStore // initialises cookie store func init() { authKeyOne := securecookie.GenerateRandomKey(64) encryptionKeyOne := securecookie.GenerateRandomKey(32) store = sessions.NewCookieStore( authKeyOne, encryptionKeyOne, ) store.Options = &sessions.Options{ MaxAge: 60 * 15, HttpOnly: true, } gob.Register(User{}) } // LoginUser listens on /auth/login endpoint func LoginUser(w http.ResponseWriter, r *http.Request) { addAccessControlHeaders(&w, r) var cred Credentials err := json.NewDecoder(r.Body).Decode(&cred) if err != nil { w.WriteHeader(http.StatusBadRequest) return } if !query.Authenticate(cred.Username, cred.Password) { logrus.Errorf("wrong credentials") w.WriteHeader(http.StatusUnauthorized) return } session, err := store.Get(r, cookieName) if err != nil { logrus.Errorf("unable to get session from cookie store, err: %v", err) w.WriteHeader(http.StatusInternalServerError) return } session.Values["user"] = User{ Username: cred.Username, Authenticated: true, } err = session.Save(r, w) if err != nil { logrus.Errorf("unable to get session from cookie store, err: %v", err) w.WriteHeader(http.StatusInternalServerError) return } logrus.Infof("login success") w.WriteHeader(http.StatusOK) } // LogoutUser listens on /auth/logout endpoint func LogoutUser(w http.ResponseWriter, r *http.Request) { addAccessControlHeaders(&w, r) session, err := store.Get(r, cookieName) if err != nil { logrus.Errorf("unable to get session from cookie store, err: %v", err) w.WriteHeader(http.StatusInternalServerError) return } session.Values["user"] = User{} session.Options.MaxAge = -1 err = session.Save(r, w) if err != nil { logrus.Errorf("unable to get session from cookie store, err: %v", err) w.WriteHeader(http.StatusInternalServerError) return } http.Redirect(w, r, "/", http.StatusFound) } // TODO: Enhance func isUserAuthenticated(w http.ResponseWriter, r *http.Request) bool { return true } // ChangePassword listens on /auth/changePassword endpoint func ChangePassword(w http.ResponseWriter, r *http.Request) { addAccessControlHeaders(&w, r) var cred Credentials err := json.NewDecoder(r.Body).Decode(&cred) if err != nil { w.WriteHeader(http.StatusBadRequest) return } if !query.UpdatePassword(cred.Username, cred.Password, cred.NewPassword) { w.WriteHeader(http.StatusUnauthorized) return } } ================================================ FILE: cmd/controller/api/apiHandlers/customGroupHandlers.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package apiHandlers import ( "encoding/json" "github.com/Sirupsen/logrus" group_v1 "github.com/vmware/purser/pkg/apis/groups/v1" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/vmware/purser/pkg/controller/dgraph/models/query" "github.com/vmware/purser/pkg/controller/eventprocessor" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net/http" ) // GetGroupsData listens on /api/groups endpoint func GetGroupsData(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) groupsData, err := query.RetrieveGroupsData() if err != nil { logrus.Errorf("unable to retrieve groups data from dgraph, %v", err) } else { encodeAndWrite(w, groupsData) } } } // DeleteGroup listens on /api/group/delete func DeleteGroup(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addAccessControlHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var err error if name, isName := queryParams[query.Name]; isName { err = getGroupClient().Delete(name[0], &meta_v1.DeleteOptions{}) if err == nil { w.WriteHeader(http.StatusOK) models.DeleteGroup(name[0]) return } } logrus.Errorf("unable to delete: query params: %v, err: %v", queryParams, err) http.Error(w, err.Error(), http.StatusBadRequest) } } // CreateGroup listens on /api/group/create func CreateGroup(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addAccessControlHeaders(&w, r) groupData, err := convertRequestBodyToJSON(r) if err != nil { logrus.Errorf("unable to parse request as either JSON or YAML, err: %v", err) http.Error(w, err.Error(), http.StatusBadRequest) return } newGroup := group_v1.Group{} if jsonErr := json.Unmarshal(groupData, &newGroup); jsonErr != nil { logrus.Errorf("unable to parse object as group, err: %v", jsonErr) http.Error(w, jsonErr.Error(), http.StatusBadRequest) return } if _, groupErr := getGroupClient().Create(&newGroup); groupErr != nil { logrus.Errorf("unable to create group: %v", groupErr) http.Error(w, groupErr.Error(), http.StatusBadRequest) return } w.WriteHeader(http.StatusOK) eventprocessor.UpdateGroup(&newGroup, getGroupClient()) } } ================================================ FILE: cmd/controller/api/apiHandlers/helpers.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package apiHandlers import ( "encoding/json" "github.com/Sirupsen/logrus" "io" "io/ioutil" "k8s.io/apimachinery/pkg/util/yaml" "net/http" "github.com/vmware/purser/pkg/controller" "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" "k8s.io/client-go/kubernetes" ) var groupClient *v1.GroupClient var kubeClient *kubernetes.Clientset func addHeaders(w *http.ResponseWriter, r *http.Request) { addAccessControlHeaders(w, r) (*w).Header().Set("Content-Type", "application/json; charset=UTF-8") (*w).WriteHeader(http.StatusOK) } func addAccessControlHeaders(w *http.ResponseWriter, r *http.Request) { (*w).Header().Set("Access-Control-Allow-Origin", r.Header.Get("Origin")) (*w).Header().Set("Access-Control-Allow-Credentials", "true") } func writeBytes(w io.Writer, data []byte) { _, err := w.Write(data) if err != nil { logrus.Errorf("Unable to encode to json: (%v)", err) } } func encodeAndWrite(w io.Writer, obj interface{}) { err := json.NewEncoder(w).Encode(obj) if err != nil { logrus.Errorf("Unable to encode to json: (%v)", err) } } func convertRequestBodyToJSON(r *http.Request) ([]byte, error) { requestData, err := ioutil.ReadAll(r.Body) if err != nil { return nil, err } groupData, err := yaml.ToJSON(requestData) return groupData, err } // SetKubeClientAndGroupClient sets groupcrd client func SetKubeClientAndGroupClient(conf controller.Config) { groupClient = conf.Groupcrdclient kubeClient = conf.Kubeclient } func getGroupClient() *v1.GroupClient { return groupClient } func getKubeClient() *kubernetes.Clientset { return kubeClient } ================================================ FILE: cmd/controller/api/apiHandlers/hierarchyAndMetricAPIHandlers.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package apiHandlers import ( "encoding/json" "fmt" "net/http" "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/vmware/purser/pkg/controller/dgraph/models/query" "github.com/vmware/purser/pkg/controller/discovery/generator" "github.com/vmware/purser/pkg/controller/eventprocessor" ) // GetHomePage is the default api home page func GetHomePage(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { _, err := fmt.Fprintf(w, "Welcome to the Purser!") if err != nil { logrus.Errorf("Unable to write welcome message to Homepage: (%v)", err) } } } // GetPodInteractions listens on /interactions/pod endpoint and returns pod interactions func GetPodInteractions(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonResp []byte if name, isName := queryParams[query.Name]; isName { jsonResp = query.RetrievePodsInteractions(name[0], false) } else { if orphanVal, isOrphan := queryParams[query.Orphan]; isOrphan && orphanVal[0] == query.False { jsonResp = query.RetrievePodsInteractions(query.All, false) } else { jsonResp = query.RetrievePodsInteractions(query.All, true) } } writeBytes(w, jsonResp) } } // GetClusterHierarchy listens on /hierarchy endpoint and returns all namespaces(or nodes and PV) in the cluster func GetClusterHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if view, isView := queryParams[query.View]; isView && view[0] == query.Physical { jsonData = query.RetrieveClusterHierarchy(query.Physical) } else { jsonData = query.RetrieveClusterHierarchy(query.Logical) } encodeAndWrite(w, jsonData) } } // GetNamespaceHierarchy listens on /hierarchy/namespace endpoint and returns all children of namespace func GetNamespaceHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.NamespaceCheck, Type: query.NamespaceType, Name: name[0], ChildFilter: query.NamespaceChildFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { jsonData = query.RetrieveClusterHierarchy(query.Logical) } encodeAndWrite(w, jsonData) } } // GetDeploymentHierarchy listens on /hierarchy/deployment endpoint and returns all children of deployment func GetDeploymentHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.DeploymentCheck, Type: query.DeploymentType, Name: name[0], ChildFilter: query.IsReplicasetFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for deployment, no name is given") } encodeAndWrite(w, jsonData) } } // GetReplicasetHierarchy listens on /hierarchy/replicaset endpoint and returns all children of replicaset func GetReplicasetHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.ReplicasetCheck, Type: query.ReplicasetType, Name: name[0], ChildFilter: query.IsPodFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for replicaset, no name is given") } encodeAndWrite(w, jsonData) } } // GetStatefulsetHierarchy listens on /hierarchy/statefulset endpoint and returns all children of statefulset func GetStatefulsetHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.StatefulsetCheck, Type: query.StatefulsetType, Name: name[0], ChildFilter: query.IsPodFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for statefulset, no name is given") } encodeAndWrite(w, jsonData) } } // GetPodHierarchy listens on /hierarchy/pod endpoint and returns all children of pod func GetPodHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.PodCheck, Type: query.PodType, Name: name[0], ChildFilter: query.IsContainerFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for pod, no name is given") } encodeAndWrite(w, jsonData) } } // GetContainerHierarchy listens on /hierarchy/container endpoint and returns all children of container func GetContainerHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.ContainerCheck, Type: query.ContainerType, Name: name[0], ChildFilter: query.IsProcFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for container, no name is given") } encodeAndWrite(w, jsonData) } } // GetEmptyHierarchy listens on /hierarchy/process and /hierarchy/pvc endpoint and returns empty data func GetEmptyHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper encodeAndWrite(w, jsonData) } } // GetNodeHierarchy listens on /hierarchy/node endpoint and returns all children of node func GetNodeHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.NodeCheck, Type: query.NodeType, Name: name[0], ChildFilter: query.IsPodFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for node, no name is given") } encodeAndWrite(w, jsonData) } } // GetPVHierarchy listens on /hierarchy/pv endpoint and returns all children of PV func GetPVHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.PVCheck, Type: query.PVType, Name: name[0], ChildFilter: query.IsPVCFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for PV, no name is given") } encodeAndWrite(w, jsonData) } } // GetDaemonsetHierarchy listens on /hierarchy/daemonset endpoint and returns all children of Daemonset func GetDaemonsetHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.DaemonsetCheck, Type: query.DaemonsetType, Name: name[0], ChildFilter: query.IsPodFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for Daemonset, no name is given") } encodeAndWrite(w, jsonData) } } // GetJobHierarchy listens on /hierarchy/job endpoint and returns all children of Job func GetJobHierarchy(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.JobCheck, Type: query.JobType, Name: name[0], ChildFilter: query.IsPodFilter, } jsonData = resourceQuery.RetrieveResourceHierarchy() } else { logrus.Errorf("wrong type of query for Job, no name is given") } encodeAndWrite(w, jsonData) } } // GetClusterMetrics listens on /metrics endpoint with option for view(physical or logical) func GetClusterMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if view, isView := queryParams[query.View]; isView && view[0] == query.Physical { jsonData = query.RetrieveClusterMetrics(query.Physical) } else { jsonData = query.RetrieveClusterMetrics(query.Logical) } query.PopulateClusterAllocationAndCapacity(&jsonData) encodeAndWrite(w, jsonData) } } // GetNamespaceMetrics listens on /metrics/namespace func GetNamespaceMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.NamespaceCheck, Type: query.NamespaceType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() } else { jsonData = query.RetrieveClusterMetrics(query.Logical) } query.PopulateClusterAllocationAndCapacity(&jsonData) encodeAndWrite(w, jsonData) } } // GetDeploymentMetrics listens on /metrics/deployment func GetDeploymentMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.DeploymentCheck, Type: query.DeploymentType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for deployment, no name is given") } encodeAndWrite(w, jsonData) } } // GetDaemonsetMetrics listens on /metrics/daemonset func GetDaemonsetMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.DaemonsetCheck, Type: query.DaemonsetType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for daemonset, no name is given") } encodeAndWrite(w, jsonData) } } // GetJobMetrics listens on /metrics/job func GetJobMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.JobCheck, Type: query.JobType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for job, no name is given") } encodeAndWrite(w, jsonData) } } // GetStatefulsetMetrics listens on /metrics/statefulset func GetStatefulsetMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.StatefulsetCheck, Type: query.StatefulsetType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for statefulset, no name is given") } encodeAndWrite(w, jsonData) } } // GetReplicasetMetrics listens on /metrics/replicaset func GetReplicasetMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.ReplicasetCheck, Type: query.ReplicasetType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for statefulset, no name is given") } encodeAndWrite(w, jsonData) } } // GetNodeMetrics listens on /metrics/node func GetNodeMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.NodeCheck, Type: query.NodeType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() resourceQuery.PopulateNodeOrPVAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for node, no name is given") } encodeAndWrite(w, jsonData) } } // GetPodMetrics listens on /metrics/pod func GetPodMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.PodCheck, Type: query.PodType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for pod, no name is given") } encodeAndWrite(w, jsonData) } } // GetContainerMetrics listens on /metrics/container func GetContainerMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.ContainerCheck, Type: query.ContainerType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for container, no name is given") } encodeAndWrite(w, jsonData) } } // GetPVMetrics listens on /metrics/pv func GetPVMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.PVCheck, Type: query.PVType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() resourceQuery.PopulateNodeOrPVAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for PV, no name is given") } encodeAndWrite(w, jsonData) } } // GetPVCMetrics listens on /metrics/pvc func GetPVCMetrics(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { addHeaders(&w, r) queryParams := r.URL.Query() logrus.Debugf("Query params: (%v)", queryParams) var jsonData query.JSONDataWrapper if name, isName := queryParams[query.Name]; isName { resourceQuery := query.Resource{ Check: query.PVCCheck, Type: query.PVCType, Name: name[0], } jsonData = resourceQuery.RetrieveResourceMetrics() query.PopulateClusterAllocationAndCapacity(&jsonData) } else { logrus.Errorf("wrong type of query for PVC, no name is given") } encodeAndWrite(w, jsonData) } } // GetPodDiscoveryNodes listens on /discovery/pod/nodes endpoint func GetPodDiscoveryNodes(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { var pods []models.Pod var err error addHeaders(&w, r) pods, err = query.RetrievePodsInteractionsForAllLivePodsWithCount() generator.GeneratePodNodesAndEdges(pods) if err != nil { logrus.Errorf("Unable to get response: (%v)", err) } nodes := generator.GetGraphNodes() if nodes != nil { logrus.Infof("No nodes found") return } err = json.NewEncoder(w).Encode(nodes) if err != nil { logrus.Errorf("Unable to encode to json: (%v)", err) } } } // GetPodDiscoveryEdges listens on /discovery/pod/edges endpoint func GetPodDiscoveryEdges(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { var err error addHeaders(&w, r) edges := generator.GetGraphEdges() if edges == nil { logrus.Infof("No edges found") return } err = json.NewEncoder(w).Encode(edges) if err != nil { logrus.Errorf("Unable to encode to json: (%v)", err) } } } // SyncCluster listens on /api/sync func SyncCluster(w http.ResponseWriter, r *http.Request) { if isUserAuthenticated(w, r) { w.WriteHeader(http.StatusAccepted) go syncResourcesInCluster() } } func syncResourcesInCluster() { eventprocessor.SyncCluster(getKubeClient()) eventprocessor.UpdateGroups(getGroupClient()) query.ComputeClusterAllocationAndCapacity() } ================================================ FILE: cmd/controller/api/logger.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package api import ( "net/http" "time" "github.com/Sirupsen/logrus" ) // Logger implements web logging logic func Logger(inner http.Handler, name string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() inner.ServeHTTP(w, r) logrus.Infof( "%s\t%s\t%s\t%s", r.Method, r.RequestURI, name, time.Since(start), ) }) } ================================================ FILE: cmd/controller/api/router.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package api import ( "github.com/gorilla/mux" ) // NewRouter returns a new instance of the router func NewRouter() *mux.Router { router := mux.NewRouter().StrictSlash(true) for _, route := range routes { handlerFunc := route.HandlerFunc handler := Logger(handlerFunc, route.Name) router. Methods(route.Method). Path(route.Pattern). Name(route.Name). Handler(handler) } return router } ================================================ FILE: cmd/controller/api/routes.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package api import ( "github.com/vmware/purser/cmd/controller/api/apiHandlers" "net/http" ) // Route structure type Route struct { Name string Method string Pattern string HandlerFunc http.HandlerFunc } // Routes list type Routes []Route var routes = Routes{ Route{ "GetHomePage", "GET", "/api", apiHandlers.GetHomePage, }, Route{ "GetPodInteractions", "GET", "/api/interactions/pod", apiHandlers.GetPodInteractions, }, Route{ "GetClusterHierarchy", "GET", "/api/hierarchy", apiHandlers.GetClusterHierarchy, }, Route{ "GetNamespaceHierarchy", "GET", "/api/hierarchy/namespace", apiHandlers.GetNamespaceHierarchy, }, Route{ "GetDeploymentHierarchy", "GET", "/api/hierarchy/deployment", apiHandlers.GetDeploymentHierarchy, }, Route{ "GetReplicasetHierarchy", "GET", "/api/hierarchy/replicaset", apiHandlers.GetReplicasetHierarchy, }, Route{ "GetStatefulsetHierarchy", "GET", "/api/hierarchy/statefulset", apiHandlers.GetStatefulsetHierarchy, }, Route{ "GetPodHierarchy", "GET", "/api/hierarchy/pod", apiHandlers.GetPodHierarchy, }, Route{ "GetContainerHierarchy", "GET", "/api/hierarchy/container", apiHandlers.GetContainerHierarchy, }, Route{ "GetProcessHierarchy", "GET", "/api/hierarchy/process", apiHandlers.GetEmptyHierarchy, }, Route{ "GetNodeHierarchy", "GET", "/api/hierarchy/node", apiHandlers.GetNodeHierarchy, }, Route{ "GetPVHierarchy", "GET", "/api/hierarchy/pv", apiHandlers.GetPVHierarchy, }, Route{ "GetPVCHierarchy", "GET", "/api/hierarchy/pvc", apiHandlers.GetEmptyHierarchy, }, Route{ "GetDaemonsetHierarchy", "GET", "/api/hierarchy/daemonset", apiHandlers.GetDaemonsetHierarchy, }, Route{ "GetJobHierarchy", "GET", "/api/hierarchy/job", apiHandlers.GetJobHierarchy, }, Route{ "GetClusterMetrics", "GET", "/api/metrics", apiHandlers.GetClusterMetrics, }, Route{ "GetNamespaceMetrics", "GET", "/api/metrics/namespace", apiHandlers.GetNamespaceMetrics, }, Route{ "GetDeploymentMetrics", "GET", "/api/metrics/deployment", apiHandlers.GetDeploymentMetrics, }, Route{ "GetDaemonsetMetrics", "GET", "/api/metrics/daemonset", apiHandlers.GetDaemonsetMetrics, }, Route{ "GetJobMetrics", "GET", "/api/metrics/job", apiHandlers.GetJobMetrics, }, Route{ "GetStatefulsetMetrics", "GET", "/api/metrics/statefulset", apiHandlers.GetStatefulsetMetrics, }, Route{ "GetReplicasetMetrics", "GET", "/api/metrics/replicaset", apiHandlers.GetReplicasetMetrics, }, Route{ "GetNodeMetrics", "GET", "/api/metrics/node", apiHandlers.GetNodeMetrics, }, Route{ "GetPodMetrics", "GET", "/api/metrics/pod", apiHandlers.GetPodMetrics, }, Route{ "GetContainerMetrics", "GET", "/api/metrics/container", apiHandlers.GetContainerMetrics, }, Route{ "GetPVMetrics", "GET", "/api/metrics/pv", apiHandlers.GetPVMetrics, }, Route{ "GetPVCMetrics", "GET", "/api/metrics/pvc", apiHandlers.GetPVCMetrics, }, Route{ "GetPodDiscoveryNodes", "GET", "/api/nodes", apiHandlers.GetPodDiscoveryNodes, }, Route{ "GetPodDiscoveryEdges", "GET", "/api/edges", apiHandlers.GetPodDiscoveryEdges, }, Route{ "GetGroupsData", "GET", "/api/groups", apiHandlers.GetGroupsData, }, Route{ "Login", "POST", "/auth/login", apiHandlers.LoginUser, }, Route{ "Logout", "POST", "/auth/logout", apiHandlers.LogoutUser, }, Route{ "ChangePassword", "POST", "/auth/changePassword", apiHandlers.ChangePassword, }, Route{ "DeleteGroup", "POST", "/api/group/delete", apiHandlers.DeleteGroup, }, Route{ "CreateGroup", "POST", "/api/group/create", apiHandlers.CreateGroup, }, Route{ "SyncCluster", "GET", "/api/sync", apiHandlers.SyncCluster, }, } ================================================ FILE: cmd/controller/config/config.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package config import ( "sync" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/client" group_client "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" subscriber_client "github.com/vmware/purser/pkg/client/clientset/typed/subscriber/v1" "github.com/vmware/purser/pkg/controller" "github.com/vmware/purser/pkg/controller/buffering" "github.com/vmware/purser/pkg/utils" ) // Setup initialzes the controller configuration func Setup(conf *controller.Config, kubeconfig string) { var err error *conf = controller.Config{} conf.KubeConfig, err = utils.GetKubeconfig(kubeconfig) if err != nil { log.Fatal(err) } conf.Kubeclient = utils.GetKubeclient(conf.KubeConfig) conf.Resource = controller.Resource{ Pod: true, Node: true, PersistentVolume: true, PersistentVolumeClaim: true, ReplicaSet: true, Deployment: true, StatefulSet: true, DaemonSet: true, Job: true, Service: true, Namespace: true, Group: true, Subscriber: true, } conf.RingBuffer = &buffering.RingBuffer{Size: buffering.BufferSize, Mutex: &sync.Mutex{}} clientset, clusterConfig := client.GetAPIExtensionClient(kubeconfig) conf.Groupcrdclient = group_client.NewGroupClient(clientset, clusterConfig) conf.Subscriberclient = subscriber_client.NewSubscriberClient(clientset, clusterConfig) } ================================================ FILE: cmd/controller/purserctrl.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "flag" "time" "github.com/vmware/purser/pkg/controller/dgraph/models/query" "github.com/vmware/purser/pkg/pricing" log "github.com/Sirupsen/logrus" "github.com/robfig/cron" "github.com/vmware/purser/cmd/controller/api" "github.com/vmware/purser/cmd/controller/config" "github.com/vmware/purser/pkg/controller" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/discovery/processor" "github.com/vmware/purser/pkg/controller/eventprocessor" "github.com/vmware/purser/pkg/utils" ) var conf controller.Config // InClusterConfigPath should be empty to get client and config for InCluster environment. const InClusterConfigPath = "" var interactions *string func init() { logLevel := flag.String("log", "info", "set log level as info or debug") dgraphURL := flag.String("dgraphURL", "purser-db", "dgraph zero url") dgraphPort := flag.String("dgraphPort", "9080", "dgraph zero port") interactions = flag.String("interactions", "disable", "enable discovery of interactions") kubeconfig := flag.String("kubeconfig", InClusterConfigPath, "path to the kubeconfig file") flag.Parse() utils.InitializeLogger(*logLevel) config.Setup(&conf, *kubeconfig) // start dgraph and create login if not exists dgraph.Start(*dgraphURL, *dgraphPort) dgraph.StoreLogin() } func main() { go api.StartServer(conf) go startCronJobForPopulatingRateCard() time.Sleep(time.Minute * 3) go eventprocessor.ProcessEvents(&conf) if *interactions == "enable" { go startInteractionsDiscovery() } go startCronJobForUpdatingCustomGroups() controller.Start(&conf) } // starts first discovery after 5 min of controller starting. Next runs will occur in every 59 min func startInteractionsDiscovery() { time.Sleep(time.Minute * 5) runDiscovery() c := cron.New() err := c.AddFunc("@every 0h59m", runDiscovery) if err != nil { log.Error(err) } err = c.AddFunc("@daily", dgraph.RemoveResourcesInactive) if err != nil { log.Error(err) } c.Start() } func runDiscovery() { processor.ProcessPodInteractions(conf) processor.ProcessServiceInteractions(conf) } func startCronJobForUpdatingCustomGroups() { query.ComputeClusterAllocationAndCapacity() runGroupUpdate() c := cron.New() err := c.AddFunc("@every 0h5m", runGroupUpdate) if err != nil { log.Error(err) } err = c.AddFunc("@every 0h5m", query.ComputeClusterAllocationAndCapacity) if err != nil { log.Error(err) } c.Start() } func runGroupUpdate() { eventprocessor.UpdateGroups(conf.Groupcrdclient) } func startCronJobForPopulatingRateCard() { cloud := &pricing.Cloud{Kubeclient: conf.Kubeclient} // find cloud provider and region cloud.CloudProvider, cloud.Region = pricing.GetClusterProviderAndRegion() cloud.PopulateRateCard() c := cron.New() err := c.AddFunc("@every 168h", cloud.PopulateRateCard) if err != nil { log.Error(err) } c.Start() } ================================================ FILE: cmd/plugin/purser.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "flag" "fmt" "os" "strings" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/client" groups_client_v1 "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" "github.com/vmware/purser/pkg/plugin" "github.com/vmware/purser/pkg/utils" ) const ( pluginVersion = "version v1.0.0" ) var ( groupClient *groups_client_v1.GroupClient // Variables used for cmd interface kubeconfig string info string version string description = fmt.Sprintf("Purser gives cost insights of kubernetes deployments.\n\n") usage = fmt.Sprintf("Usage:\n kubectl plugin purser [options] \n\n") supportedCmds = fmt.Sprintf("The supported commands are:\n get Get resource information.\n set Set resource information.\n\n") optionHelp = fmt.Sprintf("\n --info Show more details about the plugin.") optionKubeConfig = fmt.Sprintf("\n --kubeconfig Absolute path for the kube config file.") optionVersion = fmt.Sprintf("\n --version Show plugin version.") options = fmt.Sprintf("options:%s%s%s\n\n", optionHelp, optionKubeConfig, optionVersion) kubecltOption = fmt.Sprintf("\nUse \"kubectl options\" for a list of global command-line options (applies to all commands).\n\n") ) func init() { flag.StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECTL_PLUGINS_GLOBAL_FLAG_KUBECONFIG"), "path to Kubernetes config file") flag.StringVar(&info, "info", os.Getenv("KUBECTL_PLUGINS_LOCAL_FLAG_INFO"), "Show help documentation") flag.StringVar(&version, "version", os.Getenv("KUBECTL_PLUGINS_LOCAL_FLAG_VERSION"), "Show version number") flag.Usage = func() { _, err := fmt.Fprint(flag.CommandLine.Output(), description) if err != nil { log.Fatal(err) } _, err = fmt.Fprint(flag.CommandLine.Output(), usage) if err != nil { log.Fatal(err) } _, err = fmt.Fprint(flag.CommandLine.Output(), supportedCmds) if err != nil { log.Fatal(err) } _, err = fmt.Fprint(flag.CommandLine.Output(), options) if err != nil { log.Fatal(err) } _, err = fmt.Fprint(flag.CommandLine.Output(), "Example(s):\n\n") if err != nil { log.Fatal(err) } printHelp() _, err = fmt.Fprint(flag.CommandLine.Output(), kubecltOption) if err != nil { log.Fatal(err) } } if version != "" { fmt.Println(pluginVersion) os.Exit(0) } if info != "" { flag.Usage() os.Exit(0) } config, err := utils.GetKubeconfig(kubeconfig) if err != nil { log.Fatal(err) } plugin.ProvideClientSetInstance(utils.GetKubeclient(config)) client, clusterConfig := client.GetAPIExtensionClient(kubeconfig) groupClient = groups_client_v1.NewGroupClient(client, clusterConfig) } func main() { inputs := os.Args[2:] // index 1 is empty if len(inputs) == 4 && inputs[0] == Get { computeMetricInsight(inputs) } else if len(inputs) == 2 { computeStats(inputs) } else { printHelp() } } func computeMetricInsight(inputs []string) { switch inputs[1] { case Cost: computeCost(inputs) case Resources: fetchResource(inputs) } } func computeCost(inputs []string) { switch inputs[2] { case Label: plugin.GetPodsCostForLabel(inputs[3]) case Pod: plugin.GetPodCost(inputs[3]) case Node: plugin.GetAllNodesCost() default: printHelp() } } func fetchResource(inputs []string) { switch inputs[2] { case Namespace: group := plugin.GetGroupByName(groupClient, inputs[3]) if group != nil { plugin.PrintGroup(group) } else { fmt.Printf("Group %s is not present\n", inputs[3]) } case Label: if !strings.Contains(inputs[3], "=") { printHelp() } group := plugin.GetGroupByName(groupClient, createGroupNameFromLabel(inputs[3])) if group != nil { plugin.PrintGroup(group) } else { fmt.Printf("Group %s is not present\n", inputs[3]) } case Group: group := plugin.GetGroupByName(groupClient, inputs[3]) if group != nil { plugin.PrintGroup(group) } else { fmt.Printf("No group with name: %s\n", inputs[3]) } default: printHelp() } } func createGroupNameFromLabel(input string) string { inp := strings.Split(input, "=") key, val := inp[0], inp[1] groupName := key + "." + val if strings.Contains(groupName, "/") { groupName = strings.Replace(groupName, "/", "-", -1) } return strings.ToLower(groupName) } func computeStats(inputs []string) { switch inputs[0] { case Get: getStats(inputs) case Set: inputUserCosts(inputs) default: printHelp() } } func getStats(inputs []string) { switch inputs[1] { case "summary": plugin.GetClusterSummary() case "savings": plugin.GetSavings() case "user-costs": price := plugin.GetUserCosts() fmt.Printf("cpu cost per CPU per hour:\t %f$\nmem cost per GB per hour:\t %f$\nstorage cost per GB per hour:\t %f$\n", price.CPU, price.Memory, price.Storage) default: printHelp() } } func inputUserCosts(inputs []string) { if inputs[1] == "user-costs" { fmt.Printf("Enter CPU cost per cpu per hour:\t ") var cpuCostPerCPUPerHour string _, err := fmt.Scan(&cpuCostPerCPUPerHour) logError(err) fmt.Printf("Enter Memory cost per GB per hour:\t ") var memCostPerGBPerHour string _, err = fmt.Scan(&memCostPerGBPerHour) logError(err) fmt.Printf("Enter Storage cost per GB per hour:\t ") var storageCostPerGBPerHour string _, err = fmt.Scan(&storageCostPerGBPerHour) logError(err) plugin.SaveUserCosts(cpuCostPerCPUPerHour, memCostPerGBPerHour, storageCostPerGBPerHour) } else { printHelp() } } func printHelp() { pluginExt := "kubectl --kubeconfig= plugin purser " fmt.Println("Try one of the following commands...") fmt.Println(pluginExt + "get summary") fmt.Println(pluginExt + "get resources group ") fmt.Println(pluginExt + "get cost label ") fmt.Println(pluginExt + "get cost pod ") fmt.Println(pluginExt + "get cost node all") fmt.Println(pluginExt + "set user-costs") fmt.Println(pluginExt + "get user-costs") fmt.Println(pluginExt + "get savings") } func logError(err error) { if err != nil { log.Printf("failed to read user input %+v", err) } } ================================================ FILE: cmd/plugin/types.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main // These are possible actions for resources const ( Get = "get" Set = "set" ) // These are kubernetes components const ( Label = "label" Pod = "pod" Node = "node" Namespace = "namespace" Group = "group" ) // These are utilisation metrics const ( Cost = "cost" Resources = "resources" ) ================================================ FILE: docs/architecture.md ================================================ # Architecture of Purser The following diagram represents the architecture of Purser. ![Architecture](/docs/img/architecture.png) The following are the main componenets installed in Kubernetes for Purser. 1. **Kubernetes API Server** All the Purser `kubectl` commands hit the API server extension. These APIs understand the input command, compute and return the required output. 2. **Custom Controller** The custom controller watches for changes in state of pods, nodes, persistent volumes, etc. and update the inventory in CRDs. 3. **Custom Resource Definitions(CRDs)** Custom Resource Definitions are like any other resource(Pod, Node, etc.) and store the config data like `Group Definitions` and inventory. 4. **Metric Store** Metric store is used to store the utilization, allocation metrics of inventory and also calculated costs. 5. **CRON Job** CRON Job collects the stats of inventory and calculates the cost periodically and stores in Metric Store. ## Work Flow 1. Purser installation steps create Custom Controller, CRON Job and CRDs in Kubernetes. 2. Once installed the custom controller collects all the inventory(pods, nodes, pv, etc.) and stores in CRDs, later it watches for any changes in inventory and stores the changes in CRDs. 3. CRON Job kicks in periodically and collect the stats and stores the stats in metric store. CRON Job also calculates the Costs in the same cycle and stores them in the metric store. 4. Any `kubectl` command invocations are received by Kubernetes API server extension. APIs then process the required output based on the configurations(for groups), inventory, costs metrics and returns to the user. ================================================ FILE: docs/custom-group-installation-and-usage.md ================================================ # Custom Group Installation and Usage To get resource and cost visibility for a particular set of pods Purser allows user to create custom logical group. User can define the label filter logic(`AND of ORs`: Conjunctive normal form) while creating the logical group i.e, pods satisfying these conditions will belong to this custom group. ## Installing logical group definition and an example logical group To install the logical group definition into your cluster, download [purser-group-crd.yaml](../cluster/artifacts/purser-group-crd.yaml) yaml i.e, ```yaml apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: groups.vmware.purser.com spec: group: vmware.purser.com names: kind: Group listKind: GroupList plural: groups singular: group scope: Namespaced version: v1 status: acceptedNames: kind: Group listKind: GroupList plural: groups singular: group ``` and use kubectl to install this definition ```bash kubectl create -f purser-group-crd.yaml ``` _**NOTE:** This installation is needed only once per cluster_ **Installing an example logical group** Download [example-group.yaml](../cluster/artifacts/example-group.yaml) yaml i.e, ```yaml apiVersion: vmware.purser.com/v1 kind: Group metadata: name: example-group spec: name: example-group labels: expr1: app: - sample-app - sample-app2 env: - dev expr2: namespace: - ns1 - ns2 expr3: key1: - val1 key2: - val2 ``` and use kubectl to create this logical group ```bash kubectl create -f example-group.yaml kubectl get groups.vmware.purser.com ``` This will create a custom logical group with name `example-group` of type `groups.vmware.purser.com`. The label filter (used to fetch pods belonging to this group) for `example-group` will be ```yaml (app=sampl-app OR app=sample-app2 OR env=dev) AND (namespace=ns1 OR namespace=ns2) AND (key1=val1 OR key2=val2) ``` In general the syntax purser supports is: ``` expr1 AND expr2 AND expr3 AND ... where each expr is of form key1:value1 OR key2:value2 OR key1:value3 OR ... ``` ## Usage For resource and cost visibility into this newly created logical group run the following command ```bash kubectl plugin purser get resources group example-group ``` _Refer [purser installation](../README.md#installation) to install purser controller and plugin_ ## Uninstalling purser custom group To uninstall purser custom group run the following command ```bash kubectl delete -f purser-group-crd.yaml ``` where [purser-group-crd.yaml](../cluster/artifacts/purser-group-crd.yaml) is same file that you downloaded during installation. ================================================ FILE: docs/design/pricing.md ================================================ # Pricing in Purser ## User defined pricing Currently purser supports user defined pricing for cpu, memory and storage resources per hour. The default pricing for the resources are * CPU: 0.024$ per vCPU per Hour * Memory: 0.01$ per GB per Hour * Storage: 0.00013888888$ per GB per Hour These default pricing for cpu and memory have been set by taking average [prices](https://aws.amazon.com/ec2/pricing/on-demand/) in AWS ec2 instances. For storage we set pricing proportional to 0.1$ per GB per month referring to AWS [ebs pricing](https://aws.amazon.com/ebs/pricing/). User can edit these pricing using purser plugin: `kubectl plugin purser set user-costs` _(Future work)_ Option to edit these default pricing in UI. ## Using node labels for accurate pricing (WIP) Data needed to get correct pricing of node: * Cloud Provider (AWS, GCE etc) * Region (us-east etc) * Machine Type (t2.micro, m4.large) * Operating system (linux, windows etc) * Rate card which gives cost of node depending on above data * _(Future work)_ Costing based on instance type i.e., "Is the instance On-demand or Spot-Instance or Reserved-Instance etc?" * _(Future work)_ Discounts ### Getting cloud provider, region and machine type Kubelet populates few [reserved labels](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#beta-kubernetes-io-instance-type) on nodes. Using these labels we can determine region, machineType and operating system. Command `kubectl describe node ` gives labels. * Default assume cloud provider as aws. Check section [Finding Cloud Provider](#finding-cloud-provider). * Label `beta.kubernetes.io/instance-type=m4.10xlarg` gives machine type. Here for this example machineType is m4.10xlarge * Label `beta.kubernetes.io/os=linux` gives operating system. Here os is linux * Label `failure-domain.beta.kubernetes.io/region=us-west-1` gives region. Here region is us-west-1 _Note: kubelet will not set these reserved labels if the cluster is not using cloud provider._ If any of the required labels is not available then we should fall back to default pricing. #### Storage Volume Pricing `kubectl get pv` gives us storage class(ex: gp2, my-storage-class etc) for each volume. Further using command `kubectl describe storageclass ` will give output in which there will be a field `Parameters` containing labels for `type` of storage (Ex: gp2) and `zone` (Ex: us-west-1c). _Parameters_ field may not contain _zone_ label. In such case we can get region from `kubectl describe pv ` using label `failure-domain.beta.kubernetes.io/region`. If _type_ label is also not present then we should fall back to default pricing. ### Finding Cloud Provider While initiating a cluster either by kubeadm or kops or other kubernetes installers the user will set cloud-provider, if it isn't set kubernetes assumes that cluster is being deployed on bare metal. Further when a new node is created `.spec.providerID` will be set based (by _kubelet_) on cloud-provider. The value of `providerId` will be `(providerName + "://" + instanceID)`. As we need `providerName` (aws, azure etc) we can get it from `providerID`. If getting cloud provider name fails we should fallback and assume default prices for aws. *Example cluster on aws: --cloud-provider=aws command-line flag is needed (to successfully register the node with cloud provider) to be present for the API server, controller manager, and every kubelet in the cluster. References: * kubeadm: https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/ * providerID value: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/cloud-provider/cloud.go#L94 * Cluster on aws: https://blog.heptio.com/setting-up-the-kubernetes-aws-cloud-provider-6f0349b512bd * More on providerID: https://blog.scottlowe.org/2018/09/28/setting-up-the-kubernetes-aws-cloud-provider/ *Note: All above kubectl commands will have corresponding methods in kubernetes client-go ### Populating Rate Card #### Design * Identify cloud provider, region of the cluster * Embed the crawler code in purser and run cloud specific crawler to fetch the rate cards. * Populate the data in dgraph. * Update rate card periodically. * Support: AWS, Azure, PKS, VKE, GCE #### AWS: * Public API: Available * Reference: https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/price-changes.html * API call: https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/region/index.json * Example for us-east-1: https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/us-east-1/index.json * Note: aws provides sdk in golang for pricing. Reference: https://docs.aws.amazon.com/sdk-for-go/api/service/pricing/ ================================================ FILE: docs/developers-guide.md ================================================ # Developers Guide - [Prerequisites](#prerequisites) - [Workspace Setup](#workspace-setup) - [Database Setup](#database-setup) - [Running Purser Controller](#running-purser-controller) - [Running Purser UI](#running-purser-ui) - [Purser Plugin Compilation](#purser-plugin-compilation) - [Plugin Execution](#plugin-execution) ## Prerequisites 1. Ensure the following dependencies are installed on your system. - [Go](https://golang.org/dl/) - [Git](https://git-scm.com/downloads) - [Docker](https://www.docker.com/) You may use the official binaries or your usual package manager. Also set the following environment variables - Set `GOPATH` environment variable. Refer [setting GOPATH](https://github.com/golang/go/wiki/SettingGOPATH) - Add `$GOPATH/bin` in system `PATH` variable by running `export PATH=$PATH:$GOPATH/bin`. Optionally, add the above exports to your `.bash_profile` or `.bashrc` to persist across console sessions. 2. Verify that the dependencies are properly installed. ``` bash go version, should be at least 1.7 git version docker version ``` ## Workspace Setup ### Fork the repository Navigate to the [Purser repo on GitHub](https://github.com/vmware/purser) and use the 'Fork' button. This gives you a copy of the repo for pull requests back to purser in `https://github.com//purser` ### Clone and Set Upstream Remote Make a local clone of the forked repo and add the base purser repo as the upstream remote repository. ``` shell # create and change directory to $GOPATH/src/github.com/vmware mkdir -p $GOPATH/src/github.com/vmware cd $GOPATH/src/github.com/vmware # clone the forked repository and change directory to purser git clone https://github.com//purser.git cd purser # add upstream repository as the original purser repo git remote add upstream https://github.com/vmware/purser.git ``` The last git command prepares your clone to pull changes from the upstream repo and push them into the fork, which enables you to keep the fork up to date. ### Download dependencies Run the following commands to download dependencies. ``` shell make tools make deps make install ``` ## Database Setup In order to persist inventory and discovery information such as pods and service details we use [Dgraph](https://dgraph.io/) to store the inventory metrics and resource relationship. In order to install DGraph from docker image follow the following steps: - Pull the latest Dgraph version ```bash docker pull dgraph/dgraph ``` - To run Dgraph in Docker ```bash mkdir -p /tmp/data # Run dgraph-zero docker run -d -p 5080:5080 -p 6080:6080 -p 8080:8080 -p 9080:9080 -p 8000:8000 -v /tmp/data:/dgraph --name diggy dgraph/dgraph dgraph zero # In another terminal, now run dgraph-alpha docker exec -d diggy dgraph alpha --lru_mb 2048 --zero localhost:5080 ``` - Optional: To start Dgraph UI(at `localhost:8000`) for running manual queries ```bash # Run Dgraph Ratel docker exec -d diggy dgraph-ratel ``` ## Running Purser Controller To run purser controller execute following commands ```bash # change directory to purser main folder cd $GOPATH/src/github.com/vmware/purser # run purser with log level as info and interactions as disabled by default go run cmd/controller/purserctrl.go --kubeconfig= --interactions=disable --dgraphURL=localhost --log=info ``` ## Running Purser UI Install latest version of `node` and `npm`. Then to run purser UI execute the following commands ```bash # change directory to purser ui folder cd $GOPATH/src/github.com/vmware/purser/ui # install node modules npm install # run purser UI at localhost:4200 npm run startdev ``` _Refer [UI docs](../ui/README.md) for more details._ ## Purser Plugin Compilation To create purser plugin binary `purser_plugin` at path `$GOPATH/bin` run the following commands ```bash # change directory to purser main folder cd $GOPATH/src/github.com/vmware/purser # create binary at path $GOPATH/bin go build -o $GOPATH/bin/purser_plugin github.com/vmware/purser/cmd/plugin ``` **NOTE:** _Windows users need to rename `purser_plugin` to `purser_plugin.exe`_ ## Plugin Execution 1. In order to install the Purser plugin, copy the [plugin.yaml](../plugin.yaml) file to one of the specified paths defined under the section [installing kubectl plugins](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/). 2. Run the following command to check the purser plugin works locally. ``` bash kubectl --kubeconfig= plugin purser help ``` ## Useful commands and links - To contribute to purser refer [CONTRIBUTING](../CONTRIBUTING.md) and [CODE_OF_CONDUCT](../CODE_OF_CONDUCT.md) - To drop complete dgraph database: `curl -X POST localhost:8080/alter -d '{"drop_all": true}'` ================================================ FILE: docs/manual-installation.md ================================================ # Manual Installation To install Purser manually from the Binary follow the steps described below. ## Purser Setup The following steps will install Purser in your cluster at namespace `purser`. Creation of this namespace is needed because purser needs to create a service-account which requires namespace. Also, the frontend will use kubernetes DNS to call backend for data and this DNS contains a field for namespace. ``` bash # Namespace setup kubectl create ns purser # DB setup curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-database-setup.yaml -O kubectl --namespace=purser create -f purser-database-setup.yaml # Purser controller setup curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-controller-setup.yaml -O kubectl --namespace=purser create -f purser-controller-setup.yaml # Purser UI setup curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-ui-setup.yaml -O kubectl --namespace=purser create -f purser-ui-setup.yaml ``` **NOTE:** If you don't have `curl` installed you can download `purser-database-setup.yaml` from [here](./cluster/purser-database-setup.yaml), `purser-controller-setup.yaml` from [here](cluster/purser-controller-setup.yaml) and `purser-ui-setup.yaml` from [here](cluster/purser-ui-setup.yaml). Then `kubectl create -f purser-database-setup.yaml` , `kubectl create -f purser-controller-setup.yaml` and `kubectl create -f purser-ui-setup.yaml` will setup purser in your cluster. ##### Change Settings and Enable/Disable Purser Features The following settings can be customized before Controller installation: - Change the default **log level**, **dgraph url** and **dgraph port** by editing `args` field in the [purser-controller-setup.yaml](cluster/purser-controller-setup.yaml). (Default: `--log=info`, `--dgraphURL=purser-db`, `--dgraphPort=9080`) - Enable/Disable **resource interactions** capability by editing `args` field in the [purser-controller-setup.yaml](cluster/purser-controller-setup.yaml) and uncommenting `pods/exec` rule from purser-permissions. (Default: `disabled`) - Enable **subscription to inventory changes** capability by creating an object of custom resource kind `Subscriber`. (Refer: [example-subscriber.yaml](./cluster/artifacts/example-subscriber.yaml)) - Enable **customized logical grouping of resources** by creating an object of custom resource kind `Group`. (Refer: [docs](docs/custom-group-installation-and-usage.md) for custom group installation and usage) _**NOTE:** Use flag `--kubeconfig=` if your cluster configuration is not at the [default location](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable)._ ## Purser Plugin Installation (Optional) - Download the purser plugin descriptor for your environment from the [releases page](https://github.com/vmware/purser/releases/download/v1.0.0/plugin.yaml). - Move the `plugin.yaml` file into one of the paths specified under the Kubernetes [documentation](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins). - Download the purser binary corresponding to your operating system from the [releases page](https://github.com/vmware/purser/releases/tag/v1.0.0). - Move the binary into one of the directories in your environment `PATH`. ================================================ FILE: docs/plugin-installation.md ================================================ # Purser Plugin Setup _NOTE: This Plugin installation is optional. Install it if you want to use CLI of Purser._ ## Linux and macOS ``` bash # Binary installation wget -q https://github.com/vmware/purser/blob/master/build/purser-binary-install.sh && sh purser-binary-install.sh ``` Enter your cluster's configuration path when prompted. The plugin binary needs to be in your `PATH` environment variable, so once the download of the binary is finished the script tries to move it to `/usr/local/bin`. This may need your sudo permission. ## Windows/Others For installation on Windows follow the steps in the [manual installation guide](./docs/manual-installation.md). ## Uninstalling Purser Plugin ### Linux/macOS ``` bash curl https://raw.githubusercontent.com/vmware/purser/master/build/purser-binary-install.sh -O && sh purser-binary-uninstall.sh ``` ================================================ FILE: docs/plugin-usage.md ================================================ # Purser Plugin Usage Once installed, Purser is ready for use right away. You can query using native Kubernetes grouping artifacts. Purser supports the following list of commands. ``` bash # query cluster visibility in terms of savings and summary for the application. kubectl plugin purser get [summary|savings] # query resources filtered by associated namespace, labels and groups. kubectl plugin purser get resources group # query cost filtered by associated labels, pods and node. kubectl plugin purser get cost label kubectl plugin purser get cost pod kubectl plugin purser get cost node all # configure user-costs for the choice of deployment. kubectl plugin purser [set|get] user-costs ``` _Use flag `--kubeconfig=` if your cluster configuration is not at the [default location](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable)._ ## Examples 1. Get Cluster Summary ``` bash $ kubectl plugin purser get summary Cluster Summary Compute: Node count: 57 Cost: 3015.48$ Total Capacity: Cpu(vCPU): 456 Memory(GB): 1770.50 Provisioned Resources: Cpu Request(vCPU): 319 Memory Request(GB): 1032.67 Storage: Persistent Volume count: 151 Capacity(GB): 9297.00 Cost: 4124.79$ PV Claim count: 108 PV Claim Capacity(GB): 8867.00 Cost: Compute cost: 3015.48$ Storage cost: 4124.79$ Total cost: 7140.27$ ``` 2. Get Cost Of All Nodes ``` bash kubectl purser get cost node all ``` 3. Get Savings ``` bash $ kubectl plugin purser get savings Savings Summary Storage: Unused Volumes: 43 Unused Capacity(GB): 430.00 Month To Date Savings: 186.33$ Projected Monthly Savings: 1066.40$ ``` Next, define higher level groupings to define your business, logical or application constructs. ## Defining Custom Groups Refer [doc](./custom-group-installation-and-usage.md) for custom group installation and usage. ================================================ FILE: docs/purser-deployment.md ================================================ # Purser Deployment In order to deploy the Purser UI and DGraph database service, follow the below listed steps: 1. Switch the current context to point to the desired cluster. ``` bash kubectl config use-context ``` Read more about configuring and setting the `KUBECONFIG` and kubernetes context [here](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/). 2. If the cluster does not have a valid public IP, set proxy in order to expose the service externally. ``` bash kubectl proxy ``` 3. When set, you can simply deploy the Purser UI and Dgraph database service using target `make deploy-purser`. _If you wish to however, deploy the database service and the UI service separately, execute the following targets respectively._ ``` bash # deploy Dgraph database make kubectl-deploy-purser-db # deploy purser UI make kubectl-deploy-purser-ui ``` 4. Once deployed, if proxy was set the UI service can be accessed from [this url](http://127.0.0.1:8001/api/v1/namespaces/default/services/http:purser-ui:4200/proxy/home). If public IP was available for your cluster, the UI service should be accessible from path `:`. Eg. `http://:/home` 5. In order to drop the Dgraph entries from the database, delete the `Persistent Volume` corresponding to the `dgraph datadir`. ================================================ FILE: docs/sourcecode-installation.md ================================================ # Installation Through Source Code - [Prerequisites](#prerequisites) - [Server Side Installation (Controller Installation)](#server-side-installation-controller-installation) - [Client Side Installation (Plugin Installation)](#client-side-installation-plugin-installation) ## Prerequisites 1. Kubernetes Version 1.9 or greater - `kubectl` installed and configured. For details refer [here](https://kubernetes.io/docs/tasks/tools/install-kubectl/). 2. Dependencies - [Go](https://golang.org/dl/) - version > 1.7 - setup `GOPATH` environment variable by as per the [Golang documentation](https://github.com/golang/go/wiki/SettingGOPATH). - add `$GOPATH/bin` directory to your environment `$PATH` variable. - [Docker](https://www.docker.com/get-started) 3. Fetch the Purser source code from GitHub. ``` go go get github.com/vmware/purser ``` ``` bash # change directory to project root cd $GOPATH/src/github.com/vmware/purser ``` 4. For Windows users, install gnu `make` from [here](http://gnuwin32.sourceforge.net/packages/make.htm). 5. Download project dependencies with `make`. ``` bash # download project tools make tools # download project dependencies make deps # update project depedencies make update ``` ## Server Side Installation (Controller Installation) Follow the below steps to install the purser controller and custom resource definitions for the user groups in the Kubernetes cluster. ### Build Controller Binary Build the purser controller binary using `make` target. ``` bash make build ``` ### Build Container Image Update the [Makefile](./Makefile) to set the `REGISTRY` field to your Docker username and execute the following `make` targets to build and publish the docker images. ``` bash # create the container(docker image) make container # authenticate your Docker credentials docker login # publish your docker image to docker hub make push ``` ### Install Purser Plugin - Update the image name in [`purser-controller-setup.yaml`](../cluster/purser-controller-setup.yaml) to the docker image name that you pushed. - Install the controller in the cluster using `kubectl`. The following steps will install Purser in your cluster at namespace `purser`. Creation of this namespace is needed because purser needs to create a service-account which requires namespace. Also, the frontend will use kubernetes DNS to call backend for data and this DNS contains a field for namespace. ``` bash # Namespace setup kubectl create ns purser # DB setup curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-database-setup.yaml -O kubectl --namespace=purser create -f purser-database-setup.yaml # Purser controller setup kubectl --namespace=purser create -f purser-controller-setup.yaml # Purser UI setup curl https://raw.githubusercontent.com/vmware/purser/master/cluster/purser-ui-setup.yaml -O kubectl --namespace=purser create -f purser-ui-setup.yaml ``` _Use flag `--kubeconfig=` if your cluster configuration is not at the [default location](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable)._ ## Client Side Installation (Plugin Installation) - Build the purser plugin binary in the `GOPATH/bin` directory. ``` go go build -o $GOPATH/bin/purser_plugin github.com/vmware/purser/cmd/plugin ``` - Install the Purser plugin by copying the [`plugin.yaml`](../plugin.yaml) into one of the paths specified under the Kubernetes documentation section [installing kubectl plugins](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/). ================================================ FILE: openapi.yaml ================================================ --- openapi: 3.0.1 info: title: Purser description: Purser runs on server port `:3030` and exposes API endpoints to generate an insight into your Kubernetes applications by providing details of communicating services and pods. version: 1.0.0 servers: - url: http://localhost:3030 paths: /api/hierarchy: get: description: Gets the top level cluster hierachy parameters: - name: view in: query description: physical or logical depending on selection of physical entities such as nodes, persistent volumes or logical entities such as namespaces, pods etc. Default is logical. required: false style: FORM explode: true schema: type: string example: physical responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/namespace: get: description: Gets the K8s Namespace hierachy parameters: - name: name in: query description: a valid K8s Namespace name prefixed with `namespace-` required: false style: FORM explode: true schema: type: string example: namespace-kube-public responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/pvc: get: description: Gets the K8s PVC hierachy parameters: - name: name in: query description: a valid K8s PVC name prefixed with `pvc-` required: true style: FORM explode: true schema: type: string example: pvc-datadir-dgraph-0 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/job: get: description: Gets the K8s Job hierachy parameters: - name: name in: query description: a valid K8s Job name prefixed with `job-` required: true style: FORM explode: true schema: type: string example: job-kube-proxy responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/container: get: description: Gets the K8s container hierachy parameters: - name: name in: query description: a valid K8s container name prefixed with `container-` required: true style: FORM explode: true schema: type: string example: container-etcd responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/replicaset: get: description: Gets the K8s Replicaset hierachy parameters: - name: name in: query description: a valid K8s Replicaset name prefixed with `replicaset-` required: true style: FORM explode: true schema: type: string example: replicaset-kube-dns-86f4d74b45 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/pod: get: description: Gets the K8s Pod hierachy parameters: - name: name in: query description: a valid K8s Pod name prefixed with `pod-` required: true style: FORM explode: true schema: type: string example: pod-etcd-minikube responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/node: get: description: Gets the K8s Node hierachy parameters: - name: name in: query description: a valid K8s Node name prefixed with `node-` required: true style: FORM explode: true schema: type: string example: node-minikube responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/daemonset: get: description: Gets the K8s Daemonset hierachy parameters: - name: name in: query description: a valid K8s Daemonset name prefixed with `daemonset-` required: true style: FORM explode: true schema: type: string example: daemonset-kube-proxy responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/deployment: get: description: Gets the K8s Deployment hierachy parameters: - name: name in: query description: a valid K8s Deployment name prefixed with `deployment-` required: true style: FORM explode: true schema: type: string example: deployment-kube-dns responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/pv: get: description: Gets the K8s PV hierachy parameters: - name: name in: query description: a valid K8s PV name prefixed with `pv-` required: true style: FORM explode: true schema: type: string example: pv-pvc-5ffeaa3f-ed5e-11e8-b395-080027a0bfc5 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/statefulset: get: description: Gets the K8s Statefulset hierachy parameters: - name: name in: query description: a valid K8s Statefulset name prefixed with `statefulset-` required: true style: FORM explode: true schema: type: string example: statefulset-kube-dns-86f4d74b45 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/hierarchy/process: get: description: Gets the K8s container process hierachy parameters: - name: name in: query description: a valid K8s container process name prefixed with `process-` required: true style: FORM explode: true schema: type: string example: process-etcd responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Hierarchy' /api/metrics: get: description: Gets the complete K8s cluster metrics parameters: - name: view in: query description: physical or logical depending on selection of physical entities such as nodes, persistent volumes or logical entities such as namespaces, pods etc. Default is logical. required: false style: FORM explode: true schema: type: string example: logical responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/namespace: get: description: Gets the K8s Namespace metrics parameters: - name: name in: query description: a valid K8s Namespace name prefixed with `namespace-` required: false style: FORM explode: true schema: type: string example: namespace-kube-public responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/pvc: get: description: Gets the K8s PVC metrics parameters: - name: name in: query description: a valid K8s PVC name prefixed with `pvc-` required: true style: FORM explode: true schema: type: string example: pvc-datadir-dgraph-0 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/job: get: description: Gets the K8s Job metrics parameters: - name: name in: query description: a valid K8s Job name prefixed with `job-` required: true style: FORM explode: true schema: type: string example: job-kube-proxy responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/container: get: description: Gets the K8s container metrics parameters: - name: name in: query description: a valid K8s container name prefixed with `container-` required: true style: FORM explode: true schema: type: string example: container-etcd responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/replicaset: get: description: Gets the K8s Replicaset metrics parameters: - name: name in: query description: a valid K8s Replicaset name prefixed with `replicaset-` required: true style: FORM explode: true schema: type: string example: replicaset-kube-dns-86f4d74b45 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/pod: get: description: Gets the K8s Pod metrics parameters: - name: name in: query description: a valid K8s Pod name prefixed with `pod-` required: true style: FORM explode: true schema: type: string example: pod-etcd-minikube responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/node: get: description: Gets the K8s Node metrics parameters: - name: name in: query description: a valid K8s Node name prefixed with `node-` required: true style: FORM explode: true schema: type: string example: node-minikube responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/daemonset: get: description: Gets the K8s Daemonset metrics parameters: - name: name in: query description: a valid K8s Daemonset name prefixed with `daemonset-` required: true style: FORM explode: true schema: type: string example: daemonset-kube-proxy responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/deployment: get: description: Gets the K8s Deployment metrics parameters: - name: name in: query description: a valid K8s Deployment name prefixed with `deployment-` required: false style: FORM explode: true schema: type: string example: deployment-kube-dns responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/pv: get: description: Gets the K8s PV metrics parameters: - name: name in: query description: a valid K8s PV name prefixed with `pv-` required: true style: FORM explode: true schema: type: string example: pv-pvc-5ffeaa3f-ed5e-11e8-b395-080027a0bfc5 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/metrics/statefulset: get: description: Gets the K8s Statefulset metrics parameters: - name: name in: query description: a valid K8s Statefulset name prefixed with `statefulset-` required: true style: FORM explode: true schema: type: string example: statefulset-kube-dns-86f4d74b45 responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Metrics' /api/interactions/pod: get: description: Gets K8s Pods interactions parameters: - name: name in: query description: a valid K8s Pod name prefixed with `pod-` required: false style: FORM explode: true schema: type: string example: pod-kube-dns-86f4d74b45-4v66p - name: orphan in: query description: filters out orphan pods if set to false. Default is true required: false style: FORM explode: true schema: type: boolean example: "false" responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: $ref: '#/components/schemas/Interactions' /api/edges: get: description: Gets edges between Dgraph Components responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: type: array items: $ref: '#/components/schemas/Edges' /api/nodes: get: description: Gets Dgraph node Components responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: type: array items: $ref: '#/components/schemas/Nodes' /api/groups: get: description: Gets array of Group objects along with their metrics responses: 200: description: Operation Successful content: application/json; charset=UTF-8: schema: type: array items: $ref: '#/components/schemas/Groups' components: schemas: Hierarchy: type: object properties: data: $ref: '#/components/schemas/Hierarchy_data' Metrics: type: object properties: data: $ref: '#/components/schemas/Metrics_data' Interactions: type: object properties: pods: type: array items: $ref: '#/components/schemas/Interactions_pods' Nodes: type: object properties: id: type: integer format: int32 example: 1 label: type: string example: postgres-54f9679f4b-nj8vv title: type: string example: pods value: type: integer format: int32 example: 1 group: type: integer format: int32 example: 1 cid: type: array items: type: string example: postgres Edges: type: object properties: from: type: integer format: int32 example: 4 to: type: integer format: int32 example: 3 title: type: string example: 7 times communicated Groups: type: object properties: name: type: string example: Group-1 podsCount: type: integer format: int32 example: 4 mtdCPU: type: number example: 10.2 mtdMemory: type: number example: 3.9 mtdStorage: type: number example: 41.5 cpu: type: number example: 2.04 memory: type: number example: 0.78 storage: type: number example: 8.3 mtdCPUCost: type: number example: 0.2448 mtdMemoryCost: type: number example: 0.039 mtdStorageCost: type: number example: 0.00576388852 mtdCost: type: number example: 0.28956388852 Hierarchy_data_children: type: object properties: name: type: string example: namespace-default type: type: string example: namespace Hierarchy_data: type: object properties: name: type: string example: cluster type: type: string example: cluster children: type: array items: $ref: '#/components/schemas/Hierarchy_data_children' Metrics_data_children: type: object properties: name: type: string example: namespace-default type: type: string example: namespace cpu: type: number example: 0.915 memory: type: number example: 0.224609 cpuCost: type: number example: 0.02196 memoryCost: type: number example: 0.002246 Metrics_data: type: object properties: name: type: string example: cluster type: type: string example: cluster children: type: array items: $ref: '#/components/schemas/Metrics_data_children' cpu: type: number example: 0.915 memory: type: number example: 0.224609 cpuCost: type: number example: 0.02196 memoryCost: type: number example: 0.002246 Interactions_inbound: type: object properties: name: type: string example: pod-webapp-958cf5567-xb758 Interactions_pods: type: object properties: name: type: string example: pod-postgres-54f9679f4b-vsdhl inbound: type: array items: $ref: '#/components/schemas/Interactions_inbound' outbound: type: array items: $ref: '#/components/schemas/Interactions_inbound' extensions: {} ================================================ FILE: pkg/apis/groups/v1/deepcopy.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import "k8s.io/apimachinery/pkg/runtime" // DeepCopyInto copies all properties of this object into another object of the // same type that is provided as a pointer. func (in *Group) DeepCopyInto(out *Group) { out.TypeMeta = in.TypeMeta out.ObjectMeta = in.ObjectMeta out.Spec = in.Spec out.Status = in.Status } // DeepCopyObject returns a generically typed copy of an object func (in *Group) DeepCopyObject() runtime.Object { out := Group{} in.DeepCopyInto(&out) return &out } // DeepCopyObject returns a generically typed copy of an object func (in *GroupList) DeepCopyObject() runtime.Object { out := GroupList{} out.TypeMeta = in.TypeMeta out.ListMeta = in.ListMeta if in.Items != nil { out.Items = make([]*Group, len(in.Items)) for i := range in.Items { in.Items[i].DeepCopyInto(out.Items[i]) } } return &out } ================================================ FILE: pkg/apis/groups/v1/docs.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 ================================================ FILE: pkg/apis/groups/v1/register.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) // SchemeBuilder parameters var ( SchemeBuilder = runtime.NewSchemeBuilder(AddKnownTypes) AddToScheme = SchemeBuilder.AddToScheme ) // SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: CRDGroup, Version: CRDVersion} // Kind takes an unqualified kind and returns a Group qualified GroupKind func Kind(kind string) schema.GroupKind { return SchemeGroupVersion.WithKind(kind).GroupKind() } // Resource takes an unqualified resource and returns a Group qualified GroupResource func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() } // AddKnownTypes ... func AddKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &Group{}, &GroupList{}, ) meta_v1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil } ================================================ FILE: pkg/apis/groups/v1/types.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( "github.com/vmware/purser/pkg/controller/metrics" "time" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // CRD Group attributes const ( CRDPlural string = "groups" CRDGroup string = "vmware.purser.com" CRDVersion string = "v1" FullCRDName string = CRDPlural + "." + CRDGroup ) // Group describes our custom Group resource type Group struct { meta_v1.TypeMeta `json:",inline"` meta_v1.ObjectMeta `json:"metadata"` Spec GroupSpec `json:"spec"` Status GroupStatus `json:"status,omitempty"` } // GroupSpec is the spec for the Group resource type GroupSpec struct { Name string `json:"name"` Type string `json:"type,omitempty"` Expressions map[string]map[string][]string `json:"labels,omitempty"` AllocatedResources *GroupMetrics `json:"metrics,omitempty"` PITMetrics *GroupMetrics `json:"pitMetrics,omitempty"` MTDMetrics *GroupMetrics `json:"mtdMetrics,omitempty"` MTDCost *Cost `json:"mtdCost,omitempty"` PerHourCost *Cost `json:"perHourCost,omitempty"` LastMonthCost *Cost `json:"lastMonthCost,omitempty"` LastLastMonthCost *Cost `json:"lastLastMonthCost,omitempty"` LastUpdated time.Time `json:"lastUpdated,omitempty"` } // GroupMetrics ... type GroupMetrics struct { CPULimit float64 MemoryLimit float64 StorageCapacity float64 CPURequest float64 MemoryRequest float64 StorageClaim float64 } // Cost details type Cost struct { TotalCost float64 CPUCost float64 MemoryCost float64 StorageCost float64 } // GroupList is the list of Group resources type GroupList struct { meta_v1.TypeMeta `json:",inline"` meta_v1.ListMeta `json:"metadata"` Items []*Group `json:"items"` } // GroupStatus holds the status information for each Group resource type GroupStatus struct { State string `json:"state,omitempty"` Message string `json:"message,omitempty"` } // PodDetails information for the pods associated with the Group resource type PodDetails struct { Name string StartTime meta_v1.Time EndTime meta_v1.Time Containers []*Container PodVolumeClaims map[string]*PersistentVolumeClaim } // PersistentVolumeClaim information for the pods associated with the Group resource // A PVC can bound and unbound to a pod many times, so maintaining // BoundTimes and UnboundTimes as lists. // A PVC can be upgraded or downgraded, so maintaining capacityAllocated as a list // Whenever a PVC capacity changes will update UnboundTime for old capacity, and // append new capacity to capacityAllocated with bound time appended to BoundTimes // The i-th capacity allocated corresponds to the i-th bound time and to i-th unbound time. // Similarly for RequestSizeInGB type PersistentVolumeClaim struct { Name string VolumeName string RequestSizeInGB []float64 CapacityAllocatedInGB []float64 BoundTimes []meta_v1.Time UnboundTimes []meta_v1.Time } // Container information for the pods associated with the Group resource type Container struct { Name string Metrics *metrics.Metrics } ================================================ FILE: pkg/apis/subscriber/v1/deepcopy.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import "k8s.io/apimachinery/pkg/runtime" // DeepCopyInto copies all properties of this object into another object of the // same type that is provided as a pointer. func (in *Subscriber) DeepCopyInto(out *Subscriber) { out.TypeMeta = in.TypeMeta out.ObjectMeta = in.ObjectMeta out.Spec = in.Spec out.Status = in.Status } // DeepCopyObject returns a generically typed copy of an object func (in *Subscriber) DeepCopyObject() runtime.Object { out := Subscriber{} in.DeepCopyInto(&out) return &out } // DeepCopyObject returns a generically typed copy of an object func (in *SubscriberList) DeepCopyObject() runtime.Object { out := SubscriberList{} out.TypeMeta = in.TypeMeta out.ListMeta = in.ListMeta if in.Items != nil { out.Items = make([]Subscriber, len(in.Items)) for i := range in.Items { in.Items[i].DeepCopyInto(&out.Items[i]) } } return &out } ================================================ FILE: pkg/apis/subscriber/v1/docs.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 ================================================ FILE: pkg/apis/subscriber/v1/register.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) // SchemeBuilder parameters var ( SchemeBuilder = runtime.NewSchemeBuilder(AddKnownTypes) AddToScheme = SchemeBuilder.AddToScheme ) // SubscriberGroupVersion is group version used to register these objects var SubscriberGroupVersion = schema.GroupVersion{Group: SubscriberGroup, Version: SubscriberVersion} // Kind takes an unqualified kind and returns a Group qualified GroupKind func Kind(kind string) schema.GroupKind { return SubscriberGroupVersion.WithKind(kind).GroupKind() } // Resource takes an unqualified resource and returns a Group qualified GroupResource func Resource(resource string) schema.GroupResource { return SubscriberGroupVersion.WithResource(resource).GroupResource() } // AddKnownTypes ... func AddKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SubscriberGroupVersion, &Subscriber{}, &SubscriberList{}, ) meta_v1.AddToGroupVersion(scheme, SubscriberGroupVersion) return nil } ================================================ FILE: pkg/apis/subscriber/v1/types.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" // CRD Subscriber attributes const ( SubscriberPlural string = "subscribers" SubscriberGroup string = "vmware.purser.com" SubscriberVersion string = "v1" SubscriberFullName string = SubscriberPlural + "." + SubscriberGroup ) // Subscriber information type Subscriber struct { meta_v1.TypeMeta `json:",inline"` meta_v1.ObjectMeta `json:"metadata"` Spec SubscriberSpec `json:"spec"` Status SubscriberStatus `json:"status,omitempty"` } // SubscriberSpec definition details type SubscriberSpec struct { Name string `json:"name"` Headers map[string]string `json:"headers"` URL string `json:"url"` } // SubscriberStatus definition type SubscriberStatus struct { State string `json:"state,omitempty"` Message string `json:"message,omitempty"` } // SubscriberList type type SubscriberList struct { meta_v1.TypeMeta `json:",inline"` meta_v1.ListMeta `json:"metadata"` Items []Subscriber `json:"items"` } ================================================ FILE: pkg/client/clientset/typed/groups/v1/group.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( "github.com/vmware/purser/pkg/apis/groups/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" ) // GroupInterface has client methods we need to access Group object type GroupInterface interface { Create(obj *v1.Group) (*v1.Group, error) Update(obj *v1.Group) (*v1.Group, error) Delete(name string, options *meta_v1.DeleteOptions) error Get(name string) (*v1.Group, error) List(opts meta_v1.ListOptions) (*v1.GroupList, error) Watch(opts meta_v1.ListOptions) (watch.Interface, error) } // GroupClient defines the CRD Group structure type GroupClient struct { client *rest.RESTClient ns string plural string codec runtime.ParameterCodec } // Create creates a new group. func (c *GroupClient) Create(obj *v1.Group) (*v1.Group, error) { result := v1.Group{} err := c.client.Post(). Namespace(c.ns). Resource(c.plural). Body(obj). Do(). Into(&result) return &result, err } // Update modifies the group specification. func (c *GroupClient) Update(obj *v1.Group) (*v1.Group, error) { result := v1.Group{} err := c.client.Put(). Name((obj.Name)). Namespace(c.ns). Resource(c.plural). Body(obj). Do(). Into(&result) return &result, err } // Delete removes the group. func (c *GroupClient) Delete(name string, options *meta_v1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource(c.plural). Name(name). Body(options). Do(). Error() } // Get fetches the group func (c *GroupClient) Get(name string) (*v1.Group, error) { result := v1.Group{} err := c.client.Get(). Namespace(c.ns). Resource(c.plural). Name(name). Do(). Into(&result) return &result, err } // List fetches the list of groups. func (c *GroupClient) List(opts meta_v1.ListOptions) (*v1.GroupList, error) { result := v1.GroupList{} err := c.client.Get(). Namespace(c.ns). Resource(c.plural). VersionedParams(&opts, c.codec). Do(). Into(&result) return &result, err } // Watch watches for the groups. func (c *GroupClient) Watch(opts meta_v1.ListOptions) (watch.Interface, error) { opts.Watch = true return c.client. Get(). Namespace(c.ns). Resource(c.plural). VersionedParams(&opts, c.codec). Watch() } ================================================ FILE: pkg/client/clientset/typed/groups/v1/group_client.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( "reflect" "time" log "github.com/Sirupsen/logrus" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/rest" ) // NewGroupClient returns an instance of the Group Client func NewGroupClient(clientset apiextcs.Interface, config *rest.Config) *GroupClient { err := createGroupCRD(clientset) if err != nil { log.Fatalf("failed to create CRD group %v", err) } // Wait for the CRD to be created before we use it (only needed if its a new one) time.Sleep(3 * time.Second) // Create a new clientset which include our CRD schema gcrdcs, gscheme, err := newClient(config) if err != nil { log.Fatalf("failed to add CRD group schema to clientset %v", err) } // Create a CRD client interface return Group(gcrdcs, gscheme, "default") } // Group returns a new instance of the Group CRD func Group(client *rest.RESTClient, scheme *runtime.Scheme, namespace string) *GroupClient { return &GroupClient{ client: client, ns: namespace, plural: groups_v1.CRDPlural, codec: runtime.NewParameterCodec(scheme), } } func createGroupCRD(clientset apiextcs.Interface) error { crd := &apiextv1beta1.CustomResourceDefinition{ ObjectMeta: meta_v1.ObjectMeta{Name: groups_v1.FullCRDName}, Spec: apiextv1beta1.CustomResourceDefinitionSpec{ Group: groups_v1.CRDGroup, Version: groups_v1.CRDVersion, //TODO: make cluster scoped? Scope: apiextv1beta1.NamespaceScoped, Names: apiextv1beta1.CustomResourceDefinitionNames{ Plural: groups_v1.CRDPlural, Kind: reflect.TypeOf(groups_v1.Group{}).Name(), }, }, } _, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) if err != nil && apierrors.IsAlreadyExists(err) { return nil } return err } func newClient(cfg *rest.Config) (*rest.RESTClient, *runtime.Scheme, error) { config := *cfg scheme, err := setConfigDefaults(&config) if err != nil { return nil, nil, err } client, err := rest.RESTClientFor(&config) if err != nil { return nil, nil, err } return client, scheme, nil } func setConfigDefaults(config *rest.Config) (*runtime.Scheme, error) { scheme := runtime.NewScheme() SchemeBuilder := runtime.NewSchemeBuilder(groups_v1.AddKnownTypes) if err := SchemeBuilder.AddToScheme(scheme); err != nil { return nil, err } config.GroupVersion = &groups_v1.SchemeGroupVersion config.APIPath = "/apis" config.ContentType = runtime.ContentTypeJSON config.NegotiatedSerializer = serializer.DirectCodecFactory{ CodecFactory: serializer.NewCodecFactory(scheme), } return scheme, nil } ================================================ FILE: pkg/client/clientset/typed/subscriber/v1/subsciber_client.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( "reflect" "time" log "github.com/Sirupsen/logrus" subscriber_v1 "github.com/vmware/purser/pkg/apis/subscriber/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/rest" ) // NewSubscriberClient returns an instance of the Subscriber Client func NewSubscriberClient(clientset apiextcs.Interface, config *rest.Config) *SubscriberClient { err := createSubscriberCRD(clientset) if err != nil { log.Fatalf("failed to create CRD subscriber %v", err) } // Wait for the CRD to be created before we use it (only needed if its a new one) time.Sleep(3 * time.Second) // Create a new clientset which include our CRD schema crdcs, scheme, err := newClient(config) if err != nil { log.Fatalf("failed to add CRD subscriber schema to clientset %v", err) } // Create a CRD client interface return Subscriber(crdcs, scheme, "default") } // Subscriber returns an instance of the subscriber client func Subscriber(client *rest.RESTClient, scheme *runtime.Scheme, namespace string) *SubscriberClient { return &SubscriberClient{ client: client, ns: namespace, plural: subscriber_v1.SubscriberPlural, codec: runtime.NewParameterCodec(scheme), } } func createSubscriberCRD(clientset apiextcs.Interface) error { crd := &apiextv1beta1.CustomResourceDefinition{ ObjectMeta: meta_v1.ObjectMeta{Name: subscriber_v1.SubscriberFullName}, Spec: apiextv1beta1.CustomResourceDefinitionSpec{ Group: subscriber_v1.SubscriberGroup, Version: subscriber_v1.SubscriberVersion, //TODO: make cluster scoped? Scope: apiextv1beta1.NamespaceScoped, Names: apiextv1beta1.CustomResourceDefinitionNames{ Plural: subscriber_v1.SubscriberPlural, Kind: reflect.TypeOf(subscriber_v1.Subscriber{}).Name(), }, }, } _, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) // Ignore error if it already exists if err != nil && apierrors.IsAlreadyExists(err) { return nil } return err } func newClient(cfg *rest.Config) (*rest.RESTClient, *runtime.Scheme, error) { config := *cfg scheme, err := setConfigDefaults(&config) if err != nil { return nil, nil, err } client, err := rest.RESTClientFor(&config) if err != nil { return nil, nil, err } return client, scheme, nil } func setConfigDefaults(config *rest.Config) (*runtime.Scheme, error) { scheme := runtime.NewScheme() SchemeBuilder := runtime.NewSchemeBuilder(subscriber_v1.AddKnownTypes) if err := SchemeBuilder.AddToScheme(scheme); err != nil { return nil, err } config.GroupVersion = &subscriber_v1.SubscriberGroupVersion config.APIPath = "/apis" config.ContentType = runtime.ContentTypeJSON config.NegotiatedSerializer = serializer.DirectCodecFactory{ CodecFactory: serializer.NewCodecFactory(scheme)} return scheme, nil } ================================================ FILE: pkg/client/clientset/typed/subscriber/v1/subscriber.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package v1 import ( "github.com/vmware/purser/pkg/apis/subscriber/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" ) // SubscriberInterface has client methods we need to access Subscriber object type SubscriberInterface interface { Create(obj *v1.Subscriber) (*v1.Subscriber, error) Update(obj *v1.Subscriber) (*v1.Subscriber, error) Delete(name string, options *meta_v1.DeleteOptions) error Get(name string) (*v1.Subscriber, error) List(opts meta_v1.ListOptions) (*v1.SubscriberList, error) Watch(opts meta_v1.ListOptions) (watch.Interface, error) } // SubscriberClient structure type SubscriberClient struct { client *rest.RESTClient ns string plural string codec runtime.ParameterCodec } // Create creates a CRD subscriber. func (c *SubscriberClient) Create(obj *v1.Subscriber) (*v1.Subscriber, error) { result := v1.Subscriber{} err := c.client.Post(). Namespace(c.ns). Resource(c.plural). Body(obj). Do(). Into(&result) return &result, err } // Update modifies the subscriber. func (c *SubscriberClient) Update(obj *v1.Subscriber) (*v1.Subscriber, error) { result := v1.Subscriber{} err := c.client.Put(). Name((obj.Name)). Namespace(c.ns). Resource(c.plural). Body(obj). Do(). Into(&result) return &result, err } // Delete removes the subscriber. func (c *SubscriberClient) Delete(name string, options *meta_v1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource(c.plural). Name(name). Body(options). Do(). Error() } // Get returns the subscriber func (c *SubscriberClient) Get(name string) (*v1.Subscriber, error) { result := v1.Subscriber{} err := c.client.Get(). Namespace(c.ns). Resource(c.plural). Name(name). Do(). Into(&result) return &result, err } // List fetches the list of subscriber CRD clients. func (c *SubscriberClient) List(opts meta_v1.ListOptions) (*v1.SubscriberList, error) { result := v1.SubscriberList{} err := c.client.Get(). Namespace(c.ns). Resource(c.plural). VersionedParams(&opts, c.codec). Do(). Into(&result) return &result, err } // Watch watches for the subcriber CRD func (c *SubscriberClient) Watch(opts meta_v1.ListOptions) (watch.Interface, error) { opts.Watch = true return c.client. Get(). Namespace(c.ns). Resource(c.plural). VersionedParams(&opts, c.codec). Watch() } ================================================ FILE: pkg/client/clientset.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package client import ( log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/utils" apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/rest" ) // GetAPIExtensionClient returns a client for the cluster and it's config. func GetAPIExtensionClient(kubeconfigPath string) (*apiextcs.Clientset, *rest.Config) { config, err := utils.GetKubeconfig(kubeconfigPath) if err != nil { log.Fatalf("failed to fetch kubeconfig %v", err) } // create clientset and create our CRD, this only need to run once clientset, clientErr := apiextcs.NewForConfig(config) if clientErr != nil { log.Fatalf("failed to connect to the cluster %v", clientErr) } return clientset, config } ================================================ FILE: pkg/controller/buffering/ring_buffer.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package buffering import ( "sync" log "github.com/Sirupsen/logrus" ) // BufferSize the default size for the Ring Buffer const BufferSize uint32 = 5000 // RingBuffer data structure type RingBuffer struct { start, end, Size uint32 buffer [BufferSize]*interface{} Mutex *sync.Mutex } // Put adds the item into buffer if there is room in buffer. // Returns true if item is buffered otherwise false. func (r *RingBuffer) Put(inp interface{}) bool { r.Mutex.Lock() defer r.Mutex.Unlock() if r.isFull() { return false } next := next(r.end, r.Size) r.buffer[r.end] = &inp r.end = next return true } // Get returns the elements in FIFO manner or nil if buffer is empty. func (r *RingBuffer) Get() *interface{} { r.Mutex.Lock() defer r.Mutex.Unlock() if r.isEmpty() { return nil } next := next(r.start, r.Size) curval := r.buffer[r.start] r.buffer[r.start] = nil r.start = next return curval } // ReadN reads the next n available elements in the buffer. // Returns elements and number of elements read. func (r *RingBuffer) ReadN(n uint32) ([]*interface{}, uint32) { r.Mutex.Lock() defer r.Mutex.Unlock() var elements []*interface{} start := r.start for i := uint32(0); i < n; i++ { if start == r.end { break } elements = append(elements, r.buffer[start]) start = next(start, r.Size) } return elements, uint32(len(elements)) } // RemoveN removes the first n elements from the buffer. func (r *RingBuffer) RemoveN(n uint32) { r.Mutex.Lock() defer r.Mutex.Unlock() start := r.start for i := uint32(0); i < n; i++ { if start == r.end { break } r.buffer[start] = nil start = next(start, r.Size) r.start = start } } func (r *RingBuffer) isEmpty() bool { return r.start == r.end } func (r *RingBuffer) isFull() bool { return next(r.end, r.Size) == r.start } func next(cur uint32, size uint32) uint32 { return (cur + 1) % size } // PrintDetails diplays details for debugging purpose. func (r *RingBuffer) PrintDetails() { log.Debugf("Start Position = %d, End Position = %d, Buffer Size = %d", r.start, r.end, r.Size) } ================================================ FILE: pkg/controller/controller.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package controller import ( "encoding/json" "fmt" "os" "os/signal" "syscall" "time" log "github.com/Sirupsen/logrus" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" subscriber_v1 "github.com/vmware/purser/pkg/apis/subscriber/v1" apps_v1beta1 "k8s.io/api/apps/v1beta1" batch_v1 "k8s.io/api/batch/v1" api_v1 "k8s.io/api/core/v1" ext_v1beta1 "k8s.io/api/extensions/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" ) // Kubeclient is kubernetes Clientset var Kubeclient *kubernetes.Clientset // Controller holds Kubernetes controller components type Controller struct { clientset kubernetes.Interface queue workqueue.RateLimitingInterface informer cache.SharedIndexInformer conf *Config } // Event indicate the informerEvent type Event struct { key string eventType string resourceType string data interface{} captureTime meta_v1.Time } // Start runs the controller goroutine. // nolint: gocyclo, interfacer func Start(conf *Config) { Kubeclient = conf.Kubeclient if conf.Resource.Pod { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.CoreV1().Pods(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.CoreV1().Pods(meta_v1.NamespaceAll).Watch(options) }, }, &api_v1.Pod{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Pod") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Node { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.CoreV1().Nodes().List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.CoreV1().Nodes().Watch(options) }, }, &api_v1.Node{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Node") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.PersistentVolume { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.CoreV1().PersistentVolumes().List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.CoreV1().PersistentVolumes().Watch(options) }, }, &api_v1.PersistentVolume{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "PersistentVolume") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.PersistentVolumeClaim { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.CoreV1().PersistentVolumeClaims(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.CoreV1().PersistentVolumeClaims(meta_v1.NamespaceAll).Watch(options) }, }, &api_v1.PersistentVolumeClaim{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "PersistentVolumeClaim") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Service { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.CoreV1().Services(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.CoreV1().Services(meta_v1.NamespaceAll).Watch(options) }, }, &api_v1.Service{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Service") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.ReplicaSet { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.ExtensionsV1beta1().ReplicaSets(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.ExtensionsV1beta1().ReplicaSets(meta_v1.NamespaceAll).Watch(options) }, }, &ext_v1beta1.ReplicaSet{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "ReplicaSet") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.DaemonSet { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.ExtensionsV1beta1().DaemonSets(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.ExtensionsV1beta1().DaemonSets(meta_v1.NamespaceAll).Watch(options) }, }, &ext_v1beta1.DaemonSet{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "DaemonSet") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Deployment { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.AppsV1beta1().Deployments(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.AppsV1beta1().Deployments(meta_v1.NamespaceAll).Watch(options) }, }, &apps_v1beta1.Deployment{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Deployment") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.StatefulSet { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.AppsV1beta1().StatefulSets(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.AppsV1beta1().StatefulSets(meta_v1.NamespaceAll).Watch(options) }, }, &apps_v1beta1.StatefulSet{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "StatefulSet") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Job { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.BatchV1().Jobs(meta_v1.NamespaceAll).List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.BatchV1().Jobs(meta_v1.NamespaceAll).Watch(options) }, }, &batch_v1.Job{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Job") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Namespace { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return Kubeclient.CoreV1().Namespaces().List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return Kubeclient.CoreV1().Namespaces().Watch(options) }, }, &api_v1.Namespace{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Namespace") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Group { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return conf.Groupcrdclient.List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return conf.Groupcrdclient.Watch(options) }, }, &groups_v1.Group{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Group") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } if conf.Resource.Subscriber { informer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { return conf.Subscriberclient.List(options) }, WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { return conf.Subscriberclient.Watch(options) }, }, &subscriber_v1.Subscriber{}, 0, cache.Indexers{}, ) c := newResourceController(Kubeclient, informer, "Subscriber") c.conf = conf stopCh := make(chan struct{}) defer close(stopCh) go c.Run(stopCh) } sigterm := make(chan os.Signal, 1) signal.Notify(sigterm, syscall.SIGTERM) signal.Notify(sigterm, syscall.SIGINT) <-sigterm } func newResourceController(client kubernetes.Interface, informer cache.SharedIndexInformer, resourceType string) *Controller { queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) var newEvent Event var err error informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { newEvent.key, err = cache.MetaNamespaceKeyFunc(obj) newEvent.eventType = Create newEvent.resourceType = resourceType newEvent.captureTime = meta_v1.Now() log.Printf("Processing add to %v: %s", resourceType, newEvent.key) if err == nil { queue.Add(newEvent) } }, // TODO: Fixme UpdateFunc: func(old, new interface{}) { /*newEvent.key, err = cache.MetaNamespaceKeyFunc(old) newEvent.eventType = "update" newEvent.resourceType = resourceType log.Printf("Processing update to %v: %s", resourceType, newEvent.key) if err == nil { queue.Add(newEvent) }*/ }, DeleteFunc: func(obj interface{}) { newEvent.key, err = cache.DeletionHandlingMetaNamespaceKeyFunc(obj) newEvent.eventType = Delete newEvent.resourceType = resourceType newEvent.data = obj newEvent.captureTime = meta_v1.Now() log.Printf("Processing delete to %v: %s", resourceType, newEvent.key) if err == nil { queue.Add(newEvent) } }, }) return &Controller{ clientset: client, informer: informer, queue: queue, } } // Run initiates the controller func (c *Controller) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() go c.informer.Run(stopCh) if !cache.WaitForCacheSync(stopCh, c.HasSynced) { utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) return } log.Println("Purser controller synced and ready") wait.Until(c.runWorker, time.Second, stopCh) } // HasSynced is required for the cache.Controller interface. func (c *Controller) HasSynced() bool { return c.informer.HasSynced() } // LastSyncResourceVersion is required for the cache.Controller interface. func (c *Controller) LastSyncResourceVersion() string { return c.informer.LastSyncResourceVersion() } func (c *Controller) runWorker() { for c.processNextItem() { // continue looping } } func (c *Controller) processNextItem() bool { newEvent, quit := c.queue.Get() if quit { return false } defer c.queue.Done(newEvent) err := c.processItem(newEvent.(Event)) if err == nil { c.queue.Forget(newEvent) } else { log.Printf("Error processing %s (giving up): %v", newEvent.(Event).key, err) c.queue.Forget(newEvent) utilruntime.HandleError(err) } return true } func (c *Controller) processItem(newEvent Event) error { obj, _, err := c.informer.GetIndexer().GetByKey(newEvent.key) if err != nil { return fmt.Errorf("error fetching object with key %s from store: %v", newEvent.key, err) } // process events based on its type switch newEvent.eventType { case Create: str, err := json.Marshal(obj) if err != nil { log.Errorf("Error marshalling object %s", obj) } payload := &Payload{Key: newEvent.key, EventType: newEvent.eventType, ResourceType: newEvent.resourceType, CloudType: "aws", Data: string(str), CaptureTime: newEvent.captureTime} c.conf.RingBuffer.Put(payload) return nil case Update: // TODO: Decide on what needs to be propagated. return nil case Delete: str, err := json.Marshal(newEvent.data) if err != nil { log.Errorf("Error marshalling object %s", newEvent.data) } payload := &Payload{Key: newEvent.key, EventType: newEvent.eventType, ResourceType: newEvent.resourceType, CloudType: "aws", Data: string(str), CaptureTime: newEvent.captureTime} c.conf.RingBuffer.Put(payload) return nil } return nil } ================================================ FILE: pkg/controller/controller_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package controller import ( "os" "os/signal" "syscall" "testing" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/client" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" subscriber_v1 "github.com/vmware/purser/pkg/client/clientset/typed/subscriber/v1" ) // TestCrdFlow executes the CRD flow. func TestCrdFlow(t *testing.T) { clientset, clusterConfig := client.GetAPIExtensionClient("") subcrdclient := subscriber_v1.NewSubscriberClient(clientset, clusterConfig) ListSubscriberCrdInstances(subcrdclient) sigterm := make(chan os.Signal, 1) signal.Notify(sigterm, syscall.SIGTERM) signal.Notify(sigterm, syscall.SIGINT) <-sigterm } // ListSubscriberCrdInstances fetches list of subscriber CRD instances. func ListSubscriberCrdInstances(crdclient *subscriber_v1.SubscriberClient) { items, err := crdclient.List(meta_v1.ListOptions{}) if err != nil { panic(err) } log.Printf("List:\n%v\n", items) } ================================================ FILE: pkg/controller/dgraph/dgraph.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package dgraph import ( "context" "encoding/json" "fmt" log "github.com/Sirupsen/logrus" "github.com/dgraph-io/dgo" "github.com/dgraph-io/dgo/protos/api" "github.com/vmware/purser/pkg/controller/utils" "google.golang.org/grpc" ) // mutation types const ( CREATE = "create" UPDATE = "update" DELETE = "delete" ) // Dgraph variables var ( client *dgo.Dgraph connection *grpc.ClientConn ) // ID maps the external ID used in Dgraph to the UID type ID struct { Xid string `json:"xid,omitempty"` UID string `json:"uid,omitempty"` } // Start opens and creates schema in dgraph func Start(url string, port string) { err := Open(url + ":" + port) if err != nil { log.Errorf("error while opening connection to Dgraph: %v", err) } err = CreateSchema() if err != nil { log.Errorf("error while creating schema: %v", err) } } // Open creates and establishes a new Dgraph connection func Open(url string) error { conn, err := grpc.Dial(url, grpc.WithInsecure()) if err != nil { return err } connection = conn dc := api.NewDgraphClient(connection) client = dgo.NewDgraphClient(dc) return nil } // Close terminates the Dgraph connection func Close() { err := connection.Close() if err != nil { fmt.Println("Error closing connection to Dgraph ", err) } } // CreateSchema sets the Dgraph schema func CreateSchema() error { op := &api.Operation{} op.Schema = ` name: string @index(term) . username: string @index(term) . xid: string @index(term) . startTime: dateTime @index(hour) . endTime: dateTime @index(hour) . isService: bool . isPod: bool . isContainer: bool . isProc: bool . isGroup: bool . isNodePrice: bool . isStoragePrice: bool . isRateCard: bool . isLogin: bool . pod: uid @reverse . namespace: uid @reverse . deployment: uid @reverse . replicaset: uid @reverse . statefulset: uid @reverse . container: uid @reverse . service: uid @reverse . node: uid @reverse . pv: uid @reverse . daemonset: uid @reverse . job: uid @reverse . label: uid @reverse . key: string @index(term) . value: string @index(term) . cpu: float . cpuRequest: float . cpuLimit: float . cpuCapacity: float . cpuPrice: float . memory: float . memoryRequest: float . memoryLimit: float . memoryCapacity: float . memoryPrice: float . storage: float . storageRequest: float . storageLimit: float . storageCapacity: float . storagePrice: float . mtdCPU: float . mtdCPUCost: float . mtdCost: float . mtdMemory: float . mtdMemoryCost: float . price: float . podsCount: int . ` ctx := context.Background() err := client.Alter(ctx, op) return err } // GetUID returns the UID of the node in the Dgraph // returns empty string if error has occurred func GetUID(id string, nodeType string) string { query := `query Me($id:string, $nodeType:string) { getUid(func: eq(xid, $id)) @filter(has(` + nodeType + `)) { uid } }` ctx := context.Background() variables := make(map[string]string) variables["$nodeType"] = nodeType variables["$id"] = id resp, err := client.NewReadOnlyTxn().QueryWithVars(ctx, query, variables) if err != nil { log.Printf("failed to fetch UID from Dgraph %v", err) return "" } return unmarshalDgraphResponse(resp, id) } // ExecuteQueryRaw given a query and it fetches and writes result into interface func ExecuteQueryRaw(query string) ([]byte, error) { log.Debugf("query: (%v)", query) ctx := context.Background() resp, err := client.NewTxn().Query(ctx, query) if err != nil { log.Error(err) return nil, err } return resp.Json, nil } // ExecuteQuery given a query and it fetches and writes result into interface func ExecuteQuery(query string, root interface{}) error { respJSON, err := ExecuteQueryRaw(query) if err != nil { return err } err = json.Unmarshal(respJSON, root) if err != nil { log.Fatal(err) return err } return nil } // MutateNode mutates a Dgraph transaction func MutateNode(data interface{}, mutateType string) (*api.Assigned, error) { bytes := utils.JSONMarshal(data) if bytes == nil { return nil, fmt.Errorf("unable to marshal data: %v", data) } mu := &api.Mutation{ CommitNow: true, } switch mutateType { case DELETE: mu.DeleteJson = bytes default: mu.SetJson = bytes } ctx := context.Background() return client.NewTxn().Mutate(ctx, mu) } // unmarshalDgraphResponse returns empty string if error has occurred func unmarshalDgraphResponse(resp *api.Response, id string) string { type Root struct { IDs []ID `json:"getUid"` } var r Root err := json.Unmarshal(resp.Json, &r) if err != nil { log.Debugf("failed to marshal Dgraph response %v", err) return "" } if len(r.IDs) == 0 { log.Debugf("id %s is not in dgraph", id) return "" } return r.IDs[0].UID } ================================================ FILE: pkg/controller/dgraph/login.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package dgraph import ( "github.com/Sirupsen/logrus" "golang.org/x/crypto/bcrypt" ) // Login structure type Login struct { ID IsLogin bool `json:"isLogin,omitempty"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` } // Login constants const ( DefaultUsername = "admin" DefaultPassword = "purser!123" DefaultLoginXID = "purser-login-xid" IsLogin = "isLogin" ) // StoreLogin ... func StoreLogin() { uid := GetUID(DefaultLoginXID, IsLogin) if uid == "" { hashedPassword, err := bcrypt.GenerateFromPassword([]byte(DefaultPassword), bcrypt.MinCost) if err != nil { logrus.Errorf("error while hashing login information") } login := Login{ ID: ID{Xid: DefaultLoginXID}, IsLogin: true, Username: DefaultUsername, Password: string(hashedPassword), } _, err = MutateNode(login, CREATE) if err != nil { logrus.Errorf("error while storing login information") } } } ================================================ FILE: pkg/controller/dgraph/models/constants.go ================================================ package models // Cost and other cloud constants const ( // Cost constants DefaultCPUCostPerCPUPerHour = "0.024" DefaultMemCostPerGBPerHour = "0.01" DefaultStorageCostPerGBPerHour = "0.00013888888" DefaultCPUCostInFloat64 = 0.024 DefaultMemCostInFloat64 = 0.01 DefaultStorageCostInFloat64 = 0.00013888888 // Cloud provider constants AWS = "aws" // Time constants HoursInMonth = 720 // Other constants PriceError = -1.0 ) ================================================ FILE: pkg/controller/dgraph/models/container.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "fmt" "time" log "github.com/Sirupsen/logrus" "github.com/dgraph-io/dgo/protos/api" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/utils" api_v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) // Dgraph Model Constants const ( IsContainer = "isContainer" ) // Container schema in dgraph type Container struct { dgraph.ID IsContainer bool `json:"isContainer,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Pod Pod `json:"pod,omitempty"` Procs []*Proc `json:"procs,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` CPURequest float64 `json:"cpuRequest,omitempty"` CPULimit float64 `json:"cpuLimit,omitempty"` MemoryRequest float64 `json:"memoryRequest,omitempty"` MemoryLimit float64 `json:"memoryLimit,omitempty"` Type string `json:"type,omitempty"` } func newContainer(container api_v1.Container, podUID, namespaceUID string, pod api_v1.Pod) (*api.Assigned, error) { containerXid := pod.Namespace + ":" + pod.Name + ":" + container.Name requests := container.Resources.Requests limits := container.Resources.Limits c := &Container{ ID: dgraph.ID{Xid: containerXid}, Name: "container-" + container.Name, IsContainer: true, Type: "container", StartTime: pod.GetCreationTimestamp().Time.Format(time.RFC3339), Pod: Pod{ID: dgraph.ID{UID: podUID, Xid: pod.Namespace + ":" + pod.Name}}, CPURequest: utils.ConvertToFloat64CPU(requests.Cpu()), CPULimit: utils.ConvertToFloat64CPU(limits.Cpu()), MemoryRequest: utils.ConvertToFloat64GB(requests.Memory()), MemoryLimit: utils.ConvertToFloat64GB(limits.Memory()), } if namespaceUID != "" { c.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: pod.Namespace}} } return dgraph.MutateNode(c, dgraph.CREATE) } // StoreAndRetrieveContainersAndMetrics fetchs the list of containers in given pod // Create a new container in dgraph if container is not in it. func StoreAndRetrieveContainersAndMetrics(pod api_v1.Pod, podUID, namespaceUID string) ([]*Container, Metrics) { containers := []*Container{} cpuRequest := &resource.Quantity{} memoryRequest := &resource.Quantity{} cpuLimit := &resource.Quantity{} memoryLimit := &resource.Quantity{} for _, c := range pod.Spec.Containers { container, err := storeContainerIfNotExist(c, pod, podUID, namespaceUID) if err == nil { containers = append(containers, container) } requests := c.Resources.Requests limits := c.Resources.Limits utils.AddResourceAToResourceB(requests.Cpu(), cpuRequest) utils.AddResourceAToResourceB(requests.Memory(), memoryRequest) utils.AddResourceAToResourceB(limits.Cpu(), cpuLimit) utils.AddResourceAToResourceB(limits.Memory(), memoryLimit) } return containers, Metrics{ CPURequest: utils.ConvertToFloat64CPU(cpuRequest), CPULimit: utils.ConvertToFloat64CPU(cpuLimit), MemoryRequest: utils.ConvertToFloat64GB(memoryRequest), MemoryLimit: utils.ConvertToFloat64GB(memoryLimit), } } // StoreContainerProcessEdge ... func StoreContainerProcessEdge(containerXID string, procsXIDs []string) error { containerUID := dgraph.GetUID(containerXID, IsContainer) if containerUID == "" { return fmt.Errorf("container: %s not persisted in dgraph", containerXID) } procs := retrieveProcessesFromProcessesXIDs(procsXIDs) container := Container{ ID: dgraph.ID{UID: containerUID, Xid: containerXID}, Procs: procs, } _, err := dgraph.MutateNode(container, dgraph.UPDATE) return err } func storeContainerIfNotExist(c api_v1.Container, pod api_v1.Pod, podUID, namespaceUID string) (*Container, error) { podXid := pod.Namespace + ":" + pod.Name containerXid := podXid + ":" + c.Name containerUID := dgraph.GetUID(containerXid, IsContainer) var container *Container if containerUID == "" { assigned, err := newContainer(c, podUID, namespaceUID, pod) if err != nil { log.Errorf("Unable to create container: %s", containerXid) return container, err } log.Infof("Container with xid: (%s) persisted in dgraph", containerXid) containerUID = assigned.Uids["blank-0"] } container = &Container{ ID: dgraph.ID{UID: containerUID, Xid: containerXid}, } return container, nil } func deleteContainersInTerminatedPod(containers []*Container, endTime time.Time) { for _, container := range containers { container.EndTime = endTime.Format(time.RFC3339) container.Xid += container.EndTime container.Name += "*" + container.EndTime // * in name indicates dead resources } _, err := dgraph.MutateNode(containers, dgraph.UPDATE) if err != nil { log.Error(err) } deleteProcessesInTerminatedContainers(containers) } ================================================ FILE: pkg/controller/dgraph/models/daemonset.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" ext_v1beta1 "k8s.io/api/extensions/v1beta1" ) // Dgraph Model Constants const ( IsDaemonset = "isDaemonset" ) // Daemonset schema in dgraph type Daemonset struct { dgraph.ID IsDaemonset bool `json:"isDaemonset,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Pods []*Pod `json:"pod,omitempty"` Type string `json:"type,omitempty"` } func createDaemonsetObject(daemonset ext_v1beta1.DaemonSet) Daemonset { newDaemonset := Daemonset{ Name: "daemonset-" + daemonset.Name, IsDaemonset: true, Type: "daemonset", ID: dgraph.ID{Xid: daemonset.Namespace + ":" + daemonset.Name}, StartTime: daemonset.GetCreationTimestamp().Time.Format(time.RFC3339), } namespaceUID := CreateOrGetNamespaceByID(daemonset.Namespace) if namespaceUID != "" { newDaemonset.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: daemonset.Namespace}} } daemonsetDeletionTimestamp := daemonset.GetDeletionTimestamp() if !daemonsetDeletionTimestamp.IsZero() { newDaemonset.EndTime = daemonsetDeletionTimestamp.Time.Format(time.RFC3339) newDaemonset.Xid += newDaemonset.EndTime newDaemonset.Name += "*" + newDaemonset.EndTime } return newDaemonset } // StoreDaemonset create a new daemonset in the Dgraph and updates if already present. func StoreDaemonset(daemonset ext_v1beta1.DaemonSet) (string, error) { xid := daemonset.Namespace + ":" + daemonset.Name uid := dgraph.GetUID(xid, IsDaemonset) newDaemonset := createDaemonsetObject(daemonset) if uid != "" { newDaemonset.UID = uid } assigned, err := dgraph.MutateNode(newDaemonset, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } // CreateOrGetDaemonsetByID returns the uid of namespace if exists, // otherwise creates the daemonset and returns uid. func CreateOrGetDaemonsetByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsDaemonset) if uid != "" { return uid } d := Daemonset{ ID: dgraph.ID{Xid: xid}, Name: xid, IsDaemonset: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/deployment.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" "log" "github.com/vmware/purser/pkg/controller/dgraph" apps_v1beta1 "k8s.io/api/apps/v1beta1" ) // Dgraph Model Constants const ( IsDeployment = "isDeployment" ) // Deployment schema in dgraph type Deployment struct { dgraph.ID IsDeployment bool `json:"isDeployment,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Pods []*Pod `json:"pod,omitempty"` Type string `json:"type,omitempty"` } func createDeploymentObject(deployment apps_v1beta1.Deployment) Deployment { newDeployment := Deployment{ Name: "deployment-" + deployment.Name, IsDeployment: true, Type: "deployment", ID: dgraph.ID{Xid: deployment.Namespace + ":" + deployment.Name}, StartTime: deployment.GetCreationTimestamp().Time.Format(time.RFC3339), } namespaceUID := CreateOrGetNamespaceByID(deployment.Namespace) if namespaceUID != "" { newDeployment.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: deployment.Namespace}} } deploymentDeletionTimestamp := deployment.GetDeletionTimestamp() if !deploymentDeletionTimestamp.IsZero() { newDeployment.EndTime = deploymentDeletionTimestamp.Time.Format(time.RFC3339) newDeployment.Xid += newDeployment.EndTime newDeployment.Name += "*" + newDeployment.EndTime } return newDeployment } // StoreDeployment create a new deployment in the Dgraph and updates if already present. func StoreDeployment(deployment apps_v1beta1.Deployment) (string, error) { xid := deployment.Namespace + ":" + deployment.Name uid := dgraph.GetUID(xid, IsDeployment) newDeployment := createDeploymentObject(deployment) if uid != "" { newDeployment.UID = uid } assigned, err := dgraph.MutateNode(newDeployment, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } // CreateOrGetDeploymentByID returns the uid of namespace if exists, // otherwise creates the deployment and returns uid. func CreateOrGetDeploymentByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsDeployment) if uid != "" { return uid } d := Deployment{ ID: dgraph.ID{Xid: xid}, Name: xid, IsDeployment: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/group.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "github.com/Sirupsen/logrus" "github.com/dgraph-io/dgo/protos/api" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/utils" ) // Group constants const ( IsGroup = "isGroup" groupXIDPrefix = "purser-group-" ) // Group schema in dgraph type Group struct { dgraph.ID IsGroup bool `json:"isGroup,omitempty"` Name string `json:"name,omitempty"` PodsCount int `json:"podsCount,omitempty"` MtdCPU float64 `json:"mtdCPU,omitempty"` MtdMemory float64 `json:"mtdMemory,omitempty"` MtdStorage float64 `json:"mtdStorage,omitempty"` CPU float64 `json:"cpu,omitempty"` Memory float64 `json:"memory,omitempty"` Storage float64 `json:"storage,omitempty"` MtdCPUCost float64 `json:"mtdCPUCost,omitempty"` MtdMemoryCost float64 `json:"mtdMemoryCost,omitempty"` MtdStorageCost float64 `json:"mtdStorageCost,omitempty"` MtdCost float64 `json:"mtdCost,omitempty"` ProjectedCPUCost float64 `json:"projectedCPUCost,omitempty"` ProjectedMemoryCost float64 `json:"projectedMemoryCost,omitempty"` ProjectedStorageCost float64 `json:"projectedStorageCost,omitempty"` ProjectedCost float64 `json:"projectedCost,omitempty"` LastMonthCPUCost float64 `json:"lastMonthCPUCost,omitempty"` LastMonthMemoryCost float64 `json:"lastMonthMemoryCost,omitempty"` LastMonthStorageCost float64 `json:"lastMonthStorageCost,omitempty"` LastMonthCost float64 `json:"lastMonthCost,omitempty"` LastLastMonthCPUCost float64 `json:"lastLastMonthCPUCost,omitempty"` LastLastMonthMemoryCost float64 `json:"lastLastMonthMemoryCost,omitempty"` LastLastMonthStorageCost float64 `json:"lastLastMonthStorageCost,omitempty"` LastLastMonthCost float64 `json:"lastLastMonthCost,omitempty"` } // CreateOrUpdateGroup updates group if it is already present in dgraph else it creates one func CreateOrUpdateGroup(group *groups_v1.Group, podsCount int) (*api.Assigned, error) { xid := groupXIDPrefix + group.Name uid := dgraph.GetUID(xid, IsGroup) hoursRemainingInCurrentMonth := utils.GetHoursRemainingInCurrentMonth() grp := Group{ ID: dgraph.ID{Xid: xid}, IsGroup: true, Name: group.Name, PodsCount: podsCount, MtdCPU: group.Spec.MTDMetrics.CPURequest, MtdMemory: group.Spec.MTDMetrics.MemoryRequest, MtdStorage: group.Spec.MTDMetrics.StorageClaim, CPU: group.Spec.PITMetrics.CPURequest, Memory: group.Spec.PITMetrics.MemoryRequest, Storage: group.Spec.PITMetrics.StorageClaim, MtdCPUCost: group.Spec.MTDCost.CPUCost, MtdMemoryCost: group.Spec.MTDCost.MemoryCost, MtdStorageCost: group.Spec.MTDCost.StorageCost, MtdCost: group.Spec.MTDCost.TotalCost, ProjectedCPUCost: group.Spec.MTDCost.CPUCost + group.Spec.PerHourCost.CPUCost*hoursRemainingInCurrentMonth, ProjectedMemoryCost: group.Spec.MTDCost.MemoryCost + group.Spec.PerHourCost.MemoryCost*hoursRemainingInCurrentMonth, ProjectedStorageCost: group.Spec.MTDCost.StorageCost + group.Spec.PerHourCost.StorageCost*hoursRemainingInCurrentMonth, ProjectedCost: group.Spec.MTDCost.TotalCost + group.Spec.PerHourCost.TotalCost*hoursRemainingInCurrentMonth, LastMonthCPUCost: group.Spec.LastMonthCost.CPUCost, LastMonthMemoryCost: group.Spec.LastMonthCost.MemoryCost, LastMonthStorageCost: group.Spec.LastMonthCost.StorageCost, LastMonthCost: group.Spec.LastMonthCost.TotalCost, LastLastMonthCPUCost: group.Spec.LastLastMonthCost.CPUCost, LastLastMonthMemoryCost: group.Spec.LastLastMonthCost.MemoryCost, LastLastMonthStorageCost: group.Spec.LastLastMonthCost.StorageCost, LastLastMonthCost: group.Spec.LastLastMonthCost.TotalCost, } if uid != "" { grp.ID = dgraph.ID{Xid: xid, UID: uid} } return dgraph.MutateNode(grp, dgraph.CREATE) } // DeleteGroup deletes group from dgraph func DeleteGroup(name string) { xid := groupXIDPrefix + name uid := dgraph.GetUID(xid, IsGroup) if uid != "" { grp := Group{ID: dgraph.ID{UID: uid}} _, err := dgraph.MutateNode(grp, dgraph.DELETE) if err != nil { logrus.Errorf("error while deleting group: %v, err: %v", name, err) } return } logrus.Infof("Group: %s not yet persisted", name) } ================================================ FILE: pkg/controller/dgraph/models/job.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" batch_v1 "k8s.io/api/batch/v1" ) // Dgraph Model Constants const ( IsJob = "isJob" ) // Job schema in dgraph type Job struct { dgraph.ID IsJob bool `json:"isJob,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Pods []*Pod `json:"pod,omitempty"` Type string `json:"type,omitempty"` } func createJobObject(job batch_v1.Job) Job { newJob := Job{ Name: "job-" + job.Name, IsJob: true, Type: "job", ID: dgraph.ID{Xid: job.Namespace + ":" + job.Name}, StartTime: job.GetCreationTimestamp().Time.Format(time.RFC3339), } namespaceUID := CreateOrGetNamespaceByID(job.Namespace) if namespaceUID != "" { newJob.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: job.Namespace}} } jobDeletionTimestamp := job.GetDeletionTimestamp() if !jobDeletionTimestamp.IsZero() { newJob.EndTime = jobDeletionTimestamp.Time.Format(time.RFC3339) newJob.Xid += newJob.EndTime newJob.Name += "*" + newJob.EndTime } return newJob } // StoreJob create a new daemonset in the Dgraph and updates if already present. func StoreJob(job batch_v1.Job) (string, error) { xid := job.Namespace + ":" + job.Name uid := dgraph.GetUID(xid, IsJob) newJob := createJobObject(job) if uid != "" { newJob.UID = uid } assigned, err := dgraph.MutateNode(newJob, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } // CreateOrGetJobByID returns the uid of namespace if exists, // otherwise creates the job and returns uid. func CreateOrGetJobByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsJob) if uid != "" { return uid } d := Job{ ID: dgraph.ID{Xid: xid}, Name: xid, IsJob: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/label.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" ) // Dgraph Model Constants const ( Islabel = "isLabel" ) // Label structure for Key:Value type Label struct { dgraph.ID IsLabel bool `json:"isLabel,omitempty"` Key string `json:"key,omitempty"` Value string `json:"value,omitempty"` } // GetLabel if label is not in dgraph it creates and returns Label object func GetLabel(key, value string) *Label { xid := getXIDOfLabel(key, value) uid := CreateOrGetLabelByID(key, value) return &Label{ ID: dgraph.ID{Xid: xid, UID: uid}, } } // CreateOrGetLabelByID if label is not in dgraph it creates and returns uid of label func CreateOrGetLabelByID(key, value string) string { xid := getXIDOfLabel(key, value) uid := dgraph.GetUID(xid, Islabel) if uid == "" { // create new label and get its uid uid = createLabelObject(key, value) } return uid } func getXIDOfLabel(key, value string) string { return "label-" + key + "-" + value } func createLabelObject(key, value string) string { xid := getXIDOfLabel(key, value) newLabel := Label{ ID: dgraph.ID{Xid: xid}, IsLabel: true, Key: key, Value: value, } assigned, err := dgraph.MutateNode(newLabel, dgraph.CREATE) if err != nil { logrus.Fatal(err) return "" } logrus.Debugf("created label in dgraph key: (%v), value: (%v)", newLabel.Key, newLabel.Value) return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/namespace.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" api_v1 "k8s.io/api/core/v1" ) // Dgraph Model Constants const ( IsNamespace = "isNamespace" ) // Namespace schema in dgraph type Namespace struct { dgraph.ID IsNamespace bool `json:"isNamespace,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Type string `json:"type,omitempty"` } func newNamespace(namespace api_v1.Namespace) Namespace { ns := Namespace{ ID: dgraph.ID{Xid: namespace.Name}, Name: "namespace-" + namespace.Name, IsNamespace: true, Type: "namespace", StartTime: namespace.GetCreationTimestamp().Time.Format(time.RFC3339), } nsDeletionTimestamp := namespace.GetDeletionTimestamp() if !nsDeletionTimestamp.IsZero() { ns.EndTime = nsDeletionTimestamp.Time.Format(time.RFC3339) ns.Xid += ns.EndTime ns.Name += "*" + ns.EndTime } return ns } // CreateOrGetNamespaceByID returns the uid of namespace if exists, // otherwise creates the namespace and returns uid. func CreateOrGetNamespaceByID(xid string) string { if xid == "" { log.Error("Namespace is empty") return "" } uid := dgraph.GetUID(xid, IsNamespace) if uid != "" { return uid } ns := Namespace{ ID: dgraph.ID{Xid: xid}, Name: xid, IsNamespace: true, } assigned, err := dgraph.MutateNode(ns, dgraph.CREATE) if err != nil { log.Error(err) return "" } log.Infof("Namespace with xid: (%s) persisted", xid) return assigned.Uids["blank-0"] } // StoreNamespace create a new namespace in the Dgraph if it is not present. func StoreNamespace(namespace api_v1.Namespace) (string, error) { xid := namespace.Name uid := dgraph.GetUID(xid, IsNamespace) ns := newNamespace(namespace) if uid != "" { ns.UID = uid } assigned, err := dgraph.MutateNode(ns, dgraph.CREATE) if err != nil { return "", err } if uid == "" { log.Infof("Namespace with xid: (%s) persisted", xid) } return assigned.Uids["blank-0"], nil } ================================================ FILE: pkg/controller/dgraph/models/node.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" "fmt" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/utils" api_v1 "k8s.io/api/core/v1" ) // Dgraph Model Constants const ( IsNode = "isNode" DefaultNodeInstance = "purser-default" DefaultNodeOS = "purser-default" InstanceTypeLabelKey = "beta.kubernetes.io/instance-type" OSLabelKey = "beta.kubernetes.io/os" ) // Node schema in dgraph type Node struct { dgraph.ID IsNode bool `json:"isNode,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Pods []*Pod `json:"pods,omitempty"` CPUCapacity float64 `json:"cpuCapacity,omitempty"` MemoryCapacity float64 `json:"memoryCapacity,omitempty"` Type string `json:"type,omitempty"` InstanceType string `json:"instanceType,omitempty"` OS string `json:"os,omitempty"` CPUPrice float64 `json:"cpuPrice,omitempty"` MemoryPrice float64 `json:"memoryPrice,omitempty"` } func createNodeObject(node api_v1.Node) Node { newNode := Node{ Name: "node-" + node.Name, IsNode: true, Type: "node", ID: dgraph.ID{Xid: node.Name}, StartTime: node.GetCreationTimestamp().Time.Format(time.RFC3339), CPUCapacity: utils.ConvertToFloat64CPU(node.Status.Capacity.Cpu()), MemoryCapacity: utils.ConvertToFloat64GB(node.Status.Capacity.Memory()), } instanceType, os := getInstanceTypeAndOS(node) newNode.InstanceType = instanceType newNode.OS = os log.Debugf("node: %s, instanceType: %s, os: %s", node.Name, newNode.InstanceType, newNode.OS) nodeDeletionTimestamp := node.GetDeletionTimestamp() if !nodeDeletionTimestamp.IsZero() { newNode.EndTime = nodeDeletionTimestamp.Time.Format(time.RFC3339) newNode.Xid += newNode.EndTime newNode.Name += "*" + newNode.EndTime } return newNode } // createOrGetNodeByID create and returns the node if not present, otherwise simply returns node. func createOrGetNodeByID(xid string) (string, error) { if xid == "" { return "", fmt.Errorf("node xid is empty") } uid := dgraph.GetUID(xid, IsNode) if uid != "" { return uid, nil } newNode := Node{ Name: xid, IsNode: true, ID: dgraph.ID{Xid: xid}, } assigned, err := dgraph.MutateNode(newNode, dgraph.CREATE) if err != nil { return "", err } log.Infof("Node with xid: (%s) persisted", xid) return assigned.Uids["blank-0"], nil } // StoreNode create a new node in the Dgraph if it is not present. func StoreNode(node api_v1.Node) (string, error) { xid := node.Name uid := dgraph.GetUID(xid, IsNode) newNode := createNodeObject(node) if uid != "" { newNode.UID = uid } newNode.CPUPrice, newNode.MemoryPrice = getPricePerUnitResourceFromNodePrice(newNode) assigned, err := dgraph.MutateNode(newNode, dgraph.CREATE) if err != nil { return "", err } if uid == "" { log.Infof("Node with xid: (%s) persisted", xid) } return assigned.Uids["blank-0"], nil } // getInstanceTypeAndOS returns instance and os of a node func getInstanceTypeAndOS(node api_v1.Node) (string, string) { nodeLabels := node.GetLabels() instanceType := DefaultNodeInstance os := DefaultNodeOS if value, isPresent := nodeLabels[InstanceTypeLabelKey]; isPresent { instanceType = value } if value, isPresent := nodeLabels[OSLabelKey]; isPresent { os = value } return instanceType, os } ================================================ FILE: pkg/controller/dgraph/models/pod.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "fmt" "time" log "github.com/Sirupsen/logrus" "github.com/dgraph-io/dgo/protos/api" "github.com/vmware/purser/pkg/controller/dgraph" api_v1 "k8s.io/api/core/v1" ) // Dgraph Model Constants const ( IsPod = "isPod" ) // Pod schema in dgraph type Pod struct { dgraph.ID IsPod bool `json:"isPod,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Containers []*Container `json:"containers,omitempty"` Pods []*Pod `json:"pod,omitempty"` Count float64 `json:"pod|count,omitempty"` Node *Node `json:"node,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Deployment *Deployment `json:"deployment,omitempty"` Replicaset *Replicaset `json:"replicaset,omitempty"` Statefulset *Statefulset `json:"statefulset,omitempty"` Daemonset *Daemonset `json:"daemonset,omitempty"` Job *Job `json:"job,omitempty"` Pvcs []*PersistentVolumeClaim `json:"pvc,omitempty"` CPURequest float64 `json:"cpuRequest,omitempty"` CPULimit float64 `json:"cpuLimit,omitempty"` MemoryRequest float64 `json:"memoryRequest,omitempty"` MemoryLimit float64 `json:"memoryLimit,omitempty"` StorageRequest float64 `json:"storageRequest,omitempty"` Type string `json:"type,omitempty"` Cid []Service `json:"cid,omitempty"` Labels []*Label `json:"label,omitempty"` CPUPrice float64 `json:"cpuPrice,omitempty"` MemoryPrice float64 `json:"memoryPrice,omitempty"` } // Metrics ... type Metrics struct { CPURequest float64 CPULimit float64 MemoryRequest float64 MemoryLimit float64 } // newPod creates a new node for the pod in the Dgraph func newPod(k8sPod api_v1.Pod) (*api.Assigned, error) { pod := Pod{ Name: "pod-" + k8sPod.Name, IsPod: true, Type: "pod", ID: dgraph.ID{Xid: k8sPod.Namespace + ":" + k8sPod.Name}, StartTime: k8sPod.GetCreationTimestamp().Time.Format(time.RFC3339), } nodeUID, err := createOrGetNodeByID(k8sPod.Spec.NodeName) if err == nil { pod.Node = &Node{ID: dgraph.ID{UID: nodeUID, Xid: k8sPod.Spec.NodeName}} } namespaceUID := CreateOrGetNamespaceByID(k8sPod.Namespace) if namespaceUID != "" { pod.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: k8sPod.Namespace}} } pod.Pvcs, pod.StorageRequest = getPodVolumes(k8sPod) setPodOwners(&pod, k8sPod) return dgraph.MutateNode(pod, dgraph.CREATE) } // StorePod updates the pod details and create it a new node if not exists. // It also populates Containers of a pod. func StorePod(k8sPod api_v1.Pod) error { if k8sPod.Namespace == "" || k8sPod.Name == "" { return fmt.Errorf("pod name/namespace is empty, name: %s, namesapce: %s", k8sPod.Name, k8sPod.Namespace) } xid := k8sPod.Namespace + ":" + k8sPod.Name uid := dgraph.GetUID(xid, IsPod) var pod Pod if uid == "" { assigned, err := newPod(k8sPod) if err != nil { return err } log.Infof("Pod with xid: (%s) persisted in dgraph", xid) uid = assigned.Uids["blank-0"] } podDeletedTimestamp := k8sPod.GetDeletionTimestamp() if !podDeletedTimestamp.IsZero() { endTime := podDeletedTimestamp.Time.Format(time.RFC3339) pod = Pod{ ID: dgraph.ID{Xid: xid + endTime, UID: uid}, EndTime: endTime, Name: "pod-" + k8sPod.Name + "*" + endTime, } podData := RetrievePodWithContainers(xid) deleteContainersInTerminatedPod(podData.Containers, podDeletedTimestamp.Time) } else { namespaceUID := CreateOrGetNamespaceByID(k8sPod.Namespace) containers, metrics := StoreAndRetrieveContainersAndMetrics(k8sPod, uid, namespaceUID) pod = Pod{ ID: dgraph.ID{Xid: xid, UID: uid}, Name: "pod-" + k8sPod.Name, Containers: containers, CPURequest: metrics.CPURequest, CPULimit: metrics.CPULimit, MemoryRequest: metrics.MemoryRequest, MemoryLimit: metrics.MemoryLimit, } populatePodLabels(&pod, k8sPod.Labels) } // store/update CPUPrice, MemoryPrice pod.CPUPrice, pod.MemoryPrice = getPerUnitResourcePriceForNode("node-" + k8sPod.Spec.NodeName) _, err := dgraph.MutateNode(pod, dgraph.UPDATE) return err } // StorePodsInteraction store the pod interactions in Dgraph func StorePodsInteraction(sourcePodXID string, destinationPodsXIDs []string, counts []float64) error { uid := dgraph.GetUID(sourcePodXID, IsPod) if uid == "" { log.Println("Source Pod " + sourcePodXID + " is not persisted yet.") return fmt.Errorf("source pod: %s is not persisted yet", sourcePodXID) } pods := retrievePodsWithCountAsEdgeWeightFromPodsXIDs(destinationPodsXIDs, counts) source := Pod{ ID: dgraph.ID{UID: uid, Xid: sourcePodXID}, Pods: pods, } _, err := dgraph.MutateNode(source, dgraph.UPDATE) return err } func retrievePodsFromPodsXIDs(podsXIDs []string) []*Pod { pods := []*Pod{} for _, podXID := range podsXIDs { podUID := dgraph.GetUID(podXID, IsPod) if podUID == "" { log.Debugf("Pod uid is empty for pod xid: %s", podXID) continue } pod := &Pod{ ID: dgraph.ID{UID: podUID, Xid: podXID}, } pods = append(pods, pod) } return pods } func retrievePodsWithCountAsEdgeWeightFromPodsXIDs(podsXIDs []string, counts []float64) []*Pod { pods := []*Pod{} for index, podXID := range podsXIDs { podUID := dgraph.GetUID(podXID, IsPod) if podUID == "" { log.Printf("Destination pod: %s is not persisted yet", podXID) continue } pod := &Pod{ ID: dgraph.ID{UID: podUID, Xid: podXID}, Count: counts[index], } pods = append(pods, pod) } return pods } func setPodOwners(pod *Pod, k8sPod api_v1.Pod) { owners := k8sPod.GetObjectMeta().GetOwnerReferences() for _, owner := range owners { ownerXID := k8sPod.Namespace + ":" + owner.Name switch owner.Kind { case "Deployment": updateDeploymentAsPodOwner(pod, ownerXID) case "ReplicaSet": updateReplicasetAsPodOwner(pod, ownerXID) case "StatefulSet": updateStatefulsetAsPodOwner(pod, ownerXID) case "Job": updateJobAsPodOwner(pod, ownerXID) case "DaemonSet": updateDaemonsetAsPodOwner(pod, ownerXID) default: log.Error("Unknown owner type " + owner.Kind + " for pod.") } } } func updateDeploymentAsPodOwner(pod *Pod, ownerXID string) { deploymentUID := CreateOrGetDeploymentByID(ownerXID) if deploymentUID != "" { pod.Deployment = &Deployment{ID: dgraph.ID{UID: deploymentUID, Xid: ownerXID}} } } func updateReplicasetAsPodOwner(pod *Pod, ownerXID string) { replicasetUID := CreateOrGetReplicasetByID(ownerXID) if replicasetUID != "" { pod.Replicaset = &Replicaset{ID: dgraph.ID{UID: replicasetUID, Xid: ownerXID}} } } func updateStatefulsetAsPodOwner(pod *Pod, ownerXID string) { statefulsetUID := CreateOrGetStatefulsetByID(ownerXID) if statefulsetUID != "" { pod.Statefulset = &Statefulset{ID: dgraph.ID{UID: statefulsetUID, Xid: ownerXID}} } } func updateJobAsPodOwner(pod *Pod, ownerXID string) { jobUID := CreateOrGetJobByID(ownerXID) if jobUID != "" { pod.Job = &Job{ID: dgraph.ID{UID: jobUID, Xid: ownerXID}} } } func updateDaemonsetAsPodOwner(pod *Pod, ownerXID string) { daemonsetUID := CreateOrGetDaemonsetByID(ownerXID) if daemonsetUID != "" { pod.Daemonset = &Daemonset{ID: dgraph.ID{UID: daemonsetUID, Xid: ownerXID}} } } func getPodVolumes(k8sPod api_v1.Pod) ([]*PersistentVolumeClaim, float64) { podVolumes := []*PersistentVolumeClaim{} storage := 0.0 for j := 0; j < len(k8sPod.Spec.Volumes); j++ { vol := k8sPod.Spec.Volumes[j] if vol.PersistentVolumeClaim != nil { pvcXID := k8sPod.Namespace + ":" + vol.PersistentVolumeClaim.ClaimName pvcUID := CreateOrGetPersistentVolumeClaimByID(pvcXID) if pvcUID != "" { podVolumes = append(podVolumes, &PersistentVolumeClaim{ID: dgraph.ID{UID: pvcUID, Xid: pvcXID}}) pvc, err := getPVCFromUID(pvcUID) if err == nil { storage += pvc.StorageCapacity } else { log.Errorf("error while getting pvc from uid: (%v), error: (%v)", pvcUID, err) } } } } return podVolumes, storage } func populatePodLabels(pod *Pod, podLabels map[string]string) { log.Debugf("k8s pod: (%v), labels: (%v)", pod.Name, podLabels) var labels []*Label for key, value := range podLabels { labels = append(labels, GetLabel(key, value)) } pod.Labels = labels } // RetrievePodWithContainers given a name of pod it retrieves its containers func RetrievePodWithContainers(xid string) Pod { query := `query { pods(func: has(isPod)) @filter(eq(xid, "` + xid + `")) { name containers: ~pod @filter(has(isContainer)) { uid } } }` type root struct { Pods []Pod `json:"pods"` } newRoot := root{} err := dgraph.ExecuteQuery(query, &newRoot) if err != nil || len(newRoot.Pods) < 1 { log.Errorf("unable to retrieve pod with containers: %v", err) return Pod{} } return newRoot.Pods[0] } ================================================ FILE: pkg/controller/dgraph/models/pod_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "fmt" "testing" "github.com/vmware/purser/pkg/controller/dgraph" ) // TestStorePodsInteraction ... func TestStorePodsInteraction(t *testing.T) { fmt.Println("Hello World") err := dgraph.Open("127.0.0.1:9080") if err != nil { fmt.Println("Error while opening connection to Dgraph ", err) } err = dgraph.CreateSchema() if err != nil { fmt.Println("Error while creating schema ", err) } sourcePod := "weave:weave-scope-app-6d6b76b846-z92wk" destinationPods := []string{"fiaasco:ccs-billing-deployment-1-1-92-75dc8749f4-gld6q", "weave:weave-scope-agent-lbfpj"} interactionCounts := []float64{2.0} err = StorePodsInteraction(sourcePod, destinationPods, interactionCounts) if err != nil { fmt.Println("Error while building interation graph ", err) } } ================================================ FILE: pkg/controller/dgraph/models/process.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "fmt" "strings" "time" "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus" "github.com/dgraph-io/dgo/protos/api" "github.com/vmware/purser/pkg/controller/dgraph" ) // Dgraph Model Constants const ( IsProc = "isProc" ) // Proc schema in dgraph type Proc struct { dgraph.ID IsProc bool `json:"isProc,omitempty"` Name string `json:"name,omitempty"` Interacts []*Pod `json:"interacts,omitempty"` Container Container `json:"container,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Type string `json:"type,omitempty"` } func newProc(procXID, procName, containerUID, containerXID string, creationTimeStamp time.Time) (*api.Assigned, error) { newProc := Proc{ ID: dgraph.ID{Xid: procXID}, IsProc: true, Type: "process", Name: "process-" + procName, Container: Container{ID: dgraph.ID{UID: containerUID, Xid: containerXID}}, StartTime: creationTimeStamp.Format(time.RFC3339), } return dgraph.MutateNode(newProc, dgraph.CREATE) } // StoreProcess ... func StoreProcess(procXID, containerXID string, podsXIDs []string, creationTimeStamp time.Time) error { // fetch the 4th field from ns : podName : containerName : procID : procName procName := strings.Join(strings.Split(procXID, ":")[4:], "-") containerUID := dgraph.GetUID(containerXID, IsContainer) if containerUID == "" { return fmt.Errorf("container not persisted yet") } procUID := dgraph.GetUID(procXID, IsProc) if procUID == "" { assigned, err := newProc(procXID, procName, containerUID, containerXID, creationTimeStamp) if err != nil { logrus.Errorf("unable to create proc: %s", procXID) return err } log.Debugf("Process with xid: (%s) persisted in dgraph", procXID) procUID = assigned.Uids["blank-0"] } pods := retrievePodsFromPodsXIDs(podsXIDs) updatedProc := Proc{ ID: dgraph.ID{UID: procUID, Xid: procXID}, Interacts: pods, } _, err := dgraph.MutateNode(updatedProc, dgraph.UPDATE) return err } func deleteProcessesInTerminatedContainers(containers []*Container) { procs := []Proc{} for _, container := range containers { for _, proc := range container.Procs { updatedProc := Proc{ ID: dgraph.ID{UID: proc.ID.UID, Xid: proc.ID.Xid}, EndTime: container.EndTime, } procs = append(procs, updatedProc) } } _, err := dgraph.MutateNode(procs, dgraph.UPDATE) if err != nil { log.Error(err) } } func retrieveProcessesFromProcessesXIDs(procsXIDs []string) []*Proc { procs := []*Proc{} for _, procXID := range procsXIDs { procUID := dgraph.GetUID(procXID, IsProc) if procUID != "" { procs = append(procs, &Proc{ID: dgraph.ID{UID: procUID, Xid: procXID}}) } } return procs } ================================================ FILE: pkg/controller/dgraph/models/pv.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" "github.com/Sirupsen/logrus" "k8s.io/client-go/kubernetes" "log" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/utils" api_v1 "k8s.io/api/core/v1" ) // Dgraph Model Constants const ( IsPersistentVolume = "isPersistentVolume" ) // PersistentVolume schema in dgraph type PersistentVolume struct { dgraph.ID IsPersistentVolume bool `json:"isPersistentVolume,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Type string `json:"type,omitempty"` StorageCapacity float64 `json:"storageCapacity,omitempty"` StorageType string `json:"storageType,omitempty"` } func createPersistentVolumeObject(pv api_v1.PersistentVolume, client *kubernetes.Clientset) PersistentVolume { newPv := PersistentVolume{ Name: "pv-" + pv.Name, IsPersistentVolume: true, Type: "pv", ID: dgraph.ID{Xid: pv.Name}, StartTime: pv.GetCreationTimestamp().Time.Format(time.RFC3339), } capacity := pv.Spec.Capacity["storage"] newPv.StorageCapacity = utils.ConvertToFloat64GB(&capacity) newPv.StorageType = utils.GetFinalStorageTypeOfPV(pv, client) logrus.Debugf("PV: %s, storageType: %s", newPv.Name, newPv.StorageType) deletionTimestamp := pv.GetDeletionTimestamp() if !deletionTimestamp.IsZero() { newPv.EndTime = deletionTimestamp.Time.Format(time.RFC3339) newPv.Xid += newPv.EndTime newPv.Name += "*" + newPv.EndTime } return newPv } // StorePersistentVolume create a new persistent volume in the Dgraph and updates if already present. func StorePersistentVolume(pv api_v1.PersistentVolume, client *kubernetes.Clientset) (string, error) { xid := pv.Name uid := dgraph.GetUID(xid, IsPersistentVolume) newPv := createPersistentVolumeObject(pv, client) if uid != "" { newPv.UID = uid } assigned, err := dgraph.MutateNode(newPv, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } // CreateOrGetPersistentVolumeByID returns the uid of persistent volume if exists, // otherwise creates the persistent volume and returns uid. func CreateOrGetPersistentVolumeByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsPersistentVolume) if uid != "" { return uid } d := PersistentVolume{ ID: dgraph.ID{Xid: xid}, Name: xid, IsPersistentVolume: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/pvc.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" "log" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/utils" api_v1 "k8s.io/api/core/v1" ) // Dgraph Model Constants const ( IsPersistentVolumeClaim = "isPersistentVolumeClaim" ) // PersistentVolumeClaim schema in dgraph type PersistentVolumeClaim struct { dgraph.ID IsPersistentVolumeClaim bool `json:"isPersistentVolumeClaim,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Type string `json:"type,omitempty"` StorageCapacity float64 `json:"storageCapacity,omitempty"` PersistentVolume *PersistentVolume `json:"pv,omitempty"` } func createPvcObject(pvc api_v1.PersistentVolumeClaim) PersistentVolumeClaim { newPvc := PersistentVolumeClaim{ Name: "pvc-" + pvc.Name, IsPersistentVolumeClaim: true, Type: "pvc", ID: dgraph.ID{Xid: pvc.Namespace + ":" + pvc.Name}, StartTime: pvc.GetCreationTimestamp().Time.Format(time.RFC3339), } capacity := pvc.Status.Capacity["storage"] newPvc.StorageCapacity = utils.ConvertToFloat64GB(&capacity) volume := pvc.Spec.VolumeName pvUID := CreateOrGetPersistentVolumeByID(volume) if volume != "" { newPvc.PersistentVolume = &PersistentVolume{ID: dgraph.ID{UID: pvUID}} } namespaceUID := CreateOrGetNamespaceByID(pvc.Namespace) if namespaceUID != "" { newPvc.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: pvc.Namespace}} } deletionTimestamp := pvc.GetDeletionTimestamp() if !deletionTimestamp.IsZero() { newPvc.EndTime = deletionTimestamp.Time.Format(time.RFC3339) newPvc.Xid += newPvc.EndTime newPvc.Name += "*" + newPvc.EndTime } return newPvc } // StorePersistentVolumeClaim create a new pvc in the Dgraph and updates if already present. func StorePersistentVolumeClaim(pvc api_v1.PersistentVolumeClaim) (string, error) { xid := pvc.Namespace + ":" + pvc.Name uid := dgraph.GetUID(xid, IsPersistentVolumeClaim) newPvc := createPvcObject(pvc) if uid != "" { newPvc.UID = uid } assigned, err := dgraph.MutateNode(newPvc, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } // CreateOrGetPersistentVolumeClaimByID returns the uid of pvc if exists, // otherwise creates the pvc and returns uid. func CreateOrGetPersistentVolumeClaimByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsPersistentVolumeClaim) if uid != "" { return uid } d := PersistentVolumeClaim{ ID: dgraph.ID{Xid: xid}, Name: xid, IsPersistentVolumeClaim: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } func getPVCFromUID(uid string) (PersistentVolumeClaim, error) { q := `query { pvcs(func: uid(` + uid + `)) { name type storageCapacity } }` type root struct { Pvcs []PersistentVolumeClaim `json:"pvcs"` } newRoot := root{} err := dgraph.ExecuteQuery(q, &newRoot) if err != nil { return PersistentVolumeClaim{}, err } return newRoot.Pvcs[0], nil } ================================================ FILE: pkg/controller/dgraph/models/query/cluster.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" ) var executeQuery = dgraph.ExecuteQuery var executeQueryRaw = dgraph.ExecuteQueryRaw var allocatedAndCapacity *ParentWrapper func getClusterHierarchyQuery(view string) string { switch view { case Physical: return getHierarchyQueryForPhysicalResource() case Logical: return getHierarchyQueryForLogicalResource() default: return "" } } // RetrieveClusterHierarchy returns all namespaces if view is logical and returns all nodes with disks if view is physical func RetrieveClusterHierarchy(view string) JSONDataWrapper { query := getClusterHierarchyQuery(view) parentRoot := ParentWrapper{} err := executeQuery(query, &parentRoot) if err != nil { logrus.Errorf("Unable to execute query for retrieving cluster hierarchy: (%v)", err) return JSONDataWrapper{} } root := JSONDataWrapper{ Data: ParentWrapper{ Name: "cluster", Type: "cluster", Children: parentRoot.Children, }, } logrus.Debugf("data: (%v)", root.Data) return root } func getClusterMetricsQuery(view string) string { switch view { case Physical: return getMetricsQueryForPhysicalResources() case Logical: return getMetricsQueryForLogicalResources() default: return "" } } // RetrieveClusterMetrics returns all namespaces with metrics if view is logical and // returns all nodes and disks with metrics if view is physical func RetrieveClusterMetrics(view string) JSONDataWrapper { query := getClusterMetricsQuery(view) parentRoot := ParentWrapper{} err := executeQuery(query, &parentRoot) calculateAggregateMetrics(&parentRoot) if err != nil { logrus.Errorf("Unable to execute query for retrieving cluster metrics: (%v)", err) return JSONDataWrapper{} } root := JSONDataWrapper{ Data: ParentWrapper{ Name: "cluster", Type: "cluster", Children: parentRoot.Children, CPU: parentRoot.CPU, Memory: parentRoot.Memory, Storage: parentRoot.Storage, CPUCost: parentRoot.CPUCost, MemoryCost: parentRoot.MemoryCost, StorageCost: parentRoot.StorageCost, }, } logrus.Debugf("data: (%v)", root.Data) return root } func calculateAggregateMetrics(objRoot *ParentWrapper) { for _, obj := range objRoot.Children { objRoot.CPU += obj.CPU objRoot.Memory += obj.Memory objRoot.Storage += obj.Storage objRoot.CPUCost += obj.CPUCost objRoot.MemoryCost += obj.MemoryCost objRoot.StorageCost += obj.StorageCost } } // PopulateClusterAllocationAndCapacity ... func PopulateClusterAllocationAndCapacity(jsonData *JSONDataWrapper) { if allocatedAndCapacity == nil { ComputeClusterAllocationAndCapacity() } populateCapacityData(*allocatedAndCapacity, jsonData) } // ComputeClusterAllocationAndCapacity returns allocated, capacity for cpu, memory and storage func ComputeClusterAllocationAndCapacity() { allocation := RetrieveClusterMetrics(Logical) capacity := RetrieveClusterMetrics(Physical) allocatedAndCapacity = &ParentWrapper{ CPUAllocated: allocation.Data.CPU, MemoryAllocated: allocation.Data.Memory, StorageAllocated: allocation.Data.Storage, CPUCapacity: capacity.Data.CPU, MemoryCapacity: capacity.Data.Memory, StorageCapacity: capacity.Data.Storage, } } // PopulateNodeOrPVAllocationAndCapacity returns allocated, capacity for cpu, memory and storage func (r *Resource) PopulateNodeOrPVAllocationAndCapacity(jsonData *JSONDataWrapper) { q := r.getQueryForResourceMetrics() resourceData := getJSONDataFromQuery(q) populateCapacityData(resourceData.Data, jsonData) } func populateCapacityData(allocatedAndCapacityData ParentWrapper, jsonData *JSONDataWrapper) { jsonData.Data.CPUAllocated = allocatedAndCapacityData.CPUAllocated jsonData.Data.MemoryAllocated = allocatedAndCapacityData.MemoryAllocated jsonData.Data.StorageAllocated = allocatedAndCapacityData.StorageAllocated jsonData.Data.CPUCapacity = allocatedAndCapacityData.CPUCapacity jsonData.Data.MemoryCapacity = allocatedAndCapacityData.MemoryCapacity jsonData.Data.StorageCapacity = allocatedAndCapacityData.StorageCapacity } ================================================ FILE: pkg/controller/dgraph/models/query/cluster_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "fmt" "os" "testing" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/stretchr/testify/assert" ) func mockSecondsSinceMonthStart() { secondsFromFirstOfCurrentMonth = func() string { return testSecondsSinceMonthStart } } func removeMocks() { secondsFromFirstOfCurrentMonth = getSecondsSinceMonthStart executeQuery = dgraph.ExecuteQuery executeQueryRaw = dgraph.ExecuteQueryRaw } // TestMain ... func TestMain(m *testing.M) { mockSecondsSinceMonthStart() code := m.Run() removeMocks() os.Exit(code) } func mockDgraphForClusterQueries(queryType string) { executeQuery = func(query string, root interface{}) error { if query == "" { return fmt.Errorf("unable to connect/retrieve data from dgraph") } dummyParentWrapper, ok := root.(*ParentWrapper) if !ok { return fmt.Errorf("wrong root received") } var first, second Children if queryType == testHierarchy { first = Children{ Name: "namespace-first", Type: NamespaceType, } second = Children{ Name: "namespace-second", Type: NamespaceType, } } else if queryType == testMetrics { first = Children{ Name: "namespace-first", Type: NamespaceType, CPU: 0.90, Memory: 3, Storage: 1, CPUCost: 0.09, MemoryCost: 0.31, StorageCost: 0.11, } second = Children{ Name: "namespace-second", Type: NamespaceType, CPU: 0.30, Memory: 9, Storage: 2, CPUCost: 0.03, MemoryCost: 0.91, StorageCost: 0.21, } } else if queryType == testCapacity { parent := ParentWrapper{ CPUAllocated: 0.2, CPUCapacity: 0.4, MemoryAllocated: 1.2, MemoryCapacity: 1.8, StorageAllocated: 10.5, StorageCapacity: 20.1, } dummyParentWrapper.Parent = []ParentWrapper{parent} } children := []Children{first, second} dummyParentWrapper.Children = children return nil } } // TestRetrieveClusterHierarchyNoView ... func TestRetrieveClusterHierarchyNoView(t *testing.T) { mockDgraphForClusterQueries(testHierarchy) got := RetrieveClusterHierarchy("") expected := JSONDataWrapper{} assert.Equal(t, expected, got) } // TestRetrieveClusterHierarchyLogicalView ... func TestRetrieveClusterHierarchyLogicalView(t *testing.T) { mockDgraphForClusterQueries(testHierarchy) got := RetrieveClusterHierarchy(Logical) firstNamespace := Children{ Name: "namespace-first", Type: NamespaceType, } secondNamespace := Children{ Name: "namespace-second", Type: NamespaceType, } hierarchyChildren := []Children{firstNamespace, secondNamespace} expected := JSONDataWrapper{ Data: ParentWrapper{ Name: "cluster", Type: "cluster", Children: hierarchyChildren, }, } assert.Equal(t, expected, got) } // TestRetrieveClusterHierarchyPhysicalView ... func TestRetrieveClusterHierarchyPhysicalView(t *testing.T) { mockDgraphForClusterQueries(testHierarchy) got := RetrieveClusterHierarchy(Physical) firstNamespace := Children{ Name: "namespace-first", Type: NamespaceType, } secondNamespace := Children{ Name: "namespace-second", Type: NamespaceType, } hierarchyChildren := []Children{firstNamespace, secondNamespace} expected := JSONDataWrapper{ Data: ParentWrapper{ Name: "cluster", Type: "cluster", Children: hierarchyChildren, }, } assert.Equal(t, expected, got) } // TestRetrieveClusterMetricsNoView ... func TestRetrieveClusterMetricsNoView(t *testing.T) { mockDgraphForClusterQueries(testMetrics) got := RetrieveClusterMetrics("") expected := JSONDataWrapper{} assert.Equal(t, expected, got) } // TestRetrieveClusterMetricsLogicalView ... func TestRetrieveClusterMetricsLogicalView(t *testing.T) { mockDgraphForClusterQueries(testMetrics) got := RetrieveClusterMetrics(Logical) firstNamespaceWithMetrics := Children{ Name: "namespace-first", Type: NamespaceType, CPU: 0.90, Memory: 3, Storage: 1, CPUCost: 0.09, MemoryCost: 0.31, StorageCost: 0.11, } secondNamespaceWithMetrics := Children{ Name: "namespace-second", Type: NamespaceType, CPU: 0.30, Memory: 9, Storage: 2, CPUCost: 0.03, MemoryCost: 0.91, StorageCost: 0.21, } metricsChildren := []Children{firstNamespaceWithMetrics, secondNamespaceWithMetrics} expected := JSONDataWrapper{ Data: ParentWrapper{ Name: "cluster", Type: "cluster", Children: metricsChildren, CPU: 1.2, Memory: 12, Storage: 3, CPUCost: 0.12, MemoryCost: 1.22, StorageCost: 0.32, }, } assert.Equal(t, expected, got) } // TestRetrieveClusterMetricsPhysicalView ... func TestRetrieveClusterMetricsPhysicalView(t *testing.T) { mockDgraphForClusterQueries(testMetrics) got := RetrieveClusterMetrics(Physical) firstNamespaceWithMetrics := Children{ Name: "namespace-first", Type: NamespaceType, CPU: 0.90, Memory: 3, Storage: 1, CPUCost: 0.09, MemoryCost: 0.31, StorageCost: 0.11, } secondNamespaceWithMetrics := Children{ Name: "namespace-second", Type: NamespaceType, CPU: 0.30, Memory: 9, Storage: 2, CPUCost: 0.03, MemoryCost: 0.91, StorageCost: 0.21, } metricsChildren := []Children{firstNamespaceWithMetrics, secondNamespaceWithMetrics} expected := JSONDataWrapper{ Data: ParentWrapper{ Name: "cluster", Type: "cluster", Children: metricsChildren, CPU: 1.20, Memory: 12, Storage: 3, CPUCost: 0.12, MemoryCost: 1.22, StorageCost: 0.32, }, } assert.Equal(t, expected, got) } func TestPopulateClusterAllocationAndCapacityNil(t *testing.T) { mockDgraphForClusterQueries(testMetrics) old := allocatedAndCapacity allocatedAndCapacity = nil defer resetAllocatedAndCapacity(old) data := &JSONDataWrapper{} PopulateClusterAllocationAndCapacity(data) expected := JSONDataWrapper{ Data: ParentWrapper{ CPUAllocated: 1.2, CPUCapacity: 1.2, MemoryAllocated: 12, MemoryCapacity: 12, StorageAllocated: 3, StorageCapacity: 3, }, } assert.Equal(t, expected, *data) } func TestPopulateClusterAllocationAndCapacity(t *testing.T) { old := allocatedAndCapacity allocatedAndCapacity = &ParentWrapper{ CPUAllocated: 0.2, CPUCapacity: 0.4, MemoryAllocated: 1.2, MemoryCapacity: 1.8, StorageAllocated: 10.5, StorageCapacity: 20.1, } defer resetAllocatedAndCapacity(old) data := &JSONDataWrapper{} PopulateClusterAllocationAndCapacity(data) expected := getTestAllocatedCapacityData() assert.Equal(t, expected, *data) } func resetAllocatedAndCapacity(old *ParentWrapper) { allocatedAndCapacity = old } func TestPopulateNodeOrPVAllocationAndCapacity(t *testing.T) { mockDgraphForClusterQueries(testCapacity) data := &JSONDataWrapper{} r := &Resource{ Check: NodeCheck, Type: NodeType, Name: testResourceName, } r.PopulateNodeOrPVAllocationAndCapacity(data) expected := getTestAllocatedCapacityData() assert.Equal(t, expected, *data) } func getTestAllocatedCapacityData() JSONDataWrapper { return JSONDataWrapper{ Data: ParentWrapper{ CPUAllocated: 0.2, CPUCapacity: 0.4, MemoryAllocated: 1.2, MemoryCapacity: 1.8, StorageAllocated: 10.5, StorageCapacity: 20.1, }, } } ================================================ FILE: pkg/controller/dgraph/models/query/constants_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query const ( testSecondsSinceMonthStart = "1.45" testPodUIDList = "0x3e283, 0x3e288" testPodName = "pod-purser-dgraph-0" testDaemonsetName = "daemonset-purser" testResourceName = "resource-purser" testPodUID = "0x3e283" testPodXID = "purser:pod-purser-dgraph-0" testHierarchy = "hierarchy" testMetrics = "metrics" testRetrieveAllGroups = "retrieveAllGroups" testRetrieveGroupMetrics = "retrieveGroupMetrics" testRetrieveSubscribers = "retrieveSubscribers" testLabelFilterPods = "labelFilterPods" testAlivePods = "alivePods" testPodInteractions = "podInteractions" testPodPrices = "podPrices" testCapacity = "capacityAllocation" testWrongQuery = "wrongQuery" testCPUPrice = 0.24 testMemoryPrice = 0.1 ) ================================================ FILE: pkg/controller/dgraph/models/query/group.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/Sirupsen/logrus" ) // GroupMetrics structure type GroupMetrics struct { PITCpu float64 PITMemory float64 PITStorage float64 PITCpuLimit float64 PITMemoryLimit float64 MTDCpu float64 MTDMemory float64 MTDStorage float64 MTDCpuLimit float64 MTDMemoryLimit float64 CostCPU float64 CostMemory float64 CostStorage float64 CostCPUPerHour float64 CostMemoryPerHour float64 CostStoragePerHour float64 LastMonthCPUCost float64 LastMonthMemoryCost float64 LastMonthStorageCost float64 LastLastMonthCPUCost float64 LastLastMonthMemoryCost float64 LastLastMonthStorageCost float64 PodsCount int } type groupsRoot struct { Groups []models.Group `json:"groups,omitempty"` } type groupJSONMetrics struct { JSONMetrics []map[string]float64 `json:"group"` } // RetrieveGroupsData returns list of models.Group objects in json format // error is not nil if any failure is encountered func RetrieveGroupsData() ([]models.Group, error) { query := getQueryForAllGroupsData() newRoot := groupsRoot{} err := executeQuery(query, &newRoot) if err != nil { return []models.Group{}, err } return newRoot.Groups, nil } // RetrieveGroupMetricsFromPodUIDs ... func RetrieveGroupMetricsFromPodUIDs(podsUIDs string) (GroupMetrics, error) { query := getQueryForGroupMetrics(podsUIDs) newRoot := groupJSONMetrics{} err := executeQuery(query, &newRoot) if err != nil { return GroupMetrics{}, err } return convertToGroupMetrics(newRoot.JSONMetrics), nil } func convertToGroupMetrics(jsonMetrics []map[string]float64) GroupMetrics { var groupMetrics GroupMetrics for _, data := range jsonMetrics { for key, value := range data { populateMetric(&groupMetrics, key, value) break } } logrus.Debugf("JSON metrics: (%v), Group metrics: (%v)", jsonMetrics, groupMetrics) return groupMetrics } // nolint: gocyclo func populateMetric(groupMetrics *GroupMetrics, key string, value float64) { logrus.Debugf("key: %s", key) switch key { case "pitCPU": groupMetrics.PITCpu = value case "pitMemory": groupMetrics.PITMemory = value case "pitStorage": groupMetrics.PITStorage = value case "pitCPULimit": groupMetrics.PITCpuLimit = value case "pitMemoryLimit": groupMetrics.PITMemoryLimit = value case "mtdCPU": groupMetrics.MTDCpu = value case "mtdMemory": groupMetrics.MTDMemory = value case "mtdStorage": groupMetrics.MTDStorage = value case "mtdCPULimit": groupMetrics.MTDCpuLimit = value case "mtdMemoryLimit": groupMetrics.MTDMemoryLimit = value case "cpuCost": groupMetrics.CostCPU = value case "memoryCost": groupMetrics.CostMemory = value case "storageCost": groupMetrics.CostStorage = value case "cpuCostPerHour": groupMetrics.CostCPUPerHour = value case "memoryCostPerHour": groupMetrics.CostMemoryPerHour = value case "storageCostPerHour": groupMetrics.CostStoragePerHour = value case "lastMonthCPUCost": groupMetrics.LastMonthCPUCost = value case "lastMonthMemoryCost": groupMetrics.LastMonthMemoryCost = value case "lastMonthStorageCost": groupMetrics.LastMonthStorageCost = value case "lastLastMonthCPUCost": groupMetrics.LastLastMonthCPUCost = value case "lastLastMonthMemoryCost": groupMetrics.LastLastMonthMemoryCost = value case "lastLastMonthStorageCost": groupMetrics.LastLastMonthStorageCost = value case "livePods": groupMetrics.PodsCount = int(value) } } ================================================ FILE: pkg/controller/dgraph/models/query/group_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "fmt" "testing" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/stretchr/testify/assert" ) var dummyGroup models.Group func mockMetricsMap(key string, value float64) map[string]float64 { metrics := make(map[string]float64) metrics[key] = value return metrics } func mockDgraphForGroupQueries(queryType string) { executeQuery = func(query string, root interface{}) error { if queryType == testRetrieveAllGroups { dummyGroupList, ok := root.(*groupsRoot) if !ok { return fmt.Errorf("wrong root received") } dummyGroup = models.Group{ Name: "group-purser", PodsCount: 3, MtdCPU: 50.1, MtdMemory: 31.7, MtdStorage: 300, CPU: 4.1, Memory: 3.4, Storage: 10, MtdCPUCost: 5.01, MtdMemoryCost: 3.17, MtdStorageCost: 3, MtdCost: 11.18, } dummyGroupList.Groups = []models.Group{dummyGroup} return nil } else if queryType == testRetrieveGroupMetrics { groupMetrics, ok := root.(*groupJSONMetrics) if !ok { return fmt.Errorf("wrong root received") } var jsonMetrics []map[string]float64 jsonMetrics = append(jsonMetrics, mockMetricsMap("pitCPU", 1.3)) jsonMetrics = append(jsonMetrics, mockMetricsMap("pitMemory", 2.4)) jsonMetrics = append(jsonMetrics, mockMetricsMap("pitStorage", 2)) jsonMetrics = append(jsonMetrics, mockMetricsMap("pitCPULimit", 1.4)) jsonMetrics = append(jsonMetrics, mockMetricsMap("pitMemoryLimit", 2.5)) jsonMetrics = append(jsonMetrics, mockMetricsMap("mtdCPU", 13.1)) jsonMetrics = append(jsonMetrics, mockMetricsMap("mtdMemory", 24.2)) jsonMetrics = append(jsonMetrics, mockMetricsMap("mtdStorage", 20)) jsonMetrics = append(jsonMetrics, mockMetricsMap("mtdCPULimit", 14.1)) jsonMetrics = append(jsonMetrics, mockMetricsMap("mtdMemoryLimit", 25.2)) jsonMetrics = append(jsonMetrics, mockMetricsMap("cpuCost", 1.31)) jsonMetrics = append(jsonMetrics, mockMetricsMap("memoryCost", 2.42)) jsonMetrics = append(jsonMetrics, mockMetricsMap("storageCost", 0.21)) jsonMetrics = append(jsonMetrics, mockMetricsMap("livePods", 2)) groupMetrics.JSONMetrics = jsonMetrics return nil } return fmt.Errorf("no data found") } } // TestRetrieveGroupsDataWithDgraphError ... func TestRetrieveGroupsDataWithDgraphError(t *testing.T) { mockDgraphForGroupQueries(testWrongQuery) _, err := RetrieveGroupsData() assert.Error(t, err) } // TestRetrieveGroupsData ... func TestRetrieveGroupsData(t *testing.T) { mockDgraphForGroupQueries(testRetrieveAllGroups) got, err := RetrieveGroupsData() expected := []models.Group{{ Name: "group-purser", PodsCount: 3, MtdCPU: 50.1, MtdMemory: 31.7, MtdStorage: 300, CPU: 4.1, Memory: 3.4, Storage: 10, MtdCPUCost: 5.01, MtdMemoryCost: 3.17, MtdStorageCost: 3, MtdCost: 11.18, }} assert.NoError(t, err) assert.Equal(t, expected, got) } // TestGroupMetricsFromPodUIDsWithDgraphError ... func TestGroupMetricsFromPodUIDsWithDgraphError(t *testing.T) { mockDgraphForGroupQueries(testWrongQuery) _, err := RetrieveGroupMetricsFromPodUIDs("") assert.Error(t, err) } // TestGroupMetricsFromPodUIDs ... func TestGroupMetricsFromPodUIDs(t *testing.T) { mockDgraphForGroupQueries(testRetrieveGroupMetrics) got, err := RetrieveGroupMetricsFromPodUIDs(testPodUIDList) expected := GroupMetrics{ PITCpu: 1.3, PITMemory: 2.4, PITStorage: 2, PITCpuLimit: 1.4, PITMemoryLimit: 2.5, MTDCpu: 13.1, MTDMemory: 24.2, MTDStorage: 20, MTDCpuLimit: 14.1, MTDMemoryLimit: 25.2, CostCPU: 1.31, CostMemory: 2.42, CostStorage: 0.21, PodsCount: 2, } assert.Equal(t, expected, got) assert.NoError(t, err) } ================================================ FILE: pkg/controller/dgraph/models/query/helpers.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "fmt" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/vmware/purser/pkg/controller/utils" ) var secondsFromFirstOfCurrentMonth = getSecondsSinceMonthStart func getSecondsSinceMonthStart() string { return fmt.Sprintf("%f", utils.GetSecondsSince(utils.GetCurrentMonthStartTime())) } func getSecondsSinceForOtherMonths() map[string]string { secondsSince := make(map[string]string) secondsInAverageMonth := 2592000.0 // 30 * 24 * 60 * 60 secondsSinceCurrentMonthStart := utils.GetSecondsSince(utils.GetCurrentMonthStartTime()) secondsSinceLastMonthStart := secondsSinceCurrentMonthStart + secondsInAverageMonth secondsSinceLastLastMonthStart := secondsSinceCurrentMonthStart + 2*secondsInAverageMonth secondsSinceLastMonthEnd := secondsSinceCurrentMonthStart - 1.0 secondsSinceLastLastMonthEnd := secondsSinceLastMonthStart - 1.0 secondsSince["currentMonthStart"] = fmt.Sprintf("%f", secondsSinceCurrentMonthStart) secondsSince["lastMonthStart"] = fmt.Sprintf("%f", secondsSinceLastMonthStart) secondsSince["lastMonthEnd"] = fmt.Sprintf("%f", secondsSinceLastMonthEnd) secondsSince["lastLastMonthStart"] = fmt.Sprintf("%f", secondsSinceLastLastMonthStart) secondsSince["lastLastMonthEnd"] = fmt.Sprintf("%f", secondsSinceLastLastMonthEnd) return secondsSince } func getQueryForMetricsComputationWithAliasAndVariables(suffix string) string { return `name type cpu: cpu` + suffix + ` as cpuRequest memory: memory` + suffix + ` as memoryRequest storage: storage` + suffix + ` as storageRequest ` + getQueryForTimeComputation(suffix) + ` ` + getQueryForCostWithPriceWithAliasAndVariables(suffix) } func getQueryForMetricsComputationWithAlias(suffix string) string { return `name type cpu: cpu` + suffix + ` as cpuRequest memory: memory` + suffix + ` as memoryRequest storage: storage` + suffix + ` as storageRequest ` + getQueryForTimeComputation(suffix) + ` ` + getQueryForCostWithPriceWithAlias(suffix) } func getQueryForMetricsComputation(suffix string) string { return `cpu` + suffix + ` as cpuRequest memory` + suffix + ` as memoryRequest storage` + suffix + ` as storageRequest ` + getQueryForTimeComputation(suffix) + ` ` + getQueryForCostWithPrice(suffix) } func getQueryForTimeComputation(suffix string) string { secondsSinceMonthStart := secondsFromFirstOfCurrentMonth() return `st` + suffix + ` as startTime stSeconds` + suffix + ` as math(since(st` + suffix + `)) secondsSinceStart` + suffix + ` as math(cond(stSeconds` + suffix + ` > ` + secondsSinceMonthStart + `, ` + secondsSinceMonthStart + `, stSeconds` + suffix + `)) et` + suffix + ` as endTime isTerminated` + suffix + ` as count(endTime) secondsSinceEnd` + suffix + ` as math(cond(isTerminated` + suffix + ` == 0, 0.0, since(et` + suffix + `))) durationInHours` + suffix + ` as math(cond(secondsSinceStart` + suffix + ` > secondsSinceEnd` + suffix + `, (secondsSinceStart` + suffix + ` - secondsSinceEnd` + suffix + `) / 3600, 0.0))` } func getQueryForCostWithPriceWithAliasAndVariables(suffix string) string { return `pricePerCPU` + suffix + ` as cpuPrice pricePerMemory` + suffix + ` as memoryPrice cpuCost: cpuCost` + suffix + ` as math(cpu` + suffix + ` * durationInHours` + suffix + ` * pricePerCPU` + suffix + `) memoryCost: memoryCost` + suffix + ` as math(memory` + suffix + ` * durationInHours` + suffix + ` * pricePerMemory` + suffix + `) storageCost: storageCost` + suffix + ` as math(storage` + suffix + ` * durationInHours` + suffix + ` * ` + models.DefaultStorageCostPerGBPerHour + `)` } func getQueryForCostWithPriceWithAlias(suffix string) string { return `pricePerCPU` + suffix + ` as cpuPrice pricePerMemory` + suffix + ` as memoryPrice cpuCost: math(cpu` + suffix + ` * durationInHours` + suffix + ` * pricePerCPU` + suffix + `) memoryCost: math(memory` + suffix + ` * durationInHours` + suffix + ` * pricePerMemory` + suffix + `) storageCost: math(storage` + suffix + ` * durationInHours` + suffix + ` * ` + models.DefaultStorageCostPerGBPerHour + `)` } func getQueryForCostWithPrice(suffix string) string { return `pricePerCPU` + suffix + ` as cpuPrice pricePerMemory` + suffix + ` as memoryPrice cpuCost` + suffix + ` as math(cpu` + suffix + ` * durationInHours` + suffix + ` * pricePerCPU` + suffix + `) memoryCost` + suffix + ` as math(memory` + suffix + ` * durationInHours` + suffix + ` * pricePerMemory` + suffix + `) storageCost` + suffix + ` as math(storage` + suffix + ` * durationInHours` + suffix + ` * ` + models.DefaultStorageCostPerGBPerHour + `)` } func getQueryForAggregatingChildMetricsWithAlias(childSuffix string) string { return `name type cpu: sum(val(cpu` + childSuffix + `)) memory: sum(val(memory` + childSuffix + `)) storage: sum(val(storage` + childSuffix + `)) cpuCost: sum(val(cpuCost` + childSuffix + `)) memoryCost: sum(val(memoryCost` + childSuffix + `)) storageCost: sum(val(storageCost` + childSuffix + `))` } func getQueryForAggregatingChildMetrics(parentSuffix, childSuffix string) string { return `cpu` + parentSuffix + ` as sum(val(cpu` + childSuffix + `)) memory` + parentSuffix + ` as sum(val(memory` + childSuffix + `)) storage` + parentSuffix + ` as sum(val(storage` + childSuffix + `)) cpuCost` + parentSuffix + ` as sum(val(cpuCost` + childSuffix + `)) memoryCost` + parentSuffix + ` as sum(val(memoryCost` + childSuffix + `)) storageCost` + parentSuffix + ` as sum(val(storageCost` + childSuffix + `))` } func getQueryFromSubQueryWithAlias(suffix string) string { return `name type cpu: val(cpu` + suffix + `) memory: val(memory` + suffix + `) storage: val(storage` + suffix + `) cpuCost: val(cpuCost` + suffix + `) memoryCost: val(memoryCost` + suffix + `) storageCost: val(storageCost` + suffix + `)` } func (r *Resource) getQueryForPodParentMetrics() string { return `query { parent(func: has(` + r.Check + `)) @filter(eq(name, "` + r.Name + `")) { children: ~` + r.Type + ` @filter(has(isPod)) { ` + getQueryForMetricsComputationWithAliasAndVariables("Pod") + ` } ` + getQueryForAggregatingChildMetricsWithAlias("Pod") + ` } }` } func (r *Resource) getQueryForHierarchy() string { return `query { parent(func: has(` + r.Check + `)) @filter(eq(name, "` + r.Name + `")) { name type children: ~` + r.Type + ` ` + r.ChildFilter + ` { name type } } }` } ================================================ FILE: pkg/controller/dgraph/models/query/helpers_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "strconv" "testing" "github.com/stretchr/testify/assert" ) // TestGetSecondsSinceMonthStart ... func TestGetSecondsSinceMonthStart(t *testing.T) { maxSecondsInAMonth := 2678400.0 got := getSecondsSinceMonthStart() gotFloat, err := strconv.ParseFloat(got, 64) assert.NoError(t, err, "unable to convert secondsSinceMonthStart to float64") assert.False(t, gotFloat > maxSecondsInAMonth, "secondsSinceMonthStart can't be greater than 2678400") assert.False(t, gotFloat < 0, "secondsSinceMonthStart can't be less than 0") } ================================================ FILE: pkg/controller/dgraph/models/query/label.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query // CreateFilterFromListOfLabels will return a filter logic like // (eq(key, "k1") AND eq(value, "v1")) OR (eq(key, "k1") AND eq(value, "v1")) OR (eq(key, "k1") AND eq(value, "v1")) func CreateFilterFromListOfLabels(labels map[string][]string) string { separator := " OR " var filter string isFirst := true for key, values := range labels { for _, value := range values { if !isFirst { filter += separator } else { isFirst = false } filter += createFilterFromLabel(key, value) } } return filter } // createFilterFromLabel takes key: k1, value: v1 and returns (eq(key, "k1") AND eq(value, "v1")) func createFilterFromLabel(key, value string) string { return `(eq(key, "` + key + `") AND eq(value, "` + value + `"))` } ================================================ FILE: pkg/controller/dgraph/models/query/label_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "testing" "github.com/vmware/purser/test/utils" ) // TestCreateFilterForLabel ... func TestCreateFilterFromLabel(t *testing.T) { got := createFilterFromLabel("k1", "v1") expected := `(eq(key, "k1") AND eq(value, "v1"))` utils.Equals(t, expected, got) } // TestCreateFilterFromListOfLabels ... func TestCreateFilterFromListOfLabels(t *testing.T) { labels := make(map[string][]string) labels["k1"] = []string{"v1"} got := CreateFilterFromListOfLabels(labels) expected := `(eq(key, "k1") AND eq(value, "v1"))` utils.Equals(t, expected, got) labels["k2"] = []string{"v2"} got2 := CreateFilterFromListOfLabels(labels) expected1 := `(eq(key, "k2") AND eq(value, "v2")) OR (eq(key, "k1") AND eq(value, "v1"))` expected2 := `(eq(key, "k1") AND eq(value, "v1")) OR (eq(key, "k2") AND eq(value, "v2"))` utils.Assert(t, (got2 == expected1) || (got2 == expected2), "label filter didn't match") } ================================================ FILE: pkg/controller/dgraph/models/query/login.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" "golang.org/x/crypto/bcrypt" ) // Authenticate performs user authentication for service access func Authenticate(username, inputPassword string) bool { if !validateUsername(username) { return false } login, err := getLoginCredentials(username) if err != nil { logrus.Error(err) return false } return comparePasswords(login.Password, []byte(inputPassword)) } // UpdatePassword updates stored password with new one for the given username in Dgraph func UpdatePassword(username, oldPassword, newPassword string) bool { if Authenticate(username, oldPassword) { login, err := getLoginCredentials(username) if err != nil { logrus.Error(err) return false } if err = hashAndUpdatePassword(&login, newPassword); err == nil { return true } logrus.Error(err) } return false } func hashAndUpdatePassword(login *dgraph.Login, newPassword string) error { hashedPassword, err := bcrypt.GenerateFromPassword([]byte(newPassword), bcrypt.MinCost) if err != nil { return err } login.Password = string(hashedPassword) _, err = dgraph.MutateNode(login, dgraph.UPDATE) return err } // getLoginCredentials returns a struct of hashed password and username. func getLoginCredentials(username string) (dgraph.Login, error) { q := `query { login(func: has(isLogin)) @filter(eq(username, ` + username + `)) { uid username password } }` type root struct { LoginList []dgraph.Login `json:"login"` } newRoot := root{} if err := executeQuery(q, &newRoot); err != nil || newRoot.LoginList == nil { return dgraph.Login{}, err } return newRoot.LoginList[0], nil } func validateUsername(username string) bool { return username == "admin" } func comparePasswords(hashedPwd string, plainPwd []byte) bool { byteHash := []byte(hashedPwd) if err := bcrypt.CompareHashAndPassword(byteHash, plainPwd); err != nil { logrus.Error(err) return false } return true } ================================================ FILE: pkg/controller/dgraph/models/query/pod.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph/models" ) type podRoot struct { Pods []models.Pod `json:"pods"` } // RetrieveAllLivePods will return all pods without endTime in dgraph. Error is returned if any // failure is encountered in the process. func RetrieveAllLivePods() []models.Pod { query := getAllLivePodsQuery() newRoot := podRoot{} err := executeQuery(query, &newRoot) if err != nil { logrus.Errorf("unable to retrieve all live pods: %v", err) return nil } return newRoot.Pods } // RetrievePodsInteractions returns inbound and outbound interactions of a pod func RetrievePodsInteractions(name string, isOrphan bool) []byte { var query string if name == All { if isOrphan { query = `query { pods(func: has(isPod)) { name outbound: pod { name } inbound: ~pod @filter(has(isPod)) { name } } }` } else { query = `query { pods(func: has(isPod)) @filter(has(pod)) { name outbound: pod { name } inbound: ~pod @filter(has(isPod)) { name } } }` } } else { query = `query { pods(func: has(isPod)) @filter(eq(name, "` + name + `")) { name outbound: pod { name } inbound: ~pod @filter(has(isPod)) { name } } }` } result, err := executeQueryRaw(query) if err != nil { logrus.Errorf("Error while retrieving query for pods interactions. Name: (%v), isOrphan: (%v), error: (%v)", name, isOrphan, err) return nil } return result } func getPricePerResourceForPod(name string) (float64, float64) { query := `query { pods(func: has(isPod)) @filter(eq(name, "` + name + `")) { cpuPrice memoryPrice } }` newRoot := podRoot{} err := executeQuery(query, &newRoot) if err != nil || len(newRoot.Pods) < 1 { logrus.Errorf("err: %v", err) return models.DefaultCPUCostInFloat64, models.DefaultMemCostInFloat64 } pod := newRoot.Pods[0] return pod.CPUPrice, pod.MemoryPrice } // RetrievePodsInteractionsForAllLivePodsWithCount returns all pods in the dgraph func RetrievePodsInteractionsForAllLivePodsWithCount() ([]models.Pod, error) { q := `query { pods(func: has(isPod)) @filter((NOT has(endTime))) { name pod { name count } cid: ~pod @filter(has(isService)) { name } } }` type root struct { Pods []models.Pod `json:"pods"` } newRoot := root{} err := executeQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.Pods, nil } // RetrievePodsUIDsByLabelsFilter returns pods satisfying the filter conditions for labels (OR logic only) func RetrievePodsUIDsByLabelsFilter(labelFilter string) ([]string, error) { q := getQueryForPodsWithLabelFilter(labelFilter) newRoot := podRoot{} err := executeQuery(q, &newRoot) if err != nil { return nil, err } return removeDuplicates(newRoot.Pods), nil } func removeDuplicates(pods []models.Pod) []string { duplicateChecker := make(map[string]bool) var podsUIDs []string for _, pod := range pods { if _, isPresent := duplicateChecker[pod.UID]; !isPresent { podsUIDs = append(podsUIDs, pod.UID) duplicateChecker[pod.UID] = true } } return podsUIDs } ================================================ FILE: pkg/controller/dgraph/models/query/pod_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/dgraph/models" ) func mockDgraphForPodQueries(queryType string) { executeQuery = func(query string, root interface{}) error { if queryType == testLabelFilterPods { dummyPodList, ok := root.(*podRoot) if !ok { return fmt.Errorf("wrong root received") } dummyPodList.Pods = []models.Pod{ { ID: dgraph.ID{UID: testPodUID}, Name: testPodName, }, } return nil } else if queryType == testAlivePods { dummyPodList, ok := root.(*podRoot) if !ok { return fmt.Errorf("wrong root received") } dummyPodList.Pods = []models.Pod{ { ID: dgraph.ID{UID: testPodUID, Xid: testPodXID}, Name: testPodName, }, } return nil } return fmt.Errorf("no data found") } executeQueryRaw = func(query string) ([]byte, error) { return nil, fmt.Errorf("pod interactions err") } } // TestRetrievePodsUIDsByLabelsFilterWithError ... func TestRetrievePodsUIDsByLabelsFilterWithError(t *testing.T) { mockDgraphForPodQueries(testWrongQuery) // input setup labels := make(map[string][]string) labels["k1"] = []string{"v1"} inputLabelFilter := CreateFilterFromListOfLabels(labels) _, err := RetrievePodsUIDsByLabelsFilter(inputLabelFilter) assert.Error(t, err) } // TestRetrievePodsUIDsByLabelsFilter ... func TestRetrievePodsUIDsByLabelsFilter(t *testing.T) { mockDgraphForPodQueries(testLabelFilterPods) // input setup labels := make(map[string][]string) labels["k1"] = []string{"v1"} inputLabelFilter := CreateFilterFromListOfLabels(labels) got, err := RetrievePodsUIDsByLabelsFilter(inputLabelFilter) expected := []string{testPodUID} assert.NoError(t, err) assert.Equal(t, expected, got) } // TestRetrieveAllLivePodsWithDgraphError ... func TestRetrieveAllLivePodsWithDgraphError(t *testing.T) { mockDgraphForPodQueries(testWrongQuery) got := RetrieveAllLivePods() assert.Nil(t, got) } // TestRetrieveAllLivePods ... func TestRetrieveAllLivePods(t *testing.T) { mockDgraphForPodQueries(testAlivePods) got := RetrieveAllLivePods() expected := []models.Pod{ { ID: dgraph.ID{UID: testPodUID, Xid: testPodXID}, Name: testPodName, }, } assert.Equal(t, expected, got) } func TestPodInteractionsErrorCase(t *testing.T) { mockDgraphForPodQueries(testPodInteractions) gotAllOrphan := RetrievePodsInteractions("", true) gotAllNonOrphan := RetrievePodsInteractions("", false) gotWithName := RetrievePodsInteractions(testPodName, false) _, err := RetrievePodsInteractionsForAllLivePodsWithCount() assert.Nil(t, gotAllOrphan) assert.Nil(t, gotAllNonOrphan) assert.Nil(t, gotWithName) assert.Error(t, err) } func TestGetPricePerResourceForPodWithError(t *testing.T) { mockDgraphForResourceQueries(testWrongQuery, testPodName, PodType) gotCPUPrice, gotMemoryPrice := getPricePerResourceForPod(testPodName) expectedCPUPrice, expectedMemoryPrice := models.DefaultCPUCostInFloat64, models.DefaultMemCostInFloat64 assert.Equal(t, expectedCPUPrice, gotCPUPrice) assert.Equal(t, expectedMemoryPrice, gotMemoryPrice) } func TestGetPricePerResourceForPod(t *testing.T) { mockDgraphForResourceQueries(testPodPrices, testPodName, PodType) gotCPUPrice, gotMemoryPrice := getPricePerResourceForPod(testPodName) expectedCPUPrice, expectedMemoryPrice := testCPUPrice, testMemoryPrice assert.Equal(t, expectedCPUPrice, gotCPUPrice) assert.Equal(t, expectedMemoryPrice, gotMemoryPrice) } ================================================ FILE: pkg/controller/dgraph/models/query/queries.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "github.com/vmware/purser/pkg/controller/dgraph/models" ) // DeploymentMetrics query func getQueryForDeploymentMetrics(name string) string { return `query { dep as var(func: has(isDeployment)) @filter(eq(name, "` + name + `")) { ~deployment @filter(has(isReplicaset)) { ~replicaset @filter(has(isPod)) { ` + getQueryForMetricsComputation("ReplicasetPod") + ` } ` + getQueryForAggregatingChildMetrics("DeploymentReplicaset", "ReplicasetPod") + ` } ` + getQueryForAggregatingChildMetrics("Deployment", "DeploymentReplicaset") + ` } parent(func: uid(dep)) { children: ~deployment @filter(has(isReplicaset)) { ` + getQueryFromSubQueryWithAlias("DeploymentReplicaset") + ` } ` + getQueryFromSubQueryWithAlias("Deployment") + ` } }` } // PodMetrics query func getQueryForPodMetrics(name, cpuPrice, memoryPrice string) string { return `query { parent(func: has(isPod)) @filter(eq(name, "` + name + `")) { children: ~pod @filter(has(isContainer)) { name type ` + getQueryForTimeComputation("Container") + ` cpu: cpu as cpuRequest memory: memory as memoryRequest cpuCost: math(cpu * durationInHoursContainer * ` + cpuPrice + `) memoryCost: math(memory * durationInHoursContainer * ` + memoryPrice + `) } ` + getQueryForMetricsComputationWithAlias("Pod") + ` } }` } // ContainerMetrics query func getQueryForContainerMetrics(name string) string { return `query { parent(func: has(isContainer)) @filter(eq(name, "` + name + `")) { name type cpu: cpu as cpuRequest memory: memory as memoryRequest ` + getQueryForTimeComputation("") + ` cpuCost: math(cpu * durationInHours * ` + models.DefaultCPUCostPerCPUPerHour + `) memoryCost: math(memory * durationInHours * ` + models.DefaultMemCostPerGBPerHour + `) } }` } // PVMetrics query func getQueryForPVMetrics(name string) string { return `query { parent(func: has(isPersistentVolume)) @filter(eq(name, "` + name + `")) { children: ~pv @filter(has(isPersistentVolumeClaim)) { name type storage: pvcStorage as storageCapacity ` + getQueryForTimeComputation("PVC") + ` storageCost: math(pvcStorage * durationInHoursPVC * ` + models.DefaultStorageCostPerGBPerHour + `) } name type storage: storage as storageCapacity storageCapacity ` + getQueryForTimeComputation("") + ` storageCost: math(storage * durationInHours * ` + models.DefaultStorageCostPerGBPerHour + `) storageAllocated: sum(val(pvcStorage)) } }` } // PVCMetrics query func getQueryForPVCMetrics(name string) string { return `query { parent(func: has(isPersistentVolumeClaim)) @filter(eq(name, "` + name + `")) { name type storage: storage as storageCapacity ` + getQueryForTimeComputation("") + ` storageCost: math(storage * durationInHours * ` + models.DefaultStorageCostPerGBPerHour + `) } }` } // NodeMetrics query func getQueryForNodeMetrics(name string) string { return `query { parent(func: has(isNode)) @filter(eq(name, "` + name + `")) { children: ~node @filter(has(isPod)) { ` + getQueryForMetricsComputationWithAlias("Pod") + ` } name type cpu: cpu as cpuCapacity memory: memory as memoryCapacity storage: storage as sum(val(storagePod)) cpuAllocated: sum(val(cpuPod)) memoryAllocated: sum(val(memoryPod)) cpuCapacity memoryCapacity ` + getQueryForTimeComputation("") + ` ` + getQueryForCostWithPriceWithAlias("") + ` } }` } // NamespaceMetrics query func getQueryForNamespaceMetrics(name string) string { return `query { ns as var(func: has(isNamespace)) @filter(eq(name, "` + name + `")) { childs as ~namespace @filter(has(isDeployment) OR has(isStatefulset) OR has(isJob) OR has(isDaemonset) OR (has(isReplicaset) AND (NOT has(deployment)))) { name type ~deployment @filter(has(isReplicaset)) { name type ~replicaset @filter(has(isPod)) { ` + getQueryForMetricsComputation("ReplicasetPod") + ` } ` + getQueryForAggregatingChildMetrics("DeploymentReplicaset", "ReplicasetPod") + ` } ~statefulset @filter(has(isPod)) { ` + getQueryForMetricsComputation("StatefulsetPod") + ` } ~job @filter(has(isPod)) { ` + getQueryForMetricsComputation("JobPod") + ` } ~daemonset @filter(has(isPod)) { ` + getQueryForMetricsComputation("DaemonsetPod") + ` } ~replicaset @filter(has(isPod)) { ` + getQueryForMetricsComputation("ReplicasetSimplePod") + ` } ` + getQueryForAggregatingChildMetrics("SumReplicasetSimplePod", "ReplicasetSimplePod") + ` ` + getQueryForAggregatingChildMetrics("SumDaemonsetPod", "DaemonsetPod") + ` ` + getQueryForAggregatingChildMetrics("SumJobPod", "JobPod") + ` ` + getQueryForAggregatingChildMetrics("SumStatefulsetPod", "StatefulsetPod") + ` ` + getQueryForAggregatingChildMetrics("SumDeploymentReplicaset", "DeploymentReplicaset") + ` cpuNamespaceChild as math(cpu` + "SumReplicasetSimplePod" + ` + cpu` + "SumDaemonsetPod" + ` + cpu` + "SumJobPod" + ` + cpu` + "SumStatefulsetPod" + ` + cpu` + "SumDeploymentReplicaset" + `) memoryNamespaceChild as math(memory` + "SumReplicasetSimplePod" + ` + memory` + "SumDaemonsetPod" + ` + memory` + "SumJobPod" + ` + memory` + "SumStatefulsetPod" + ` + memory` + "SumDeploymentReplicaset" + `) storageNamespaceChild as math(storage` + "SumReplicasetSimplePod" + ` + storage` + "SumDaemonsetPod" + ` + storage` + "SumJobPod" + ` + storage` + "SumStatefulsetPod" + ` + storage` + "SumDeploymentReplicaset" + `) cpuCostNamespaceChild as math(cpuCost` + "SumReplicasetSimplePod" + ` + cpuCost` + "SumDaemonsetPod" + ` + cpuCost` + "SumJobPod" + ` + cpuCost` + "SumStatefulsetPod" + ` + cpuCost` + "SumDeploymentReplicaset" + `) memoryCostNamespaceChild as math(memoryCost` + "SumReplicasetSimplePod" + ` + memoryCost` + "SumDaemonsetPod" + ` + memoryCost` + "SumJobPod" + ` + memoryCost` + "SumStatefulsetPod" + ` + memoryCost` + "SumDeploymentReplicaset" + `) storageCostNamespaceChild as math(storageCost` + "SumReplicasetSimplePod" + ` + storageCost` + "SumDaemonsetPod" + ` + storageCost` + "SumJobPod" + ` + storageCost` + "SumStatefulsetPod" + ` + storageCost` + "SumDeploymentReplicaset" + `) } ` + getQueryForAggregatingChildMetrics("Namespace", "NamespaceChild") + ` } parent(func: uid(ns)) { children: ~namespace @filter(uid(childs)) { ` + getQueryFromSubQueryWithAlias("NamespaceChild") + ` } ` + getQueryFromSubQueryWithAlias("Namespace") + ` } }` } // LogicalResourcesMetrics query func getMetricsQueryForLogicalResources() string { return `query { ns as var(func: has(isNamespace)) { ~namespace @filter(has(isPod) AND (NOT has(endTime))) { ` + getQueryForMetricsComputation("NamespacePod") + ` } ` + getQueryForAggregatingChildMetrics("Namespace", "NamespacePod") + ` } children(func: uid(ns)) { ` + getQueryFromSubQueryWithAlias("Namespace") + ` } }` } // PhysicalResourcesMetrics query func getMetricsQueryForPhysicalResources() string { return `query { children(func: has(name)) @filter((has(isNode) OR has(isPersistentVolume)) AND (NOT has(endTime))) { name type cpu: cpu as cpuCapacity memory: memory as memoryCapacity storage: storage as storageCapacity ` + getQueryForTimeComputation("") + ` ` + getQueryForCostWithPriceWithAlias("") + ` } }` } // LogicalResourcesHierarchy query func getHierarchyQueryForLogicalResource() string { return `query { children(func: has(isNamespace)) { name type } }` } // PhysicalResourcesHierarchy query func getHierarchyQueryForPhysicalResource() string { return `query { children(func: has(name)) @filter(has(isNode) OR has(isPersistentVolume)) { name type } }` } /* The following functions are related to Queries for Custom Groups */ func getQueryForAllGroupsData() string { return `query { groups(func: has(isGroup)) { name podsCount mtdCPU mtdMemory mtdStorage cpu memory storage mtdCPUCost mtdMemoryCost mtdStorageCost mtdCost projectedCPUCost projectedMemoryCost projectedStorageCost projectedCost lastMonthCPUCost lastMonthMemoryCost lastMonthStorageCost lastMonthCost lastLastMonthCPUCost lastLastMonthMemoryCost lastLastMonthStorageCost lastLastMonthCost } }` } func getQueryForGroupMetrics(podsUIDs string) string { secondsSince := getSecondsSinceForOtherMonths() return `query { var(func: uid(` + podsUIDs + `)) { podCpu as cpuRequest podMemory as memoryRequest pvcStorage as storageRequest podCpuLimit as cpuLimit podMemoryLimit as memoryLimit cpuRequestCount as count(cpuRequest) memoryRequestCount as count(memoryRequest) storageRequestCount as count(storageRequest) cpuLimitCount as count(cpuLimit) memoryLimitCount as count(memoryLimit) podEndTime as endTime isTerminated as count(endTime) secondsSincePodEndTime as math(cond(isTerminated == 0, 0.0, since(podEndTime))) podStartTime as startTime secondsSincePodStartTime as math(since(podStartTime)) secondsSinceCurrentMonthTrueStart as math(cond(secondsSincePodStartTime > ` + secondsSince["currentMonthStart"] + `, ` + secondsSince["currentMonthStart"] + `, secondsSincePodStartTime)) currentMonthTrueDurationInHours as math(cond(secondsSinceCurrentMonthTrueStart > secondsSincePodEndTime, (secondsSinceCurrentMonthTrueStart - secondsSincePodEndTime)/3600, 0.0)) secondsSinceLastMonthTrueStart as math(cond(secondsSincePodStartTime > ` + secondsSince["lastMonthStart"] + `, ` + secondsSince["lastMonthStart"] + `, secondsSincePodStartTime)) secondsSinceLastMonthTrueEnd as math(cond(secondsSincePodEndTime > ` + secondsSince["lastMonthEnd"] + `, secondsSincePodEndTime, ` + secondsSince["lastMonthEnd"] + `)) lastMonthTrueDurationInHours as math(cond(secondsSinceLastMonthTrueStart > secondsSinceLastMonthTrueEnd, (secondsSinceLastMonthTrueStart - secondsSinceLastMonthTrueEnd)/3600, 0.0)) secondsSinceLastLastMonthTrueStart as math(cond(secondsSincePodStartTime > ` + secondsSince["lastLastMonthStart"] + `, ` + secondsSince["lastLastMonthStart"] + `, secondsSincePodStartTime)) secondsSinceLastLastMonthTrueEnd as math(cond(secondsSincePodEndTime > ` + secondsSince["lastLastMonthEnd"] + `, secondsSincePodEndTime, ` + secondsSince["lastLastMonthEnd"] + `)) lastLastMonthTrueDurationInHours as math(cond(secondsSinceLastLastMonthTrueStart > secondsSinceLastLastMonthTrueEnd, (secondsSinceLastLastMonthTrueStart - secondsSinceLastLastMonthTrueEnd)/3600, 0.0)) isAlive as math(cond(isTerminated == 0, 1, 0)) pitPodCPU as math(cond(isTerminated == 0, cond(cpuRequestCount > 0, podCpu, 0.0), 0.0)) pitPodMemory as math(cond(isTerminated == 0, cond(memoryRequestCount > 0, podMemory, 0.0), 0.0)) pitPvcStorage as math(cond(isTerminated == 0, cond(storageRequestCount > 0, pvcStorage, 0.0), 0.0)) pitPodCPULimit as math(cond(isTerminated == 0, cond(cpuLimitCount > 0, podCpuLimit, 0.0), 0.0)) pitPodMemoryLimit as math(cond(isTerminated == 0, cond(memoryLimitCount > 0, podMemoryLimit, 0.0), 0.0)) mtdPodCPU as math(podCpu * currentMonthTrueDurationInHours) mtdPodMemory as math(podMemory * currentMonthTrueDurationInHours) mtdPvcStorage as math(pvcStorage * currentMonthTrueDurationInHours) mtdPodCPULimit as math(podCpuLimit * currentMonthTrueDurationInHours) mtdPodMemoryLimit as math(podMemoryLimit * currentMonthTrueDurationInHours) pricePerCPU as cpuPrice pricePerMemory as memoryPrice podCpuCost as math(mtdPodCPU * pricePerCPU) podMemoryCost as math(mtdPodMemory * pricePerMemory) podStorageCost as math(mtdPvcStorage * ` + models.DefaultStorageCostPerGBPerHour + `) podLiveCPUCostPerHour as math(pitPodCPU * pricePerCPU) podLiveMemoryCostPerHour as math(pitPodMemory * pricePerMemory) podLiveStorageCostPerHour as math(pitPvcStorage * ` + models.DefaultStorageCostPerGBPerHour + `) podCPUCostPerHour as math(podCpu * pricePerCPU) podMemoryCostPerHour as math(podMemory * pricePerMemory) podStorageCostPerHour as math(pvcStorage * ` + models.DefaultStorageCostPerGBPerHour + `) podCPUCostLastMonth as math(podCPUCostPerHour * lastMonthTrueDurationInHours) podMemoryCostLastMonth as math(podMemoryCostPerHour * lastMonthTrueDurationInHours) podStorageCostLastMonth as math(podStorageCostPerHour * lastMonthTrueDurationInHours) podCPUCostLastLastMonth as math(podCPUCostPerHour * lastLastMonthTrueDurationInHours) podMemoryCostLastLastMonth as math(podMemoryCostPerHour * lastLastMonthTrueDurationInHours) podStorageCostLastLastMonth as math(podStorageCostPerHour * lastLastMonthTrueDurationInHours) } group() { pitCPU: sum(val(pitPodCPU)) pitMemory: sum(val(pitPodMemory)) pitStorage: sum(val(pitPvcStorage)) pitCPULimit: sum(val(pitPodCPULimit)) pitMemoryLimit: sum(val(pitPodMemoryLimit)) mtdCPU: sum(val(mtdPodCPU)) mtdMemory: sum(val(mtdPodMemory)) mtdStorage: sum(val(mtdPvcStorage)) mtdCPULimit: sum(val(mtdPodCPULimit)) mtdMemoryLimit: sum(val(mtdPodMemoryLimit)) cpuCost: sum(val(podCpuCost)) memoryCost: sum(val(podMemoryCost)) storageCost: sum(val(podStorageCost)) cpuCostPerHour: sum(val(podLiveCPUCostPerHour)) memoryCostPerHour: sum(val(podLiveMemoryCostPerHour)) storageCostPerHour: sum(val(podLiveStorageCostPerHour)) lastMonthCPUCost: sum(val(podCPUCostLastMonth)) lastMonthMemoryCost: sum(val(podMemoryCostLastMonth)) lastMonthStorageCost: sum(val(podStorageCostLastMonth)) lastLastMonthCPUCost: sum(val(podCPUCostLastLastMonth)) lastLastMonthMemoryCost: sum(val(podMemoryCostLastLastMonth)) lastLastMonthStorageCost: sum(val(podStorageCostLastLastMonth)) livePods: sum(val(isAlive)) } }` } func getQueryForSubscribersRetrieval() string { return `query { subscribers(func: has(isSubscriber)) @filter(NOT(has(endTime))) { name spec { headers url } } }` } func getAllLivePodsQuery() string { return `query { pods(func: has(isPod)) @filter(NOT has(endTime)) { uid xid name } }` } func getQueryForPodsWithLabelFilter(labelFilter string) string { return `query { var(func: has(isLabel)) @filter(` + labelFilter + `) { podUIDs as ~label @filter(has(isPod)) { name } } pods(func: uid(podUIDs)) { uid name } }` } ================================================ FILE: pkg/controller/dgraph/models/query/resource.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "strconv" "github.com/Sirupsen/logrus" ) // Cluster resource constants const ( ContainerCheck = "isContainer" ContainerType = "container" IsProcFilter = "@filter(has(isProc))" DaemonsetCheck = "isDaemonset" DaemonsetType = "daemonset" IsPodFilter = "@filter(has(isPod))" DeploymentCheck = "isDeployment" DeploymentType = "deployment" IsReplicasetFilter = "@filter(has(isReplicaset))" JobCheck = "isJob" JobType = "job" NamespaceCheck = "isNamespace" NamespaceType = "namespace" NamespaceChildFilter = "@filter(has(isDeployment) OR has(isStatefulset) OR has(isJob) OR has(isDaemonset) OR (has(isReplicaset) AND (NOT has(deployment))))" NodeCheck = "isNode" NodeType = "node" PodCheck = "isPod" PodType = "pod" IsContainerFilter = "@filter(has(isContainer))" PVCheck = "isPersistentVolume" PVType = "pv" IsPVCFilter = "@filter(has(isPersistentVolumeClaim))" PVCCheck = "isPersistentVolumeClaim" PVCType = "pvc" ReplicasetCheck = "isReplicaset" ReplicasetType = "replicaset" StatefulsetCheck = "isStatefulset" StatefulsetType = "statefulset" ) // Resource structure type Resource struct { Check string Type string Name string ChildFilter string } // RetrieveResourceHierarchy returns hierarchy for a given resource func (r *Resource) RetrieveResourceHierarchy() JSONDataWrapper { if r.Name == All { logrus.Errorf("wrong type of query, empty name is given") return JSONDataWrapper{} } query := r.getQueryForHierarchy() return getJSONDataFromQuery(query) } // RetrieveResourceMetrics returns metrics for a given resource func (r *Resource) RetrieveResourceMetrics() JSONDataWrapper { if r.Name == All { logrus.Errorf("wrong type of query, empty name is given") return JSONDataWrapper{} } query := r.getQueryForResourceMetrics() return getJSONDataFromQuery(query) } func (r *Resource) getQueryForResourceMetrics() string { switch r.Type { case DeploymentType: return getQueryForDeploymentMetrics(r.Name) case NamespaceType: return getQueryForNamespaceMetrics(r.Name) case NodeType: return getQueryForNodeMetrics(r.Name) case PVType: return getQueryForPVMetrics(r.Name) case PVCType: return getQueryForPVCMetrics(r.Name) case ContainerType: return getQueryForContainerMetrics(r.Name) case PodType: cpuPriceInFloat64, memoryPriceInFloat64 := getPricePerResourceForPod(r.Name) cpuPrice := strconv.FormatFloat(cpuPriceInFloat64, 'f', 11, 64) memoryPrice := strconv.FormatFloat(memoryPriceInFloat64, 'f', 11, 64) return getQueryForPodMetrics(r.Name, cpuPrice, memoryPrice) } return r.getQueryForPodParentMetrics() } // getJSONDataFromQuery executes query and wraps the data in a desired structure(JSONDataWrapper) func getJSONDataFromQuery(query string) JSONDataWrapper { parentRoot := ParentWrapper{} err := executeQuery(query, &parentRoot) if err != nil || len(parentRoot.Parent) == 0 { logrus.Errorf("Unable to execute query, err: (%v)", err) return JSONDataWrapper{} } root := JSONDataWrapper{ Data: parentRoot.Parent[0], } return root } ================================================ FILE: pkg/controller/dgraph/models/query/resource_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "fmt" "testing" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/stretchr/testify/assert" ) func mockDgraphForResourceQueries(queryType, resourceName, resourceType string) { executeQuery = func(query string, root interface{}) error { if queryType == testPodPrices { newRoot, ok := root.(*podRoot) if !ok { return fmt.Errorf("wrong pod root received") } pod := models.Pod{ CPUPrice: testCPUPrice, MemoryPrice: testMemoryPrice, } newRoot.Pods = []models.Pod{pod} return nil } dummyParentWrapper, ok := root.(*ParentWrapper) if !ok { return fmt.Errorf("wrong root received") } var parent ParentWrapper if queryType == testMetrics { firstPodWithMetrics := Children{ Name: "pod-purser-1", Type: PodType, CPU: 0.25, Memory: 0.1, Storage: 1.2, CPUCost: 0.024, MemoryCost: 0.09, StorageCost: 0.1, } secondPodWithMetrics := Children{ Name: "pod-purser-2", Type: PodType, CPU: 0.15, Memory: 0.2, Storage: 0.2, CPUCost: 0.014, MemoryCost: 0.19, StorageCost: 0.01, } parent = ParentWrapper{ Name: resourceName, Type: resourceType, Children: []Children{firstPodWithMetrics, secondPodWithMetrics}, CPU: 0.40, Memory: 0.28, Storage: 1.4, CPUCost: 0.038, MemoryCost: 0.28, StorageCost: 0.11, } dummyParentWrapper.Parent = []ParentWrapper{parent} return nil } else if queryType == testHierarchy { firstPod := Children{ Name: "pod-purser-1", Type: PodType, } secondPod := Children{ Name: "pod-purser-2", Type: PodType, } parent = ParentWrapper{ Name: resourceName, Type: resourceType, Children: []Children{firstPod, secondPod}, } dummyParentWrapper.Parent = []ParentWrapper{parent} return nil } return fmt.Errorf("unable to retrieve data from dgraph") } } // TestRetrieveResourceHierarchyWithNameEmpty ... func TestRetrieveResourceHierarchyWithNameEmpty(t *testing.T) { input := &Resource{ Check: DaemonsetCheck, Type: DaemonsetType, Name: "", ChildFilter: IsPodFilter, } got := input.RetrieveResourceHierarchy() expected := JSONDataWrapper{} assert.Equal(t, expected, got) } // TestRetrieveResourceHierarchy ... func TestRetrieveResourceHierarchy(t *testing.T) { mockDgraphForResourceQueries(testHierarchy, testDaemonsetName, DaemonsetType) input := &Resource{ Check: DaemonsetCheck, Type: DaemonsetType, Name: testDaemonsetName, ChildFilter: IsPodFilter, } got := input.RetrieveResourceHierarchy() firstPod := Children{ Name: "pod-purser-1", Type: PodType, } secondPod := Children{ Name: "pod-purser-2", Type: PodType, } expected := JSONDataWrapper{ Data: ParentWrapper{ Name: testDaemonsetName, Type: DaemonsetType, Children: []Children{firstPod, secondPod}, }, } assert.Equal(t, expected, got) } // TestRetrieveResourceHierarchyWithDgraphError ... func TestRetrieveResourceHierarchyWithDgraphError(t *testing.T) { mockDgraphForResourceQueries(testWrongQuery, testDaemonsetName, DaemonsetType) input := &Resource{ Check: DaemonsetCheck, Type: DaemonsetType, Name: testDaemonsetName, ChildFilter: IsPodFilter, } got := input.RetrieveResourceHierarchy() expected := JSONDataWrapper{} assert.Equal(t, expected, got) } // TestRetrieveResourceMetricsWithNameEmpty ... func TestRetrieveResourceMetricsWithNameEmpty(t *testing.T) { input := &Resource{ Check: DaemonsetCheck, Type: DaemonsetType, Name: "", } got := input.RetrieveResourceMetrics() expected := JSONDataWrapper{} assert.Equal(t, expected, got) } // TestRetrieveDaemonsetMetrics ... func TestRetrieveDaemonsetMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testDaemonsetName, DaemonsetType) input := &Resource{ Check: DaemonsetCheck, Type: DaemonsetType, Name: testDaemonsetName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testDaemonsetName, DaemonsetType) assert.Equal(t, expected, got) } // TestRetrieveDeploymentMetrics ... func TestRetrieveDeploymentMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testResourceName, DeploymentType) input := &Resource{ Check: DeploymentCheck, Type: DeploymentType, Name: testResourceName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testResourceName, DeploymentType) assert.Equal(t, expected, got) } // TestRetrieveNamespacetMetrics ... func TestRetrieveNamespacetMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testResourceName, NamespaceType) input := &Resource{ Check: NamespaceCheck, Type: NamespaceType, Name: testResourceName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testResourceName, NamespaceType) assert.Equal(t, expected, got) } // TestRetrievePVMetrics ... func TestRetrievePVMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testResourceName, PVType) input := &Resource{ Check: PVCheck, Type: PVType, Name: testResourceName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testResourceName, PVType) assert.Equal(t, expected, got) } // TestRetrievePVCMetrics ... func TestRetrievePVCMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testResourceName, PVCType) input := &Resource{ Check: PVCCheck, Type: PVCType, Name: testResourceName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testResourceName, PVCType) assert.Equal(t, expected, got) } // TestRetrieveContainerMetrics ... func TestRetrieveContainerMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testResourceName, ContainerType) input := &Resource{ Check: ContainerCheck, Type: ContainerType, Name: testResourceName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testResourceName, ContainerType) assert.Equal(t, expected, got) } // TestRetrieveNodeMetrics ... func TestRetrieveNodeMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testResourceName, NodeType) input := &Resource{ Check: NodeCheck, Type: NodeType, Name: testResourceName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testResourceName, NodeType) assert.Equal(t, expected, got) } // TestRetrievePodMetrics ... func TestRetrievePodMetrics(t *testing.T) { mockDgraphForResourceQueries(testMetrics, testPodName, PodType) input := &Resource{ Check: PodCheck, Type: PodType, Name: testPodName, } got := input.RetrieveResourceMetrics() expected := getExpectedTestMetrics(testPodName, PodType) assert.Equal(t, expected, got) } func getExpectedTestMetrics(name, resourceType string) JSONDataWrapper { firstPodWithMetrics := Children{ Name: "pod-purser-1", Type: PodType, CPU: 0.25, Memory: 0.1, Storage: 1.2, CPUCost: 0.024, MemoryCost: 0.09, StorageCost: 0.1, } secondPodWithMetrics := Children{ Name: "pod-purser-2", Type: PodType, CPU: 0.15, Memory: 0.2, Storage: 0.2, CPUCost: 0.014, MemoryCost: 0.19, StorageCost: 0.01, } expected := JSONDataWrapper{ Data: ParentWrapper{ Name: name, Type: resourceType, Children: []Children{firstPodWithMetrics, secondPodWithMetrics}, CPU: 0.40, Memory: 0.28, Storage: 1.4, CPUCost: 0.038, MemoryCost: 0.28, StorageCost: 0.11, }, } return expected } ================================================ FILE: pkg/controller/dgraph/models/query/subscriber.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "github.com/vmware/purser/pkg/controller/dgraph/models" ) type subscriberRoot struct { Subscribers []models.SubscriberCRD `json:"subscribers"` } // RetrieveSubscribers gets all live subscribers func RetrieveSubscribers() ([]models.SubscriberCRD, error) { q := getQueryForSubscribersRetrieval() newRoot := subscriberRoot{} err := executeQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.Subscribers, nil } ================================================ FILE: pkg/controller/dgraph/models/query/subscriber_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query import ( "fmt" "testing" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/stretchr/testify/assert" ) func mockDgraphForSubscriberQueries(queryType string) { executeQuery = func(query string, root interface{}) error { dummySubscriberList, ok := root.(*subscriberRoot) if !ok { return fmt.Errorf("wrong root received") } if queryType == testRetrieveSubscribers { dummySubscriber := models.SubscriberCRD{ Name: "subscriber-purser", Spec: models.SubscriberSpec{ URL: "http://purser.com", }, } dummySubscriberList.Subscribers = []models.SubscriberCRD{dummySubscriber} return nil } return fmt.Errorf("no data found") } } // TestRetrieveSubscribersWithDgraphError ... func TestRetrieveSubscribersWithDgraphError(t *testing.T) { mockDgraphForSubscriberQueries(testWrongQuery) _, err := RetrieveSubscribers() assert.Error(t, err) } // TestRetrieveSubscribers ... func TestRetrieveSubscribers(t *testing.T) { mockDgraphForSubscriberQueries(testRetrieveSubscribers) got, err := RetrieveSubscribers() expected := []models.SubscriberCRD{{ Name: "subscriber-purser", Spec: models.SubscriberSpec{ URL: "http://purser.com", }, }} assert.Equal(t, expected, got) assert.NoError(t, err) } ================================================ FILE: pkg/controller/dgraph/models/query/types.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package query // Constants used in query parameters const ( All = "" Name = "name" Orphan = "orphan" View = "view" Physical = "physical" Logical = "logical" False = "false" ) // Children structure type Children struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` CPU float64 `json:"cpu,omitempty"` Memory float64 `json:"memory,omitempty"` Storage float64 `json:"storage,omitempty"` CPUCost float64 `json:"cpuCost,omitempty"` MemoryCost float64 `json:"memoryCost,omitempty"` StorageCost float64 `json:"storageCost,omitempty"` } // ParentWrapper structure type ParentWrapper struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Children []Children `json:"children,omitempty"` Parent []ParentWrapper `json:"parent,omitempty"` CPU float64 `json:"cpu,omitempty"` Memory float64 `json:"memory,omitempty"` Storage float64 `json:"storage,omitempty"` CPUCost float64 `json:"cpuCost,omitempty"` MemoryCost float64 `json:"memoryCost,omitempty"` StorageCost float64 `json:"storageCost,omitempty"` CPUAllocated float64 `json:"cpuAllocated,omitempty"` MemoryAllocated float64 `json:"memoryAllocated,omitempty"` StorageAllocated float64 `json:"storageAllocated,omitempty"` CPUCapacity float64 `json:"cpuCapacity,omitempty"` MemoryCapacity float64 `json:"memoryCapacity,omitempty"` StorageCapacity float64 `json:"storageCapacity,omitempty"` } // JSONDataWrapper structure type JSONDataWrapper struct { Data ParentWrapper `json:"data,omitempty"` } ================================================ FILE: pkg/controller/dgraph/models/rateCard.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "fmt" "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" ) // RateCard constants const ( IsRateCard = "isRateCard" IsNodePrice = "isNodePrice" IsStoragePrice = "isStoragePrice" RateCardXID = "purser-rateCard" ) // RateCard structure type RateCard struct { dgraph.ID IsRateCard bool `json:"isRateCard,omitempty"` CloudProvider string `json:"cloudProvider,omitempty"` Region string `json:"region,omitempty"` NodePrices []*NodePrice `json:"nodePrices,omitempty"` StoragePrices []*StoragePrice `json:"storagePrices,omitempty"` } // NodePrice structure // Unit of Node Price should be USD($)-(per Hour) type NodePrice struct { dgraph.ID IsNodePrice bool `json:"isNodePrice,omitempty"` InstanceType string `json:"instanceType,omitempty"` InstanceFamily string `json:"instanceFamily,omitempty"` OperatingSystem string `json:"operatingSystem,omitempty"` Price float64 `json:"price,omitempty"` PricePerCPU float64 `json:"cpuPrice,omitempty"` PricePerMemory float64 `json:"memoryPrice,omitempty"` } // StoragePrice structure // Unit of Storage Price should be USD($)-(per GB)-(per Hour) type StoragePrice struct { dgraph.ID IsStoragePrice bool `json:"isStoragePrice,omitempty"` VolumeType string `json:"volumeType,omitempty"` UsageType string `json:"usageType,omitempty"` Price float64 `json:"price,omitempty"` } // StoreRateCard given a cloudProvider and region it gets rate card and stores(create/update) in dgraph func StoreRateCard(rateCard *RateCard) { logrus.Debugf("IsRateCardNil: %v", rateCard == nil) if rateCard != nil { uid := dgraph.GetUID(RateCardXID, IsRateCard) if uid != "" { rateCard.ID = dgraph.ID{UID: uid, Xid: RateCardXID} } logrus.Debugf("RateCard: (%v)", rateCard) _, err := dgraph.MutateNode(rateCard, dgraph.CREATE) if err != nil { logrus.Errorf("Unable to store rateCard reason: %v", err) return } logrus.Infof("Successfully stored/updated rateCard") } } // StoreNodePrice given nodePrice and its XID it stores(create/update) in dgraph func StoreNodePrice(nodePrice *NodePrice, productXID string) string { uid := dgraph.GetUID(productXID, IsNodePrice) if uid != "" { nodePrice.ID = dgraph.ID{Xid: productXID, UID: uid} } logrus.Debugf("nodePrice: %v, productXID: %v\n", *nodePrice, productXID) assigned, err := dgraph.MutateNode(nodePrice, dgraph.CREATE) if err != nil { logrus.Errorf("Unable to store nodePrice: (%v), reason: %v", nodePrice, err) return "" } logrus.Debugf("Successfully stored/updated nodePrice: %v", productXID) if uid != "" { return uid } return assigned.Uids["blank-0"] } // StoreStoragePrice given storagePrice and its XID it stores(create/update) in dgraph func StoreStoragePrice(storagePrice *StoragePrice, productXID string) string { uid := dgraph.GetUID(productXID, IsStoragePrice) if uid != "" { storagePrice.ID = dgraph.ID{Xid: productXID, UID: uid} } logrus.Debugf("storagePrice: %v, productXID: %v\n", *storagePrice, productXID) assigned, err := dgraph.MutateNode(storagePrice, dgraph.CREATE) if err != nil { logrus.Errorf("Unable to store storagePrice: (%v), reason: %v", storagePrice, err) return "" } logrus.Debugf("Successfully stored/updated storagePrice: %v", productXID) if uid != "" { return uid } return assigned.Uids["blank-0"] } // retrieveNode given a node name it returns pointer to models.Node - nil in case of error func retrieveNode(name string) (*Node, error) { query := `query { nodes(func: has(isNode)) @filter(eq(name, "` + name + `")) { name type startTime endTime cpuCapacity memoryCapacity instanceType os } }` type root struct { Nodes []Node `json:"nodes"` } newRoot := root{} err := dgraph.ExecuteQuery(query, &newRoot) if err != nil { return nil, err } else if len(newRoot.Nodes) < 1 { return nil, fmt.Errorf("no node with name: %v", name) } return &newRoot.Nodes[0], nil } // retrieveNodePrice given a node name it returns pointer to models.Node - nil in case of error func retrieveNodePrice(xid string) (*NodePrice, error) { query := `query { nodePrices(func: has(isNodePrice)) @filter(eq(xid, "` + xid + `")) { instanceType instanceFamily operatingSystem price cpuPrice memoryPrice } }` type root struct { NodePrices []NodePrice `json:"nodePrices"` } newRoot := root{} err := dgraph.ExecuteQuery(query, &newRoot) if err != nil { return nil, err } else if len(newRoot.NodePrices) < 1 { return nil, fmt.Errorf("no node with xid: %v", xid) } return &newRoot.NodePrices[0], nil } // getPerUnitResourcePriceForNode returns price per cpu and price per memory func getPerUnitResourcePriceForNode(nodeName string) (float64, float64) { node, err := retrieveNode(nodeName) if err == nil { return getPricePerUnitResourceFromNodePrice(*node) } return DefaultCPUCostInFloat64, DefaultMemCostInFloat64 } func getPricePerUnitResourceFromNodePrice(node Node) (float64, float64) { nodePriceXID := node.InstanceType + "-" + node.OS nodePrice, err := retrieveNodePrice(nodePriceXID) if err == nil { return nodePrice.PricePerCPU, nodePrice.PricePerMemory } return DefaultCPUCostInFloat64, DefaultMemCostInFloat64 } ================================================ FILE: pkg/controller/dgraph/models/replicaset.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" ext_v1beta1 "k8s.io/api/extensions/v1beta1" ) // Dgraph Model Constants const ( IsReplicaset = "isReplicaset" ) // Replicaset schema in dgraph type Replicaset struct { dgraph.ID IsReplicaset bool `json:"isReplicaset,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Deployment *Deployment `json:"deployment,omitempty"` Pods []*Pod `json:"pod,omitempty"` Type string `json:"type,omitempty"` } func createReplicasetObject(replicaset ext_v1beta1.ReplicaSet) Replicaset { newReplicaset := Replicaset{ Name: "replicaset-" + replicaset.Name, IsReplicaset: true, Type: "replicaset", ID: dgraph.ID{Xid: replicaset.Namespace + ":" + replicaset.Name}, StartTime: replicaset.GetCreationTimestamp().Time.Format(time.RFC3339), } namespaceUID := CreateOrGetNamespaceByID(replicaset.Namespace) if namespaceUID != "" { newReplicaset.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: replicaset.Namespace}} } replicasetDeletionTimestamp := replicaset.GetDeletionTimestamp() if !replicasetDeletionTimestamp.IsZero() { newReplicaset.EndTime = replicasetDeletionTimestamp.Time.Format(time.RFC3339) newReplicaset.Xid += newReplicaset.EndTime newReplicaset.Name += "*" + newReplicaset.EndTime } setReplicasetOwners(&newReplicaset, replicaset) return newReplicaset } // StoreReplicaset create a new replicaset in the Dgraph and updates if already present. func StoreReplicaset(replicaset ext_v1beta1.ReplicaSet) (string, error) { xid := replicaset.Namespace + ":" + replicaset.Name uid := dgraph.GetUID(xid, IsReplicaset) newReplicaset := createReplicasetObject(replicaset) if uid != "" { newReplicaset.UID = uid } assigned, err := dgraph.MutateNode(newReplicaset, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } func setReplicasetOwners(r *Replicaset, replicaset ext_v1beta1.ReplicaSet) { owners := replicaset.GetObjectMeta().GetOwnerReferences() if owners == nil { return } for _, owner := range owners { if owner.Kind == "Deployment" { deploymentXID := replicaset.Namespace + ":" + owner.Name deploymentUID := CreateOrGetDeploymentByID(deploymentXID) if deploymentUID != "" { r.Deployment = &Deployment{ID: dgraph.ID{UID: deploymentUID, Xid: deploymentXID}} } } else { log.Error("Unknown owner type " + owner.Kind + " for replicaset.") } } } // CreateOrGetReplicasetByID returns the uid of namespace if exists, // otherwise creates the replicaset and returns uid. func CreateOrGetReplicasetByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsReplicaset) if uid != "" { return uid } d := Replicaset{ ID: dgraph.ID{Xid: xid}, Name: xid, IsReplicaset: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/service.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "fmt" "time" log "github.com/Sirupsen/logrus" "github.com/dgraph-io/dgo/protos/api" "github.com/vmware/purser/pkg/controller/dgraph" api_v1 "k8s.io/api/core/v1" ) // Dgraph Model Constants const ( IsService = "isService" ) // Service model structure in Dgraph type Service struct { dgraph.ID IsService bool `json:"isService,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Pod []*Pod `json:"pod,omitempty"` Interacts []*Service `json:"interacts,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Type string `json:"type,omitempty"` } func newService(svc api_v1.Service) (*api.Assigned, error) { newService := Service{ Name: "service-" + svc.Name, IsService: true, Type: "service", ID: dgraph.ID{Xid: svc.Namespace + ":" + svc.Name}, StartTime: svc.GetCreationTimestamp().Time.Format(time.RFC3339), } namespaceUID := CreateOrGetNamespaceByID(svc.Namespace) if namespaceUID != "" { newService.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: svc.Namespace}} } return dgraph.MutateNode(newService, dgraph.CREATE) } // StoreService create a new node in the Dgraph if it is not present. func StoreService(service api_v1.Service) error { xid := service.Namespace + ":" + service.Name uid := dgraph.GetUID(xid, IsService) if uid == "" { assigned, err := newService(service) if err != nil { return err } log.Infof("Service with xid: (%s) persisted in dgraph", xid) uid = assigned.Uids["blank-0"] } svcDeletionTimestamp := service.GetDeletionTimestamp() if !svcDeletionTimestamp.IsZero() { et := svcDeletionTimestamp.Time.Format(time.RFC3339) updatedService := Service{ ID: dgraph.ID{Xid: xid + et, UID: uid}, EndTime: et, Name: "service-" + service.Name + "*" + et, } _, err := dgraph.MutateNode(updatedService, dgraph.UPDATE) return err } return nil } // StoreServicesInteraction stores the service interaction data in the Dgraph func StoreServicesInteraction(sourceServiceXID string, destinationServicesXIDs []string) error { uid := dgraph.GetUID(sourceServiceXID, IsService) if uid == "" { log.Println("Source Service " + sourceServiceXID + " is not persisted yet.") return fmt.Errorf("source service: %s is not persisted yet", sourceServiceXID) } services := retrieveServicesFromServicesXIDs(destinationServicesXIDs) source := Service{ ID: dgraph.ID{UID: uid, Xid: sourceServiceXID}, Interacts: services, } _, err := dgraph.MutateNode(source, dgraph.UPDATE) return err } // StorePodServiceEdges saves pods in Services object in the dgraph func StorePodServiceEdges(svcXID string, podsXIDsInService []string) error { svcUID := dgraph.GetUID(svcXID, IsService) if svcUID != "" { svcPods := retrievePodsFromPodsXIDs(podsXIDsInService) updatedService := Service{ ID: dgraph.ID{UID: svcUID, Xid: svcXID}, Pod: svcPods, } _, err := dgraph.MutateNode(updatedService, dgraph.UPDATE) return err } return fmt.Errorf("service with xid: (%s) not in dgraph", svcXID) } // RetrieveAllServices returns all pods in the dgraph func RetrieveAllServices() ([]Service, error) { const q = `query { service(func: has(isService)) { name interacts @facets { name } pod { name } } }` type root struct { Services []Service `json:"service"` } newRoot := root{} err := dgraph.ExecuteQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.Services, nil } // RetrieveAllServicesWithDstPods returns all pods in the dgraph func RetrieveAllServicesWithDstPods() ([]Service, error) { const q = `query { services(func: has(isService)) { xid name pod { name interacts @facets { name } } } }` type root struct { Services []Service `json:"services"` } newRoot := root{} err := dgraph.ExecuteQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.Services, nil } // RetrieveServiceList ... func RetrieveServiceList() ([]Service, error) { const q = `query { serviceList(func: has(isService)) { name } }` type root struct { ServiceList []Service `json:"serviceList"` } newRoot := root{} err := dgraph.ExecuteQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.ServiceList, nil } func retrieveServicesFromServicesXIDs(svcsXIDs []string) []*Service { services := []*Service{} for _, svcXID := range svcsXIDs { svcUID := dgraph.GetUID(svcXID, IsService) if svcUID == "" { continue } service := &Service{ ID: dgraph.ID{UID: svcUID, Xid: svcXID}, } services = append(services, service) } return services } ================================================ FILE: pkg/controller/dgraph/models/statefulset.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" apps_v1beta1 "k8s.io/api/apps/v1beta1" ) // Dgraph Model Constants const ( IsStatefulset = "isStatefulset" ) // Statefulset schema in dgraph type Statefulset struct { dgraph.ID IsStatefulset bool `json:"isStatefulset,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Namespace *Namespace `json:"namespace,omitempty"` Pods []*Pod `json:"pods,omitempty"` Type string `json:"type,omitempty"` } func createStatefulsetObject(statefulset apps_v1beta1.StatefulSet) Statefulset { newStatefulset := Statefulset{ Name: "statefulset-" + statefulset.Name, IsStatefulset: true, Type: "statefulset", ID: dgraph.ID{Xid: statefulset.Namespace + ":" + statefulset.Name}, StartTime: statefulset.GetCreationTimestamp().Time.Format(time.RFC3339), } namespaceUID := CreateOrGetNamespaceByID(statefulset.Namespace) if namespaceUID != "" { newStatefulset.Namespace = &Namespace{ID: dgraph.ID{UID: namespaceUID, Xid: statefulset.Namespace}} } statefulsetDeletionTimestamp := statefulset.GetDeletionTimestamp() if !statefulsetDeletionTimestamp.IsZero() { newStatefulset.EndTime = statefulsetDeletionTimestamp.Time.Format(time.RFC3339) newStatefulset.Xid += newStatefulset.EndTime newStatefulset.Name += "*" + newStatefulset.EndTime } return newStatefulset } // StoreStatefulset create a new statefulset in the Dgraph and updates if already present. func StoreStatefulset(statefulset apps_v1beta1.StatefulSet) (string, error) { xid := statefulset.Namespace + ":" + statefulset.Name uid := dgraph.GetUID(xid, IsStatefulset) newStatefulset := createStatefulsetObject(statefulset) if uid != "" { newStatefulset.UID = uid } assigned, err := dgraph.MutateNode(newStatefulset, dgraph.CREATE) if err != nil { return "", err } return assigned.Uids["blank-0"], nil } // CreateOrGetStatefulsetByID returns the uid of namespace if exists, // otherwise creates the stateful and returns uid. func CreateOrGetStatefulsetByID(xid string) string { if xid == "" { return "" } uid := dgraph.GetUID(xid, IsStatefulset) if uid != "" { return uid } d := Statefulset{ ID: dgraph.ID{Xid: xid}, Name: xid, IsStatefulset: true, } assigned, err := dgraph.MutateNode(d, dgraph.CREATE) if err != nil { log.Fatal(err) return "" } return assigned.Uids["blank-0"] } ================================================ FILE: pkg/controller/dgraph/models/subscriber.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package models import ( "github.com/Sirupsen/logrus" "time" subscribers_v1 "github.com/vmware/purser/pkg/apis/subscriber/v1" "github.com/vmware/purser/pkg/controller/dgraph" ) // Dgraph Model Constants const ( IsSubscriber = "isSubscriber" ) // SubscriberCRD schema in dgraph type SubscriberCRD struct { dgraph.ID IsSubscriber bool `json:"isSubscriber,omitempty"` Name string `json:"name,omitempty"` StartTime string `json:"startTime,omitempty"` EndTime string `json:"endTime,omitempty"` Type string `json:"type,omitempty"` Spec SubscriberSpec `json:"spec"` } // SubscriberSpec definition details type SubscriberSpec struct { Name string `json:"name"` Headers map[string]string `json:"headers"` URL string `json:"url"` } func createSubscriberCRDObject(subscriber subscribers_v1.Subscriber) SubscriberCRD { newSubscriber := SubscriberCRD{ Name: subscriber.Name, IsSubscriber: true, Type: subscribers_v1.SubscriberGroup, ID: dgraph.ID{Xid: "subscriber-" + subscriber.Name}, StartTime: subscriber.GetCreationTimestamp().Time.Format(time.RFC3339), Spec: SubscriberSpec{ Name: subscriber.Spec.Name, Headers: subscriber.Spec.Headers, URL: subscriber.Spec.URL, }, } deletionTimestamp := subscriber.GetDeletionTimestamp() if !deletionTimestamp.IsZero() { newSubscriber.EndTime = deletionTimestamp.Time.Format(time.RFC3339) } return newSubscriber } // StoreSubscriberCRD create a new subscriber CRD in the Dgraph and updates if already present. func StoreSubscriberCRD(subscriber subscribers_v1.Subscriber) (string, error) { xid := "subscriber-" + subscriber.Name uid := dgraph.GetUID(xid, IsSubscriber) if uid != "" { return uid, nil } newSubscriber := createSubscriberCRDObject(subscriber) assigned, err := dgraph.MutateNode(newSubscriber, dgraph.CREATE) if err != nil { return "", err } logrus.Infof("Subscriber: (%v) persisted in dgraph", subscriber.Name) return assigned.Uids["blank-0"], nil } ================================================ FILE: pkg/controller/dgraph/purge.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package dgraph import ( "github.com/vmware/purser/pkg/controller/utils" log "github.com/Sirupsen/logrus" "time" ) type resource struct { ID } // RemoveResourcesInactive deletes all resources which have their deletion time stamp before // the start of current month. func RemoveResourcesInactive() { err := removeOldDeletedResources() if err != nil { log.Println(err) } err = removeOldDeletedPods() if err != nil { log.Error(err) } } func removeOldDeletedResources() error { uids, err := retrieveResourcesWithEndTimeBeforeCurrentMonthStart() if err != nil { return err } if len(uids) == 0 { log.Println("No old deleted resources are present in dgraph") return nil } _, err = MutateNode(uids, DELETE) return err } func removeOldDeletedPods() error { uids, err := retrievePodsWithEndTimeBeforeThreeMonths() if err != nil { return err } if len(uids) == 0 { log.Println("No old deleted pods are present in dgraph") return nil } _, err = MutateNode(uids, DELETE) return err } func retrieveResourcesWithEndTimeBeforeCurrentMonthStart() ([]resource, error) { q := `query { resources(func: le(endTime, "` + utils.ConverTimeToRFC3339(utils.GetCurrentMonthStartTime()) + `")) @filter(NOT(has(isPod))) { uid } }` type root struct { Resources []resource `json:"resources"` } newRoot := root{} err := ExecuteQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.Resources, nil } func retrievePodsWithEndTimeBeforeThreeMonths() ([]resource, error) { q := `query { resources(func: le(endTime, "` + utils.ConverTimeToRFC3339(utils.GetCurrentMonthStartTime().Add(-time.Hour*24*30*2)) + `")) @filter(has(isPod)) { uid } }` type root struct { Resources []resource `json:"resources"` } newRoot := root{} err := ExecuteQuery(q, &newRoot) if err != nil { return nil, err } return newRoot.Resources, nil } ================================================ FILE: pkg/controller/discovery/executer/exec.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package executer import ( "bytes" "fmt" "io" "strings" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/remotecommand" ) // ExecToPodThroughAPI uninteractively exec to the pod with the command specified. func ExecToPodThroughAPI(conf controller.Config, pod corev1.Pod, command, containerName string, stdin io.Reader) (string, string, error) { // Prepare the API URL used to execute another process within the Pod. In this case, // we'll run a remote shell. req := conf.Kubeclient.CoreV1().RESTClient().Post(). Resource("pods"). Name(pod.Name). Namespace(pod.Namespace). SubResource("exec") scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { return "", "", fmt.Errorf("error adding to scheme: %v", err) } parameterCodec := runtime.NewParameterCodec(scheme) req.VersionedParams(&corev1.PodExecOptions{ Command: strings.Fields(command), Container: containerName, Stdin: stdin != nil, Stdout: true, Stderr: true, TTY: false, }, parameterCodec) log.Debug("Request URL:", req.URL().String()) exec, err := remotecommand.NewSPDYExecutor(conf.KubeConfig, "POST", req.URL()) if err != nil { return "", "", fmt.Errorf("error while creating Executor: %v", err) } // Connect this process' std{in,out,err} to the remote shell process. var stdout, stderr bytes.Buffer err = exec.Stream(remotecommand.StreamOptions{ Stdin: stdin, Stdout: &stdout, Stderr: &stderr, Tty: false, }) if err != nil { return "", "", fmt.Errorf("error in Stream: %v", err) } return stdout.String(), stderr.String(), nil } ================================================ FILE: pkg/controller/discovery/generator/graph.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package generator import ( "strconv" "github.com/vmware/purser/pkg/controller/dgraph/models" ) // Node represents each node in the graph // ID: unique id of pod // Label: pod name // Title: string "pods" // Value: number of times pod has communicated with others // Group: Connected component number, used for coloring different components in different colors // CID: list of all services the pod belongs to. type Node struct { ID int `json:"id"` Label string `json:"label"` Title string `json:"title"` Value int `json:"value"` Group int `json:"group"` Cid []string `json:"cid"` } // Edge represents each edge in the graph // From: unique id of source pod // TO: unique id of destination pod // Title: string containing number of times these two pods communicated type Edge struct { From int `json:"from"` To int `json:"to"` Title string `json:"title"` } var ( uniqueID int nodes *[]Node edges *[]Edge ) // GetGraphNodes returns graph-nodes for pod interactions func GetGraphNodes() []Node { return *nodes } // GetGraphEdges returns graph-edges for pod interactions func GetGraphEdges() []Edge { return *edges } // GeneratePodNodesAndEdges ... func GeneratePodNodesAndEdges(pods []models.Pod) { uniqueID = 0 uniqueIDs, numConnections, inboundAndOutboundConnections := getPodUniqueIDsAndNumConnections(pods) podNodes := createPodNodes(pods, uniqueIDs, numConnections, inboundAndOutboundConnections) podEdges := createPodEdges(pods, uniqueIDs) setGraphNodes(podNodes) setGraphEdges(podEdges) } func getPodUniqueIDsAndNumConnections(pods []models.Pod) (map[string]int, map[string]int, map[string]int) { uniqueIDs := make(map[string]int) numConnections := make(map[string]int) inboundAndOutboundConnections := make(map[string]int) for _, pod := range pods { setPodUniqueIDsAndNumConnections(pod, uniqueIDs, numConnections, inboundAndOutboundConnections) } return uniqueIDs, numConnections, inboundAndOutboundConnections } func setPodUniqueIDsAndNumConnections(pod models.Pod, uniqueIDs, numConnections, inboundAndOutboundConnections map[string]int) { if _, isPresent := uniqueIDs[pod.Name]; !isPresent { uniqueID++ uniqueIDs[pod.Name] = uniqueID numConnections[pod.Name] = 0 for _, dstPod := range pod.Pods { numConnections[pod.Name] += int(dstPod.Count) inboundAndOutboundConnections[pod.Name] += int(dstPod.Count) inboundAndOutboundConnections[dstPod.Name] += int(dstPod.Count) } } } func createPodNodes(pods []models.Pod, uniqueIDs, numConnections, inboundAndOutboundConnections map[string]int) []Node { nodes := []Node{} duplicateChecker := make(map[string]bool) for _, pod := range pods { if _, isNotOrphan := inboundAndOutboundConnections[pod.Name]; isNotOrphan { if _, isPresent := duplicateChecker[pod.Name]; !isPresent { duplicateChecker[pod.Name] = true svcCid := []string{} for _, svc := range pod.Cid { svcCid = append(svcCid, svc.Name) } newPodNode := createPodNode(pod.Name, uniqueIDs[pod.Name], numConnections[pod.Name], svcCid) nodes = append(nodes, newPodNode) } } } return nodes } func createPodEdges(pods []models.Pod, uniqueIDs map[string]int) []Edge { edges := []Edge{} for _, pod := range pods { srcID := uniqueIDs[pod.Name] for _, dstPod := range pod.Pods { destID := uniqueIDs[dstPod.Name] edges = append(edges, createPodEdge(srcID, destID, int(dstPod.Count))) } } return edges } func createPodNode(podName string, podID int, podConnections int, cid []string) Node { return Node{ ID: podID, Label: podName, Title: "pods", Value: podConnections, Group: 1, // needed for UI, colors different group differently(not needed for our use case) Cid: cid, } } func createPodEdge(fromID int, toID int, count int) Edge { return Edge{ From: fromID, To: toID, Title: strconv.Itoa(count) + " times communicated", } } func setGraphNodes(podNodes []Node) { nodes = &podNodes } func setGraphEdges(podEdges []Edge) { edges = &podEdges } ================================================ FILE: pkg/controller/discovery/linker/podlinks.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package linker import ( "strings" "sync" log "github.com/Sirupsen/logrus" corev1 "k8s.io/api/core/v1" "github.com/vmware/purser/pkg/controller/dgraph/models" ) // InteractionsWrapper ... type InteractionsWrapper struct { PodInteractions map[string](map[string]float64) ProcessToPodInteraction map[string](map[string]bool) ContainerProcessInteraction map[string][]string } // podIPTable: maps pod name with pod IP address // podToPodTable: maps src pod to the interacting dest pod along with the interaction frequency count. var ( podIPTable = make(map[string]string) podToPodTable = make(map[string](map[string]float64)) ) var ( mu sync.Mutex ) const ( // KeySpliter splits the key into resource namespace and name used for processing Xids KeySpliter = ":" ) // PopulatePodIPTable populates the podIP<->podName map func PopulatePodIPTable(pods *corev1.PodList) { for _, pod := range pods.Items { podIP := pod.Status.PodIP podIPTable[podIP] = pod.Namespace + KeySpliter + pod.Name } } // GenerateAndStorePodInteractions generates source to destination Pod mapping and stores it in Dgraph. func GenerateAndStorePodInteractions() { log.Info("Storing Pod Interactions ....") for srcPodName, communication := range podToPodTable { dstPods := []string{} counts := []float64{} for dstPodName, count := range communication { dstPods = append(dstPods, dstPodName) counts = append(counts, count) } err := models.StorePodsInteraction(srcPodName, dstPods, counts) if err != nil { log.Errorf("failed to store pod interaction in Dgraph %v", err) } } log.Info("Finished storing pod interactions.") } // PopulateMappingTables updates PodToPodTable func PopulateMappingTables(tcpDump []string, pod corev1.Pod, process Process, containerName string, interactions *InteractionsWrapper) { podXID := pod.Namespace + KeySpliter + pod.Name containerXID := podXID + KeySpliter + containerName procXID := containerXID + KeySpliter + process.ID + KeySpliter + process.Name populateContainerProcessTable(containerXID, procXID, interactions) for _, address := range tcpDump { address := strings.Split(address, KeySpliter) srcIP, dstIP := address[0], address[2] srcName, dstName := podIPTable[srcIP], podIPTable[dstIP] updatePodInteractions(srcName, dstName, interactions) updatePodProcessInteractions(procXID, dstName, interactions) } } func updatePodInteractions(srcName, dstName string, interactions *InteractionsWrapper) { if dstName != "" && srcName != "" { log.Debugf("pod interactions srcName: (%s), dstName: (%s)", srcName, dstName) if _, ok := interactions.PodInteractions[srcName]; !ok { interactions.PodInteractions[srcName] = make(map[string]float64) } if _, isPresent := interactions.PodInteractions[srcName][dstName]; !isPresent { interactions.PodInteractions[srcName][dstName] = 1 } else { interactions.PodInteractions[srcName][dstName]++ } } } // UpdatePodToPodTable ... func UpdatePodToPodTable(podInteractions map[string](map[string]float64)) { mu.Lock() for srcPod, interaction := range podInteractions { if _, ok := podToPodTable[srcPod]; !ok { podToPodTable[srcPod] = interaction } else { for dstPod, count := range interaction { if _, isPresent := podToPodTable[srcPod][dstPod]; !isPresent { podToPodTable[srcPod][dstPod] = count } else { podToPodTable[srcPod][dstPod] += count } } } } mu.Unlock() } ================================================ FILE: pkg/controller/discovery/linker/processlinks.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package linker import ( "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph/models" ) // Process holds the details for the executing processes inside the container type Process struct { ID, Name string } // StoreProcessInteractions stores process, container to process edge, process to pods edge func StoreProcessInteractions(containerProcessInteraction map[string][]string, processPodInteraction map[string](map[string]bool), creationTime time.Time) { for containerXID, procsXIDs := range containerProcessInteraction { for _, procXID := range procsXIDs { podsXIDs := []string{} for podXID := range processPodInteraction[procXID] { podsXIDs = append(podsXIDs, podXID) } err := models.StoreProcess(procXID, containerXID, podsXIDs, creationTime) if err != nil { log.Errorf("failed to store process details: %s, err: (%v)", procXID, err) } } err := models.StoreContainerProcessEdge(containerXID, procsXIDs) if err != nil { log.Errorf("failed to store edge from container: %s to procs, err: (%v)", containerXID, err) } } } func populateContainerProcessTable(containerXID, procXID string, interactions *InteractionsWrapper) { if _, isPresent := interactions.ContainerProcessInteraction[containerXID]; !isPresent { interactions.ContainerProcessInteraction[containerXID] = []string{} } interactions.ContainerProcessInteraction[containerXID] = append(interactions.ContainerProcessInteraction[containerXID], procXID) } func updatePodProcessInteractions(procXID, dstName string, interactions *InteractionsWrapper) { if dstName != "" { if _, isPresent := interactions.ProcessToPodInteraction[procXID]; !isPresent { interactions.ProcessToPodInteraction[procXID] = make(map[string]bool) } interactions.ProcessToPodInteraction[procXID][dstName] = true } } ================================================ FILE: pkg/controller/discovery/linker/servicelinks.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package linker import ( "sync" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph/models" corev1 "k8s.io/api/core/v1" ) var ( podToSvcTable = make(map[string][]string) serviceMu sync.Mutex ) // PopulatePodToServiceTable populates the pod<->service map func PopulatePodToServiceTable(svc corev1.Service, pods *corev1.PodList) { var podsXIDsInService []string serviceKey := svc.Namespace + KeySpliter + svc.Name serviceMu.Lock() for _, pod := range pods.Items { podKey := pod.Namespace + KeySpliter + pod.Name podToSvcTable[podKey] = append(podToSvcTable[podKey], serviceKey) podsXIDsInService = append(podsXIDsInService, podKey) } serviceMu.Unlock() err := models.StorePodServiceEdges(serviceKey, podsXIDsInService) if err != nil { log.Errorf("failed to store pod services edges: %s\n", err) } } // GenerateAndStoreSvcInteractions parses through pod interactions and generates a source to // destination service interaction. func GenerateAndStoreSvcInteractions() { services, err := models.RetrieveAllServicesWithDstPods() if err != nil { log.Errorf("failed to fetch services: %s\n", err) } for _, service := range services { destinationPods := getDestinationPods(service.Pod) destinationServices := getServicesXIDsFromPods(destinationPods) err = models.StoreServicesInteraction(service.Xid, destinationServices) if err != nil { log.Errorf("failed to store services interactions: %s\n", err) } } } func getDestinationPods(podsInService []*models.Pod) []*models.Pod { var destinationPods []*models.Pod for _, pod := range podsInService { destinationPods = append(destinationPods, pod.Pods...) } return destinationPods } func getServicesXIDsFromPods(pods []*models.Pod) []string { var servicesXIDs []string duplicateChecker := make(map[string]bool) for _, pod := range pods { svcsXIDs := podToSvcTable[pod.Xid] for _, svcXID := range svcsXIDs { if _, isPresent := duplicateChecker[svcXID]; !isPresent { duplicateChecker[svcXID] = true servicesXIDs = append(servicesXIDs, svcXID) } } } return servicesXIDs } ================================================ FILE: pkg/controller/discovery/processor/container.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package processor import ( "fmt" "strings" "github.com/vmware/purser/pkg/controller" "github.com/vmware/purser/pkg/controller/discovery/linker" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/discovery/executer" "github.com/vmware/purser/pkg/controller/utils" corev1 "k8s.io/api/core/v1" ) func processContainerDetails(conf controller.Config, pod corev1.Pod, containers []corev1.Container) linker.InteractionsWrapper { interactions := linker.InteractionsWrapper{ PodInteractions: make(map[string](map[string]float64)), ProcessToPodInteraction: make(map[string](map[string]bool)), ContainerProcessInteraction: make(map[string][]string), } for _, container := range containers { pidList, cmdList := getPIDList(conf, pod, container.Name) for index, pid := range pidList { process := linker.Process{ID: pid, Name: cmdList[index]} getProcessDump(conf, pod, container.Name, process, &interactions) } } return interactions } func getPIDList(conf controller.Config, pod corev1.Pod, containerName string) ([]string, []string) { command := "ps -A -o pid,cmd" output, err := executeCommandInPod(conf, pod, command, containerName) if err != nil { return nil, nil } pidCMDList := strings.Split(output, "\n") var pidList, cmdList []string for _, pidCMD := range pidCMDList { if pidCMD != "" { pidCMDClean := strings.Split((strings.TrimSpace(pidCMD)), " ") pidList = append(pidList, pidCMDClean[0]) cmdList = append(cmdList, strings.Join(pidCMDClean[1:], "-")) } } // ignore first line i.e, PID CMD headers if len(pidList) >= 1 { return pidList[1:], cmdList[1:] } return pidList, cmdList } func getProcessDump(conf controller.Config, pod corev1.Pod, containerName string, process linker.Process, interactions *linker.InteractionsWrapper) { //get tcp information from /proc/pid/net/tcp for each process if process.ID != "" { tcpCommand := "cat /proc/" + process.ID + "/net/tcp" tcpOutput, err := executeCommandInPod(conf, pod, tcpCommand, containerName) if err == nil { //to clean dump only to have required fields tcpDump := utils.PurgeTCPData(tcpOutput) linker.PopulateMappingTables(tcpDump, pod, process, containerName, interactions) } tcp6Command := "cat /proc/" + process.ID + "/net/tcp6" tcp6Output, err := executeCommandInPod(conf, pod, tcp6Command, containerName) if err == nil { //to clean dump only to have required fields tcp6Dump := utils.PurgeTCP6Data(tcp6Output) linker.PopulateMappingTables(tcp6Dump, pod, process, containerName, interactions) } } } func executeCommandInPod(conf controller.Config, pod corev1.Pod, command, containerName string) (string, error) { output, stderr, err := executer.ExecToPodThroughAPI(conf, pod, command, containerName, nil) if err != nil { log.Debugf("Failed `exec`ing to the container %q, command %q Error: %+v", pod.Name, command, err) } if len(stderr) > 0 { log.Warnf("stderr: %v", stderr) err = fmt.Errorf("stderr: %v", stderr) } return output, err } ================================================ FILE: pkg/controller/discovery/processor/pod.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package processor import ( "github.com/vmware/purser/pkg/controller/utils" "sync" "github.com/vmware/purser/pkg/controller" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/discovery/linker" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const maxGoRoutines = 20 var wg sync.WaitGroup // ProcessPodInteractions fetches details of all the running processes in each container of // each pod in a given namespace and generates a 1:1 mapping between the communicating pods. func ProcessPodInteractions(conf controller.Config) { k8sPods := utils.RetrievePodList(conf.Kubeclient, metav1.ListOptions{}) if k8sPods == nil { log.Info("No pods retrieved from cluster") return } linker.PopulatePodIPTable(k8sPods) processPodDetails(conf, k8sPods) linker.GenerateAndStorePodInteractions() log.Infof("Successfully generated Pod To Pod mapping.") } func processPodDetails(conf controller.Config, pods *corev1.PodList) { podsCount := len(pods.Items) log.Infof("Processing total of (%d) Pods.", podsCount) freeRoutines := maxGoRoutines numChannelsReceived := 0 ch := make(chan int, 1) wg.Add(podsCount) { for index, pod := range pods.Items { log.Debugf("Processing Pod: (%s), (%d/%d) ... ", pod.Name, index+1, podsCount) // wait for a free goroutine if freeRoutines < 1 { // wait for a go routine to send to channel i.e, it will wait until a go routine finishes. <-ch numChannelsReceived++ } // decrease 1 from freeRoutines before starting a new go routine freeRoutines-- go func(pod corev1.Pod, index int) { defer wg.Done() containers := pod.Spec.Containers interactions := processContainerDetails(conf, pod, containers) linker.UpdatePodToPodTable(interactions.PodInteractions) linker.StoreProcessInteractions(interactions.ContainerProcessInteraction, interactions.ProcessToPodInteraction, pod.GetCreationTimestamp().Time) log.Debugf("Finished processing Pod: (%s), (%d/%d)", pod.Name, index+1, podsCount) // increase 1 from freeRoutines after processing a pod. freeRoutines++ // send 1 to channel ch <- 1 }(pod, index) } } // receive channel from remaining go routines for i := 0; i < podsCount-numChannelsReceived; i++ { <-ch } wg.Wait() close(ch) } ================================================ FILE: pkg/controller/discovery/processor/svc.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package processor import ( "github.com/vmware/purser/pkg/controller/utils" "sync" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller" "github.com/vmware/purser/pkg/controller/discovery/linker" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" ) var svcwg sync.WaitGroup // ProcessServiceInteractions parses through the list of services and it's associated pods to // generate a 1:1 mapping between the communicating services. func ProcessServiceInteractions(conf controller.Config) { services := utils.RetrieveServiceList(conf.Kubeclient, metav1.ListOptions{}) if services == nil { log.Info("No services retrieved from cluster") return } processServiceDetails(conf.Kubeclient, services) linker.GenerateAndStoreSvcInteractions() log.Infof("Successfully generated Service To Service mapping.") } func processServiceDetails(client *kubernetes.Clientset, services *corev1.ServiceList) { svcCount := len(services.Items) log.Infof("Processing total of (%d) Services.", svcCount) svcwg.Add(svcCount) { for index, svc := range services.Items { log.Debugf("Processing Service (%d/%d): %s ", index+1, svcCount, svc.GetName()) go func(svc corev1.Service, index int) { defer svcwg.Done() selectorSet := labels.Set(svc.Spec.Selector) if selectorSet != nil { options := metav1.ListOptions{ LabelSelector: selectorSet.AsSelector().String(), } pods := utils.RetrievePodList(client, options) if pods != nil { linker.PopulatePodToServiceTable(svc, pods) } } log.Debugf("Finished processing Service (%d/%d)", index+1, svcCount) }(svc, index) } } svcwg.Wait() } ================================================ FILE: pkg/controller/eventprocessor/notifier.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package eventprocessor import ( "bytes" "encoding/json" "fmt" "github.com/vmware/purser/pkg/controller/dgraph/models" "net/http" "time" log "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller" ) // ReadSize defines the default payload read size const ReadSize uint32 = 50 type notifier struct { url string headers map[string]string } func notifySubscribers(payload []*interface{}, subscribers []models.SubscriberCRD) { notifiers := getNotifiers(subscribers) for _, n := range notifiers { req, err := n.createNewRequest(payload) if err != nil { log.Errorf("Failed to unmarshal payload and create new request %v", err) } else { err := retry(3, time.Second, func() error { return sendData(req) }) if err != nil { log.Errorf("Notification to subscriber %v failed after 3 retries %v", n.url, err) } } } } func (n notifier) createNewRequest(payload []*interface{}) (*http.Request, error) { payloadWrapper := controller.PayloadWrapper{Data: payload} jsonStr, err := json.Marshal(payloadWrapper) if err != nil { return nil, fmt.Errorf("error unmarshalling payload %v", err) } req, err := http.NewRequest("POST", n.url, bytes.NewBuffer(jsonStr)) if err != nil { return nil, fmt.Errorf("error creating HTTP request %v", err) } n.setReqHeaders(req) return req, nil } func sendData(req *http.Request) error { client := &http.Client{} resp, err := client.Do(req) if err != nil { return fmt.Errorf("error sending data to %v: %v", req.URL, err) } if resp != nil { if resp.StatusCode != 200 { return fmt.Errorf("payload data posting failed for %v, %s", req.URL, resp.Status) } log.Debugf("Payload data posted successfully for %v", req.URL) } return nil } func (n *notifier) setReqHeaders(r *http.Request) { r.Header.Set("Content-Type", "application/json") for key, value := range n.headers { r.Header.Set(key, value) } } func getNotifiers(subscribers []models.SubscriberCRD) []*notifier { var notifiers []*notifier if len(subscribers) > 0 { for _, sub := range subscribers { notifier := ¬ifier{ url: sub.Spec.URL, headers: sub.Spec.Headers, } notifiers = append(notifiers, notifier) } } else { log.Debug("No subscribers available.") } return notifiers } func retry(attempts int, sleep time.Duration, fn func() error) error { if err := fn(); err != nil { if attempts--; attempts > 0 { time.Sleep(sleep) return retry(attempts, 2*sleep, fn) } return err } return nil } ================================================ FILE: pkg/controller/eventprocessor/processor.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package eventprocessor import ( "encoding/json" "time" "github.com/vmware/purser/pkg/controller/dgraph/models/query" log "github.com/Sirupsen/logrus" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" subcriber_v1 "github.com/vmware/purser/pkg/apis/subscriber/v1" "github.com/vmware/purser/pkg/controller" "github.com/vmware/purser/pkg/controller/dgraph/models" apps_v1beta1 "k8s.io/api/apps/v1beta1" batch_v1 "k8s.io/api/batch/v1" api_v1 "k8s.io/api/core/v1" ext_v1beta1 "k8s.io/api/extensions/v1beta1" ) // ProcessEvents processes the event and notifies the subscribers. func ProcessEvents(conf *controller.Config) { for { conf.RingBuffer.PrintDetails() for { data, size := conf.RingBuffer.ReadN(ReadSize) if size == 0 { log.Debug("No new events to process.") break } ProcessPayloads(data, conf) subscribers, err := query.RetrieveSubscribers() if err == nil { notifySubscribers(data, subscribers) } else { log.Errorf("unable to retrieve subscribers from dgraph: %v", err) } conf.RingBuffer.RemoveN(size) conf.RingBuffer.PrintDetails() } time.Sleep(10 * time.Second) } } // ProcessPayloads store payload info in dgraph. If payload is of type group then it updates its group spec func ProcessPayloads(payloads []*interface{}, conf *controller.Config) { for _, event := range payloads { payload := (*event).(*controller.Payload) handlePayloadBasedOnResource(payload, conf) } } // nolint: gocyclo func handlePayloadBasedOnResource(payload *controller.Payload, conf *controller.Config) { var err error switch payload.ResourceType { case "Pod": pod := api_v1.Pod{} unmarshalPayload(payload, &pod) err = models.StorePod(pod) case "Service": service := api_v1.Service{} unmarshalPayload(payload, &service) err = models.StoreService(service) case "Node": node := api_v1.Node{} unmarshalPayload(payload, &node) _, err = models.StoreNode(node) case "Namespace": ns := api_v1.Namespace{} unmarshalPayload(payload, &ns) _, err = models.StoreNamespace(ns) case "Deployment": deployment := apps_v1beta1.Deployment{} unmarshalPayload(payload, &deployment) _, err = models.StoreDeployment(deployment) case "ReplicaSet": replicaset := ext_v1beta1.ReplicaSet{} unmarshalPayload(payload, &replicaset) _, err = models.StoreReplicaset(replicaset) case "StatefulSet": statefulset := apps_v1beta1.StatefulSet{} unmarshalPayload(payload, &statefulset) _, err = models.StoreStatefulset(statefulset) case "PersistentVolume": pv := api_v1.PersistentVolume{} unmarshalPayload(payload, &pv) _, err = models.StorePersistentVolume(pv, conf.Kubeclient) case "PersistentVolumeClaim": pvc := api_v1.PersistentVolumeClaim{} unmarshalPayload(payload, &pvc) _, err = models.StorePersistentVolumeClaim(pvc) case "DaemonSet": daemonset := ext_v1beta1.DaemonSet{} unmarshalPayload(payload, &daemonset) _, err = models.StoreDaemonset(daemonset) case "Job": job := batch_v1.Job{} unmarshalPayload(payload, &job) _, err = models.StoreJob(job) case "Group": groupCRD := &groups_v1.Group{} unmarshalPayload(payload, &groupCRD) handlePayloadForGroup(payload, conf, groupCRD.Name) case "Subscriber": subscriberCRD := subcriber_v1.Subscriber{} unmarshalPayload(payload, &subscriberCRD) _, err = models.StoreSubscriberCRD(subscriberCRD) } checkDgraphError(payload.ResourceType, err) } func unmarshalPayload(payload *controller.Payload, resource interface{}) { err := json.Unmarshal([]byte(payload.Data), resource) if err != nil { log.Errorf("Error un marshalling payload " + payload.Data) } } func checkDgraphError(resource string, err error) { if err != nil { log.Errorf("Error while persisting %s %v", resource, err) } } func handlePayloadForGroup(payload *controller.Payload, conf *controller.Config, groupName string) { if payload.EventType == controller.Delete { models.DeleteGroup(groupName) } else { group, err := conf.Groupcrdclient.Get(groupName) if err != nil { log.Errorf("Unable to get group from client: (%v)", err) } else { UpdateGroup(group, conf.Groupcrdclient) } } } ================================================ FILE: pkg/controller/eventprocessor/sync.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package eventprocessor import ( "time" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph/models/query" "github.com/vmware/purser/pkg/controller/utils" "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) // SyncCluster will handle missed events func SyncCluster(kubeClient *kubernetes.Clientset) { endTime := time.Now().Format(time.RFC3339) syncPods(kubeClient, endTime) } // syncPods handles missed creation and deletion of pod events func syncPods(kubeClient *kubernetes.Clientset, endTime string) { logrus.Infof("[SYNC] started syncing pods") livePodsFromDgraph := query.RetrieveAllLivePods() logrus.Infof("[SYNC] number of livePodsFromDgraph: %d", len(livePodsFromDgraph)) podsInCluster := utils.RetrievePodList(kubeClient, v1.ListOptions{}) if podsInCluster == nil { logrus.Errorf("[SYNC] got no podsInCluster, aborting sync") return } logrus.Infof("[SYNC] number of pods in cluster: %d", len(podsInCluster.Items)) handleDeadPodsAndNewPods(livePodsFromDgraph, podsInCluster, endTime) logrus.Infof("[SYNC] finished syncing of pods") } // if dead pods end time isn't updated in dgraph this function will update it // if an pod creation event is missed then this function will create a new pod in dgraph func handleDeadPodsAndNewPods(livePodsFromDgraph []models.Pod, podsInCluster *corev1.PodList, endTime string) { // create a map from pod xid to k8s pod pointer podXIDToPod := make(map[string]*corev1.Pod) for _, pod := range podsInCluster.Items { xid := pod.Namespace + ":" + pod.Name if _, isPresent := podXIDToPod[xid]; !isPresent { podXIDToPod[xid] = &pod } } var deadPods []models.Pod podsXIDs := make(map[string]bool) // create a map from pod xid to bool for _, pod := range livePodsFromDgraph { if _, isAlive := podXIDToPod[pod.Xid]; !isAlive { // pod is in dgraph but not in cluster -> pod got deleted but end time not updated in dgraph -> // missed pod deletion event -> update pod in dgraph with end time deadPod := models.Pod{ ID: dgraph.ID{Xid: pod.Xid + endTime, UID: pod.UID}, EndTime: endTime, Name: "pod-" + pod.Name + "*" + endTime, } deadPods = append(deadPods, deadPod) } if _, isPresent := podsXIDs[pod.Xid]; !isPresent { podsXIDs[pod.Xid] = true } } // update deletion time stamps for dead pods _, err := dgraph.MutateNode(deadPods, dgraph.UPDATE) if err != nil { logrus.Errorf("[SYNC] unable to update deleted pods with end time: # deleted pods: %d, err: %v", len(deadPods), err) } // create new pod if it isn't in dgraph for podXID, pod := range podXIDToPod { if _, isPresent := podsXIDs[podXID]; !isPresent { // pod is in cluster but not in dgraph -> missed pod creation event -> create new pod in dgraph err = models.StorePod(*pod) if err != nil { logrus.Errorf("[SYNC] Error while persisting pod: %s, err: %v", podXID, err) } } } } ================================================ FILE: pkg/controller/eventprocessor/updater.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package eventprocessor import ( "time" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/vmware/purser/pkg/controller/dgraph/models/query" "github.com/vmware/purser/pkg/controller/utils" log "github.com/Sirupsen/logrus" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" groupsClient_v1 "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // UpdateGroups retrieve all groups and updates them func UpdateGroups(groupCRDClient *groupsClient_v1.GroupClient) { log.Infof("Started updating groups") groups := utils.RetrieveGroupList(groupCRDClient, meta_v1.ListOptions{}) if groups == nil { log.Debugf("GroupList is nil") return } log.Debugf("Retrieved groups of length: %d", len(groups.Items)) for _, group := range groups.Items { UpdateGroup(group, groupCRDClient) } } // UpdateGroup given a group it updates its spec with metrics func UpdateGroup(group *groups_v1.Group, groupCRDClient *groupsClient_v1.GroupClient) { if group == nil { log.Warn("Received empty group to update") return } groupMetrics := getGroupMetrics(group) log.Debugf("GroupMetrics computed from dgraph data: (%v)", groupMetrics) group.Spec.MTDMetrics = &groups_v1.GroupMetrics{ CPURequest: groupMetrics.MTDCpu, MemoryRequest: groupMetrics.MTDMemory, StorageClaim: groupMetrics.MTDStorage, } group.Spec.PITMetrics = &groups_v1.GroupMetrics{ CPURequest: groupMetrics.PITCpu, MemoryRequest: groupMetrics.PITMemory, StorageClaim: groupMetrics.PITStorage, } group.Spec.MTDCost = &groups_v1.Cost{ CPUCost: groupMetrics.CostCPU, MemoryCost: groupMetrics.CostMemory, StorageCost: groupMetrics.CostStorage, TotalCost: groupMetrics.CostCPU + groupMetrics.CostMemory + groupMetrics.CostStorage, } group.Spec.PerHourCost = &groups_v1.Cost{ CPUCost: groupMetrics.CostCPUPerHour, MemoryCost: groupMetrics.CostMemoryPerHour, StorageCost: groupMetrics.CostStoragePerHour, TotalCost: groupMetrics.CostCPUPerHour + groupMetrics.CostMemoryPerHour + groupMetrics.CostStoragePerHour, } group.Spec.LastMonthCost = &groups_v1.Cost{ CPUCost: groupMetrics.LastMonthCPUCost, MemoryCost: groupMetrics.LastMonthMemoryCost, StorageCost: groupMetrics.LastMonthStorageCost, TotalCost: groupMetrics.LastMonthCPUCost + groupMetrics.LastMonthMemoryCost + groupMetrics.LastMonthStorageCost, } group.Spec.LastLastMonthCost = &groups_v1.Cost{ CPUCost: groupMetrics.LastLastMonthCPUCost, MemoryCost: groupMetrics.LastLastMonthMemoryCost, StorageCost: groupMetrics.LastLastMonthStorageCost, TotalCost: groupMetrics.LastLastMonthCPUCost + groupMetrics.LastLastMonthMemoryCost + groupMetrics.LastLastMonthStorageCost, } group.Spec.LastUpdated = time.Now() _, err := groupCRDClient.Update(group) if err != nil { log.Errorf("unable to update group: (%s), error: (%v)", group.Name, err) return } log.Debugf("Updated group spec: (%v)", group.Spec) log.Infof("Group spec is updated with metrics for group: (%s)", group.Name) _, err = models.CreateOrUpdateGroup(group, groupMetrics.PodsCount) if err != nil { log.Errorf("unable to create or update group in dgraph: (%s), error: (%v)", group.Name, err) } } func getGroupMetrics(group *groups_v1.Group) query.GroupMetrics { log.Debugf("Group: (%v), expressions: (%v)", group.Name, group.Spec.Expressions) // for each label-expression retrieve UIDs of pods that satisfy the label-expression podUIDsFromExpressions := getPodUIDsFromAllExpressions(group.Spec.Expressions) log.Debugf("Group: (%v), pod uids: (%v)", group.Name, podUIDsFromExpressions) // Across all the podUIDs computed from label-expressions, map pod's UID with number of occurrences of it podUIDsCounter := mapPodUIDsToNumberOfOccurences(podUIDsFromExpressions) log.Debugf("Group: (%v), pod uids counter: (%v)", group.Name, podUIDsCounter) // if number of occurrences of UID == number of expressions that means the pod satisfies all the expressions(i.e, AND) // get uid-query to retrieve such pods i.e, "uid1, uid2, uid2..." uidQueryForPods := getUIDQueryForPods(podUIDsCounter, len(group.Spec.Expressions)) log.Debugf("Group: (%v), uidQuery: (%v)", group.Name, uidQueryForPods) // get group metrics groupMetrics, err := query.RetrieveGroupMetricsFromPodUIDs(uidQueryForPods) if err != nil { log.Errorf("Unable to retrieve group metrics, group: %v, UIDs: (%v)", group.Name, uidQueryForPods) return query.GroupMetrics{} } return groupMetrics } // for each label-expression retrieve UIDs of pods that satisfy the label-expression // appends results from each expression and returns the array of such results i.e, // [[pod1-from-exp1, pod2-from-exp1], [pod1-from-exp2], [pod1-from-exp3, pod2-from-exp3, pod3-from-exp3]] func getPodUIDsFromAllExpressions(expressions map[string]map[string][]string) [][]string { var podsUIDsFromExpressions [][]string for _, selector := range expressions { labelFilter := query.CreateFilterFromListOfLabels(selector) podsUIDsFromSelector, err := query.RetrievePodsUIDsByLabelsFilter(labelFilter) if err == nil { podsUIDsFromExpressions = append(podsUIDsFromExpressions, podsUIDsFromSelector) } } return podsUIDsFromExpressions } // across all the podUIDs computed from label-expressions, map pod's UID with number of occurrences of it and return map func mapPodUIDsToNumberOfOccurences(podsFromExpressions [][]string) map[string]int { podsUIDsCounter := make(map[string]int) for _, podsFromExpression := range podsFromExpressions { for _, pod := range podsFromExpression { if _, isPresent := podsUIDsCounter[pod]; !isPresent { podsUIDsCounter[pod] = 0 } podsUIDsCounter[pod]++ } } return podsUIDsCounter } // returns UIDs for pods that satisfy (number of its occurrences == expressions count) i.e, // if number of occurrences of UID == number of expressions that means the pod satisfies all the expressions(-> AND) // returns uid-query(i.e, "uid1, uid2, uid2...") that can retrieve desired pods func getUIDQueryForPods(podsUIDsCounter map[string]int, expressionsCount int) string { separator := ", " isFirst := true var uidQueryForPods string for podUID, count := range podsUIDsCounter { if count == expressionsCount { if !isFirst { uidQueryForPods += separator } else { isFirst = false } uidQueryForPods += podUID } } return uidQueryForPods } ================================================ FILE: pkg/controller/metrics/metrics.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package metrics import ( log "github.com/Sirupsen/logrus" api_v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) // Metrics types type Metrics struct { CPULimit *resource.Quantity MemoryLimit *resource.Quantity CPURequest *resource.Quantity MemoryRequest *resource.Quantity } // CalculatePodStatsFromContainers returns the cumulative metrics from the containers. func CalculatePodStatsFromContainers(pod *api_v1.Pod) *Metrics { cpuLimit := &resource.Quantity{} memoryLimit := &resource.Quantity{} cpuRequest := &resource.Quantity{} memoryRequest := &resource.Quantity{} for _, c := range pod.Spec.Containers { limits := c.Resources.Limits if limits != nil { cpuLimit.Add(*limits.Cpu()) memoryLimit.Add(*limits.Memory()) } requests := c.Resources.Requests if requests != nil { cpuRequest.Add(*requests.Cpu()) memoryRequest.Add(*requests.Memory()) } } return &Metrics{ CPULimit: cpuLimit, MemoryLimit: memoryLimit, CPURequest: cpuRequest, MemoryRequest: memoryRequest, } } // PrintPodStats displays the pod stats. func PrintPodStats(pod *api_v1.Pod, metrics *Metrics) { log.Printf("Pod:\t%s\n", pod.Name) log.Printf("\tCPU Limit = %s\n", metrics.CPULimit.String()) log.Printf("\tMemory Limit = %s\n", metrics.MemoryLimit.String()) log.Printf("\tCPU Request = %s\n", metrics.CPURequest.String()) log.Printf("\tMemory Request = %s\n", metrics.MemoryRequest.String()) } ================================================ FILE: pkg/controller/payload.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package controller import meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" // PayloadWrapper holds additional information about payload type PayloadWrapper struct { Data []*interface{} `json:"data"` } // Payload holds payload information type Payload struct { Key string `json:"key"` EventType string `json:"eventType"` ResourceType string `json:"resourceType"` CloudType string `json:"cloudType"` Data string `json:"data"` CaptureTime meta_v1.Time } ================================================ FILE: pkg/controller/persistentVolume.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package controller import ( log "github.com/Sirupsen/logrus" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" "github.com/vmware/purser/pkg/controller/utils" api_v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // UpdatePodVolumeClaims activePodVolumeClaims: current active(bounded) podVolumeClaims for the pod. // old podVolumeClaims: Claims(map) for the pod before this update. // compares old podVolumeClaims and activePodVolumeClaims. This function hadles 3 cases. // Case Unbound pvc: // Present as 'bounded' in old podVolumeClaims. Not present in activePodVolumeClaims. // Case New PVC: // Not present in old podVolumeClaims. Present in activePodVolumeClaims. // Case Bound an unbounded pvc: // Present as 'unbounded' in old podVolumeClaims. Present in activePodVolumeClaims. func UpdatePodVolumeClaims(pod api_v1.Pod, podDetails groups_v1.PodDetails, eventTime meta_v1.Time) groups_v1.PodDetails { activePodVolumeClaims := getactivePodVolumeClaims(pod) podVolumeClaims := podDetails.PodVolumeClaims if podVolumeClaims == nil { podVolumeClaims = map[string]*groups_v1.PersistentVolumeClaim{} } for claimName := range podVolumeClaims { // isPresent: true if pvc is present in activePodVolumeClaims. _, isPresent := activePodVolumeClaims[claimName] // isBounded: true if pvc is present in old podVolumeClaims as 'bounded' isBounded := checkBounded(podVolumeClaims[claimName]) if (!isPresent) && isBounded { // Case Unbound pvc log.Info("Unbounded pvc: " + claimName + " from pod: " + podDetails.Name) podVolumeClaims[claimName].UnboundTimes = append(podVolumeClaims[claimName].UnboundTimes, eventTime) } else if isPresent && (!isBounded) { // Case Bound an unbounded pvc log.Info("Bounded pvc: " + claimName + " to pod: " + podDetails.Name) podVolumeClaims[claimName].BoundTimes = append(podVolumeClaims[claimName].BoundTimes, eventTime) } } // check for new pvc for claimKey := range activePodVolumeClaims { _, isPresent := podVolumeClaims[claimKey] if !isPresent { // Case New PVC log.Info("Bounded new pvc: " + claimKey + "to pod: " + podDetails.Name) podVolumeClaims[claimKey] = activePodVolumeClaims[claimKey] } } // TODO: handle Case Resizing of PVC podDetails.PodVolumeClaims = podVolumeClaims return podDetails } func getactivePodVolumeClaims(pod api_v1.Pod) map[string]*groups_v1.PersistentVolumeClaim { namespace := pod.GetNamespace() podVolumeClaims := map[string]*groups_v1.PersistentVolumeClaim{} for j := 0; j < len(pod.Spec.Volumes); j++ { vol := pod.Spec.Volumes[j] if vol.PersistentVolumeClaim != nil { claimName := vol.PersistentVolumeClaim.ClaimName podVolumeClaims[claimName] = collectPersistentVolumeClaim(claimName, namespace) podVolumeClaims[claimName].BoundTimes = append(podVolumeClaims[claimName].BoundTimes, pod.GetCreationTimestamp()) } } return podVolumeClaims } func collectPersistentVolumeClaim(claimName, namespace string) *groups_v1.PersistentVolumeClaim { pvc, err := Kubeclient.CoreV1().PersistentVolumeClaims(namespace).Get(claimName, meta_v1.GetOptions{}) if isPVCError(err, claimName) { return nil } request := pvc.Spec.Resources.Requests["storage"].DeepCopy() capacity := pvc.Status.Capacity["storage"].DeepCopy() return &groups_v1.PersistentVolumeClaim{ Name: pvc.GetObjectMeta().GetName(), VolumeName: pvc.Spec.VolumeName, RequestSizeInGB: []float64{utils.BytesToGB(request.Value())}, CapacityAllocatedInGB: []float64{utils.BytesToGB(capacity.Value())}, BoundTimes: []meta_v1.Time{}, UnboundTimes: []meta_v1.Time{}, } } // PvcHandlePodDeletion action to be taken when pod is deleted. // Unbound all bounded pvcs. func PvcHandlePodDeletion(podDetails *groups_v1.PodDetails) { pvMap := podDetails.PodVolumeClaims for claimName := range pvMap { if checkBounded(pvMap[claimName]) { log.Info("Unbounded pvc: " + claimName + " Reason deletion of pod: " + podDetails.Name) pvMap[claimName].UnboundTimes = append(pvMap[claimName].UnboundTimes, podDetails.EndTime) } } podDetails.PodVolumeClaims = pvMap } // If length of bound times is 1 more than unbound times it means that the pvc is still bound to pod. func checkBounded(pvc *groups_v1.PersistentVolumeClaim) bool { return len(pvc.BoundTimes)-len(pvc.UnboundTimes) == 1 } // false if no error func isPVCError(err error, claimName string) bool { if err == nil { return false } if errors.IsNotFound(err) { log.Errorf("Persistent Volume Claim %s not found\n", claimName) return true } if statusError, isStatus := err.(*errors.StatusError); isStatus { log.Errorf("Error getting persistence volume Claim %s : %v\n", claimName, statusError.ErrStatus.Message) return true } log.Errorf("Error: Unable to get PVC: %s\n", claimName) return true } ================================================ FILE: pkg/controller/types.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package controller import ( groups_v1 "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" subscriber_v1 "github.com/vmware/purser/pkg/client/clientset/typed/subscriber/v1" "github.com/vmware/purser/pkg/controller/buffering" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) // These are the event types supported for controllers const ( Create = "create" Delete = "delete" Update = "update" ) // Resource contains resource configuration type Resource struct { Pod bool `json:"po"` Node bool `json:"node"` PersistentVolume bool `json:"pv"` PersistentVolumeClaim bool `json:"pvc"` Service bool `json:"service"` ReplicaSet bool `json:"replicaset"` StatefulSet bool `json:"statefulset"` Deployment bool `json:"deployment"` Job bool `json:"job"` DaemonSet bool `json:"daemonset"` Namespace bool `json:"namespace"` Group bool `json:"groups.vmware.purser.com"` Subscriber bool `json:"subscribers.vmware.purser.com"` } // Config contains config objects type Config struct { KubeConfig *rest.Config Resource Resource `json:"resource"` RingBuffer *buffering.RingBuffer Groupcrdclient *groups_v1.GroupClient Subscriberclient *subscriber_v1.SubscriberClient Kubeclient *kubernetes.Clientset } ================================================ FILE: pkg/controller/utils/jsonutils.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "encoding/json" "net/http" log "github.com/Sirupsen/logrus" ) // JSONMarshal marshal object and returns in byte. If there is an error then it return nil. func JSONMarshal(obj interface{}) []byte { bytes, err := json.Marshal(obj) if err != nil { log.Error(err) } return bytes } // GetJSONResponse retrieves json response and converts it to target object. // Returns error if any failure is encountered. func GetJSONResponse(client *http.Client, url string, target interface{}) error { resp, err := client.Get(url) if err != nil { return err } defer closeResponse(resp) return json.NewDecoder(resp.Body).Decode(target) } func closeResponse(resp *http.Response) { err := resp.Body.Close() if err != nil { log.Errorf("unable to close response body. Reason: %v", err) } } ================================================ FILE: pkg/controller/utils/k8sUtils.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( log "github.com/Sirupsen/logrus" storagev1 "k8s.io/api/storage/v1" groupsv1 "github.com/vmware/purser/pkg/apis/groups/v1" groups "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" api_v1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) // k8sUtils constants const ( StorageDefault = "purser-default" ) // RetrievePodList returns list of pods in the given namespace. func RetrievePodList(client *kubernetes.Clientset, options metav1.ListOptions) *corev1.PodList { pods, err := client.CoreV1().Pods(metav1.NamespaceAll).List(options) if err != nil { log.Errorf("failed to retrieve pods: %v", err) return nil } return pods } // RetrieveNodeList returns list of nodes func RetrieveNodeList(client *kubernetes.Clientset, options metav1.ListOptions) *corev1.NodeList { nodes, err := client.CoreV1().Nodes().List(options) if err != nil { log.Errorf("failed to retrieve nodes: %v", err) return nil } return nodes } // RetrieveServiceList returns list of services in the given namespace. func RetrieveServiceList(client *kubernetes.Clientset, options metav1.ListOptions) *corev1.ServiceList { services, err := client.CoreV1().Services(metav1.NamespaceAll).List(options) if err != nil { log.Errorf("failed to retrieve services: %v", err) return nil } return services } // RetrieveGroupList returns list of group CRDs in the given namespace. func RetrieveGroupList(groupClient *groups.GroupClient, options metav1.ListOptions) *groupsv1.GroupList { crdGroups, err := groupClient.List(options) if err != nil { log.Errorf("failed to retrieve group list: %v ", err) return nil } return crdGroups } // RetrieveStorageClass returns storage class with the given name. Nil if error is encountered func RetrieveStorageClass(client *kubernetes.Clientset, options metav1.GetOptions, name string) (*storagev1.StorageClass, error) { storageClass, err := client.StorageV1().StorageClasses().Get(name, options) if err != nil { log.Errorf("failed to retrieve storage class: %s, err: %v", name, err) return nil, err } return storageClass, err } // GetFinalStorageTypeOfPV ... // input: persistent volume // output: the type(final) of PV's storage class // i.e., if PV has storage class A, A is of type B(storage class) and so on.. // until a storage class X is of its own type X. Then this function returns the final type of PV's storage as X // // "purser-default" is returned in special cases: // 1. if A is of type B, if B is of type A (i.e., if a cycle is found) // 2. an error is encountered // 3. if A is not having any type i.e., "" (empty string case) func GetFinalStorageTypeOfPV(pv api_v1.PersistentVolume, client *kubernetes.Clientset) string { cycleChecker := make(map[string]bool) log.Debugf("PV: %s, storageClass: %s", pv.Name, pv.Spec.StorageClassName) return getFinalTypeOfStorageClass(client, pv.Spec.StorageClassName, cycleChecker) } // getFinalTypeOfStorageClass // this is helper function for func getStorageType func getFinalTypeOfStorageClass(client *kubernetes.Clientset, storageClassName string, cycleChecker map[string]bool) string { if _, isVisited := cycleChecker[storageClassName]; isVisited { return StorageDefault } cycleChecker[storageClassName] = true storageClass, err := RetrieveStorageClass(client, metav1.GetOptions{}, storageClassName) if err != nil { return StorageDefault } storageType := storageClass.Parameters["type"] if storageType == "" { return StorageDefault } else if storageType == storageClassName { return storageClassName } return getFinalTypeOfStorageClass(client, storageType, cycleChecker) } ================================================ FILE: pkg/controller/utils/purge.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "encoding/hex" "fmt" "strings" "github.com/Sirupsen/logrus" ) // PurgeTCPData handles IP conversion from Hex to Dec and cleans up data to contain only // inter pod address information. func PurgeTCPData(data string) []string { var tcpDump []string tcpDumpHex := getTCPDumpHexFromData(data) for _, address := range tcpDumpHex { localIP, localPort := hexToDecIP(address[6:14]), address[15:19] remoteIP, remotePort := hexToDecIP(address[20:28]), address[29:33] if isLocalHost(localIP, remoteIP) { continue } addressMapping := localIP + ":" + localPort + ":" + remoteIP + ":" + remotePort tcpDump = append(tcpDump, addressMapping) } return tcpDump } // PurgeTCP6Data handles IP conversion from Hex to Dec and cleans up data to contain only // inter pod address information. func PurgeTCP6Data(data string) []string { var tcpDump []string tcpDumpHex := getTCPDumpHexFromData(data) for _, address := range tcpDumpHex { localIP, localPort := hexToDecIP(address[30:38]), address[39:43] remoteIP, remotePort := hexToDecIP(address[68:76]), address[77:81] if isLocalHost(localIP, remoteIP) { continue } addressMapping := localIP + ":" + localPort + ":" + remoteIP + ":" + remotePort tcpDump = append(tcpDump, addressMapping) } return tcpDump } func getTCPDumpHexFromData(data string) []string { tcpDumpHex := strings.Split(data, "\n") if len(tcpDumpHex) <= 1 { return nil } // ignore title and last one as it is empty tcpDumpHex = tcpDumpHex[1 : len(tcpDumpHex)-1] return tcpDumpHex } func hexToDecIP(hexIP string) string { decBytes, err := hex.DecodeString(hexIP) if err != nil { logrus.Warnf("failed to decode string to hex %v", err) } return fmt.Sprintf("%v.%v.%v.%v", decBytes[3], decBytes[2], decBytes[1], decBytes[0]) } func isLocalHost(localIP, remoteIP string) bool { return strings.Compare(localIP, "0.0.0.0") == 0 || strings.Compare(localIP, "127.0.0.1") == 0 || strings.Compare(remoteIP, "0.0.0.0") == 0 } ================================================ FILE: pkg/controller/utils/purge_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "testing" "github.com/vmware/purser/test/utils" ) func TestHexToDecIP(t *testing.T) { act := hexToDecIP("030310AC") exp := "172.16.3.3" utils.Equals(t, exp, act) } ================================================ FILE: pkg/controller/utils/timeUtils.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import "time" // GetCurrentMonthStartTime returns month start time as k8s apimachinery Time object func GetCurrentMonthStartTime() time.Time { now := time.Now() monthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.Local) return monthStart } // ConverTimeToRFC3339 returns query time in RFC3339 format func ConverTimeToRFC3339(queryTime time.Time) string { return queryTime.Format(time.RFC3339) } // GetSecondsSince returns number of seconds since query time func GetSecondsSince(queryTime time.Time) float64 { return time.Since(queryTime).Seconds() } // GetHoursRemainingInCurrentMonth returns number of hours remaining in the month func GetHoursRemainingInCurrentMonth() float64 { now := time.Now() monthEnd := time.Date(now.Year(), now.Month(), 30, 23, 59, 0, 0, time.Local) return -time.Since(monthEnd).Hours() } ================================================ FILE: pkg/controller/utils/unitConversions.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "strconv" log "github.com/Sirupsen/logrus" "k8s.io/apimachinery/pkg/api/resource" ) // BytesToGB converts from bytes(int64) to GB(float64) func BytesToGB(val int64) float64 { return float64BytesToFloat64GB(float64(val)) } // ConvertToFloat64GB quantity to float64 GB func ConvertToFloat64GB(quantity *resource.Quantity) float64 { return float64BytesToFloat64GB(resourceToFloat64(quantity)) } // ConvertToFloat64CPU quantity to float64 vCPU func ConvertToFloat64CPU(quantity *resource.Quantity) float64 { return resourceToFloat64(quantity) } // AddResourceAToResourceB ... func AddResourceAToResourceB(resA, resB *resource.Quantity) { if resA != nil { resB.Add(*resA) } } // float64BytesToFloat64GB from bytes (float64) to GB(float64) func float64BytesToFloat64GB(val float64) float64 { return val / (1024.0 * 1024.0 * 1024.0) } // resourceToFloat64 ... func resourceToFloat64(quantity *resource.Quantity) float64 { decVal := quantity.AsDec() decValueFloat, err := strconv.ParseFloat(decVal.String(), 64) if err != nil { log.Errorf("error while converting into string: (%s) to float\n", decVal.String()) } return decValueFloat // 0 if not isSuccess } ================================================ FILE: pkg/controller/utils/unitConversions_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "testing" "github.com/vmware/purser/test/utils" "k8s.io/apimachinery/pkg/api/resource" ) func TestBytesToGB(t *testing.T) { act := BytesToGB(124235312345978) exp := 115703.15095221438 utils.Equals(t, exp, act) } func TestConvertToFloat64GB(t *testing.T) { quantities := getTestQuantities() exp := [3]float64{0.011175870895385742, 0.01171875, 0.011175870895385742} for index, quantity := range quantities { act := ConvertToFloat64GB(&quantity) utils.Equals(t, exp[index], act) } } func TestConvertToFloat64CPU(t *testing.T) { quantities := getTestQuantities() exp := [3]float64{1.2e+07, 1.2582912e+07, 1.2e+07} for index, quantity := range quantities { act := ConvertToFloat64CPU(&quantity) utils.Equals(t, exp[index], act) } } func getTestQuantities() [3]resource.Quantity { var quantities [3]resource.Quantity quantities[0], _ = resource.ParseQuantity("12e6") quantities[1], _ = resource.ParseQuantity("12Mi") quantities[2], _ = resource.ParseQuantity("12M") return quantities } ================================================ FILE: pkg/plugin/costing.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( "fmt" "github.com/vmware/purser/pkg/plugin/metrics" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) // Cost details type Cost struct { TotalCost float64 CPUCost float64 MemoryCost float64 StorageCost float64 } // ClientSetInstance helps in accessing kubernetes apis through client. var ClientSetInstance *kubernetes.Clientset // ProvideClientSetInstance sets the client set instance. func ProvideClientSetInstance(clientset *kubernetes.Clientset) { ClientSetInstance = clientset } // GetPodsCostForLabel returns pods cost for given label. func GetPodsCostForLabel(label string) { pods := getPodsForLabelThroughClient(label) pods = getPodsCost(pods) printPodsVerbose(pods) } // GetClusterSummary summarizes cluster metrics. func GetClusterSummary() { pods := GetClusterPods() podMetrics := metrics.CalculatePodStatsFromContainers(pods) fmt.Println("==============================") fmt.Printf("Cluster Summary\n") fmt.Println("==============================") fmt.Println() fmt.Println("\tCompute:") nodes := GetClusterNodes() fmt.Printf("\t\t%s\t\t\t%d\n", "Node count:", len(nodes)) nodeMetrics := metrics.CalculateNodeStats(nodes) fmt.Printf("\t\tTotal Capacity:\n") fmt.Printf("\t\t\t%s\t\t%d\n", "CPU(vCPU):", nodeMetrics.CPULimit.Value()) fmt.Printf("\t\t\t%s\t\t%.2f\n", "Memory(GB):", bytesToGB(nodeMetrics.MemoryLimit.Value())) fmt.Printf("\t\tProvisioned Resources:\n") fmt.Printf("\t\t\t%s\t%d\n", "CPU Request(vCPU):", podMetrics.CPURequest.Value()) fmt.Printf("\t\t\t%s\t%.2f\n", "Memory Request(GB):", bytesToGB(podMetrics.MemoryRequest.Value())) price := GetUserCosts() hoursInMonthTillNow := totalHoursTillNow() cpuCost := float64(nodeMetrics.CPULimit.Value()) * hoursInMonthTillNow * price.CPU memCost := bytesToGB(nodeMetrics.MemoryLimit.Value()) * hoursInMonthTillNow * price.Memory computeCost := cpuCost + memCost fmt.Println() fmt.Printf("\tStorage:\n") pvs := GetClusterVolumes() storageCost, storageCapacity := getPvCostAndCapacity(pvs) fmt.Printf("\t\t%s\t%d\n", "Persistent Volume count:", len(pvs)) fmt.Printf("\t\t%s\t\t\t%.2f\n", "Capacity(GB):", bytesToGB(storageCapacity)) pvcs := GetClusterPersistentVolumeClaims() _, pvcCapacity := getPvcCostAndCapacity(pvcs) fmt.Printf("\t\t%s\t\t\t%d\n", "PV Claim count:", len(pvcs)) fmt.Printf("\t\t%s\t\t%.2f\n", "PV Claim Capacity(GB):", bytesToGB(pvcCapacity)) fmt.Println() fmt.Printf("\tMonth To Date Cost:\n") fmt.Printf("\t\t%s\t\t%.2f\n", "Compute cost($):", computeCost) fmt.Printf("\t\t%s\t\t%.2f\n", "Storage cost($):", storageCost) fmt.Printf("\t\t%s\t\t\t%.2f\n", "Total cost($):", computeCost+storageCost) } // GetSavings returns the savings summary. func GetSavings() { fmt.Printf("Savings Summary\n") fmt.Printf("Storage:\n") pvs := GetClusterVolumes() storageCost, storageCapacity := getPvCostAndCapacity(pvs) pvcs := GetClusterPersistentVolumeClaims() pvcCost, pvcCapacity := getPvcCostAndCapacity(pvcs) mtdSaving := storageCost - pvcCost projectedSaving := projectToMonth(mtdSaving) fmt.Printf(" %-30s %d\n", "Unused Volumes:", len(pvs)-len(pvcs)) fmt.Printf(" %-30s %.2f\n", "Unused Capacity(GB):", bytesToGB(storageCapacity-pvcCapacity)) fmt.Printf(" %-30s %.2f\n", "Month To Date Savings($):", mtdSaving) fmt.Printf(" %-30s %.2f\n", "Projected Monthly Savings($):", projectedSaving) } // GetPodCost returns the cumulative cost for the pods. func GetPodCost(podName string) { pod := getPodDetailsFromClient(podName) pods := getPodsCost([]*Pod{pod}) printPodsVerbose(pods) } // GetAllNodesCost returns the cumulative cost of all the nodes. func GetAllNodesCost() { nodes := GetClusterNodes() price := GetUserCosts() hoursInMonthTillNow := totalHoursTillNow() fmt.Println("Node name\tNode cpu-cost\tNode mem-cost\tNode total-cost") for i := 0; i < len(nodes); i++ { node := nodes[i] nodeMetrics := metrics.CalculateNodeStats([]v1.Node{node}) cpuCost := float64(nodeMetrics.CPULimit.Value()) * hoursInMonthTillNow * price.CPU memoryCost := bytesToGB(nodeMetrics.MemoryLimit.Value()) * hoursInMonthTillNow * price.Memory totalComputeCost := cpuCost + memoryCost fmt.Printf("%s\t%f\t%f\t%f\n", node.Name, cpuCost, memoryCost, totalComputeCost) } } func calculateCost(pods []*Pod, pvcs map[string]*PersistentVolumeClaim) []*Pod { price := GetUserCosts() for i := 0; i <= len(pods)-1; i++ { pod := pods[i] pods[i].cost = calculateCostOfPod(*pod, pvcs, price) } return pods } func calculateCostOfPod(pod Pod, pvcs map[string]*PersistentVolumeClaim, price *Price) *Cost { podDurationInHours := currentMonthActiveTimeInHours(pod.startTime, metav1.Now()) podCPUCost := float64(pod.podMetrics.CPURequest.Value()) * podDurationInHours * (price.CPU) podMemoryCost := bytesToGB(pod.podMetrics.MemoryRequest.Value()) * podDurationInHours * (price.Memory) podStorageCost := 0.0 for _, pvc := range pod.pvcs { if pvcs[*pvc] != nil { podStorageCost += pvcs[*pvc].capacityAllotedInGB * podDurationInHours * (price.Storage) } else { fmt.Printf("Persistent volume claim is not present for %s\n", *pvc) } } podTotalCost := podCPUCost + podMemoryCost + podStorageCost return &Cost{ TotalCost: podTotalCost, CPUCost: podCPUCost, MemoryCost: podMemoryCost, StorageCost: podStorageCost, } } func getPvCostAndCapacity(pvs []v1.PersistentVolume) (float64, int64) { price := GetUserCosts() hoursInMonthTillNow := totalHoursTillNow() var storageCapacity = resource.Quantity{} for _, pv := range pvs { storageCapacity.Add(pv.Spec.Capacity["storage"]) } storageCost := bytesToGB(storageCapacity.Value()) * hoursInMonthTillNow * price.Storage return storageCost, storageCapacity.Value() } func getPvcCostAndCapacity(pvcs []v1.PersistentVolumeClaim) (float64, int64) { price := GetUserCosts() hoursInMonthTillNow := totalHoursTillNow() var pvcCapacity = resource.Quantity{} for _, pvc := range pvcs { pvcCapacity.Add(pvc.Spec.Resources.Requests["storage"]) } pvcCost := bytesToGB(pvcCapacity.Value()) * hoursInMonthTillNow * price.Storage return pvcCost, pvcCapacity.Value() } func getPodsCost(pods []*Pod) []*Pod { pvcs := map[string]*PersistentVolumeClaim{} for _, pod := range pods { for _, pvc := range pod.pvcs { pvcs[*pvc] = nil } } pvcs = collectPersistentVolumeClaims(pvcs) pods = calculateCost(pods, pvcs) return pods } ================================================ FILE: pkg/plugin/grouping.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( "fmt" "time" log "github.com/Sirupsen/logrus" groups_v1 "github.com/vmware/purser/pkg/apis/groups/v1" groups "github.com/vmware/purser/pkg/client/clientset/typed/groups/v1" ) // GetGroupByName return group CRD by name. func GetGroupByName(groupClient *groups.GroupClient, groupName string) *groups_v1.Group { group, err := groupClient.Get(groupName) if err != nil { log.Errorf("failed to get custom group by name %s, %v", groupName, err) return nil } return group } // PrintGroup displays the group information. func PrintGroup(group *groups_v1.Group) { pitGroupMetrics := group.Spec.PITMetrics mtdGroupMetrics := group.Spec.MTDMetrics cost := group.Spec.MTDCost fmt.Printf("%-30s %s\n", "Group Name:", group.Name) fmt.Println() if pitGroupMetrics != nil { fmt.Println("Point in Time Resource Stats:") fmt.Printf(" %-30s%.2f\n", "CPU Limit(vCPU):", pitGroupMetrics.CPULimit) fmt.Printf(" %-30s%.2f\n", "Memory Limit(GB):", pitGroupMetrics.MemoryLimit) fmt.Printf(" %-30s%.2f\n", "CPU Request(vCPU):", pitGroupMetrics.CPURequest) fmt.Printf(" %-30s%.2f\n", "Memory Request(GB):", pitGroupMetrics.MemoryRequest) fmt.Printf(" %-30s%.2f\n", "Storage Claimed(GB):", pitGroupMetrics.StorageClaim) } if mtdGroupMetrics != nil { fmt.Println() fmt.Printf("%-30s\n", "Month to Date Active Resource Stats:") fmt.Printf(" %-30s%.2f\n", "CPU Request(vCPU-hours):", mtdGroupMetrics.CPURequest) fmt.Printf(" %-30s%.2f\n", "Memory Request(GB-hours):", mtdGroupMetrics.MemoryRequest) fmt.Printf(" %-30s%.2f\n", "Storage Claimed(GB-hours):", mtdGroupMetrics.StorageClaim) } if cost != nil { fmt.Println() fmt.Printf("%-30s\n", "Month to Date Cost Stats:") fmt.Printf(" %-30s%.2f\n", "CPU Cost($):", cost.CPUCost) fmt.Printf(" %-30s%.2f\n", "Memory Cost($):", cost.MemoryCost) fmt.Printf(" %-30s%.2f\n", "Storage Cost($):", cost.StorageCost) fmt.Printf(" %-30s%.2f\n", "Total Cost($):", cost.TotalCost) } fmt.Println() fmt.Printf("Last updated %f minutes ago", time.Since(group.Spec.LastUpdated).Minutes()) fmt.Println() } ================================================ FILE: pkg/plugin/metrics/metrics.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package metrics import ( "fmt" api_v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) // Metrics information type Metrics struct { CPULimit *resource.Quantity MemoryLimit *resource.Quantity CPURequest *resource.Quantity MemoryRequest *resource.Quantity } // GroupMetrics Details // Here Active resource is the resource quantity active in the current month type GroupMetrics struct { CPULimit float64 MemoryLimit float64 CPURequest float64 MemoryRequest float64 StorageClaimed float64 } // CalculatePodStatsFromContainers returns pods stats from containers. func CalculatePodStatsFromContainers(pods []v1.Pod) *Metrics { cpuLimit := &resource.Quantity{} memoryLimit := &resource.Quantity{} cpuRequest := &resource.Quantity{} memoryRequest := &resource.Quantity{} for _, pod := range pods { for _, c := range pod.Spec.Containers { limits := c.Resources.Limits if limits != nil { cpuLimit.Add(*limits.Cpu()) memoryLimit.Add(*limits.Memory()) } requests := c.Resources.Requests if requests != nil { cpuRequest.Add(*requests.Cpu()) memoryRequest.Add(*requests.Memory()) } } } return &Metrics{ CPULimit: cpuLimit, MemoryLimit: memoryLimit, CPURequest: cpuRequest, MemoryRequest: memoryRequest, } } // CalculateNodeStats returns node metrics. func CalculateNodeStats(nodes []v1.Node) *Metrics { cpuLimit := &resource.Quantity{} memoryLimit := &resource.Quantity{} cpuRequest := &resource.Quantity{} memoryRequest := &resource.Quantity{} for _, node := range nodes { cpuLimit.Add(*node.Status.Capacity.Cpu()) memoryLimit.Add(*node.Status.Capacity.Memory()) } return &Metrics{ CPULimit: cpuLimit, MemoryLimit: memoryLimit, CPURequest: cpuRequest, MemoryRequest: memoryRequest, } } // PrintPodStats displays stats for Pod func PrintPodStats(pod *api_v1.Pod, metrics *Metrics) { fmt.Printf("Pod:\t%s\n", pod.Name) fmt.Printf("\tCPU Limit = %s\n", metrics.CPULimit.String()) fmt.Printf("\tMemory Limit = %s\n", metrics.MemoryLimit.String()) fmt.Printf("\tCPU Request = %s\n", metrics.CPURequest.String()) fmt.Printf("\tMemory Request = %s\n", metrics.MemoryRequest.String()) } ================================================ FILE: pkg/plugin/node.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // GetClusterNodes returns the list of nodes in the cluster. func GetClusterNodes() []v1.Node { nodes, err := ClientSetInstance.CoreV1().Nodes().List(metav1.ListOptions{}) if err != nil { panic(err.Error()) } return nodes.Items } ================================================ FILE: pkg/plugin/pod.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( "fmt" "strings" "github.com/vmware/purser/pkg/plugin/metrics" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) // Pod Information type Pod struct { name string nodeName string cost *Cost pvcs []*string podMetrics *metrics.Metrics startTime metav1.Time } // GetClusterPods returns the list of pods in cluster. func GetClusterPods() []v1.Pod { pods, err := ClientSetInstance.CoreV1().Pods("").List(metav1.ListOptions{}) if err != nil { panic(err.Error()) } return pods.Items } func getPodDetailsFromClient(podName string) *Pod { pod, err := ClientSetInstance.CoreV1().Pods("default").Get(podName, metav1.GetOptions{}) if errors.IsNotFound(err) { fmt.Printf("Node %s not found\n", podName) return nil } else if statusError, isStatus := err.(*errors.StatusError); isStatus { fmt.Printf("Error getting Node %s : %v\n", podName, statusError.ErrStatus.Message) return nil } else if err != nil { panic(err.Error()) } else { return &Pod{ name: pod.GetObjectMeta().GetName(), nodeName: pod.Spec.NodeName, pvcs: getPodVolumes(pod), podMetrics: metrics.CalculatePodStatsFromContainers([]v1.Pod{*pod}), startTime: *pod.Status.StartTime, } } } func getPodsForLabelThroughClient(label string) []*Pod { vals := strings.Split(label, "=") if len(vals) != 2 { panic("Label should be of form key=val") } m := map[string]string{vals[0]: vals[1]} pods, err := ClientSetInstance.CoreV1().Pods("").List(metav1.ListOptions{LabelSelector: labels.SelectorFromSet(m).String()}) if err != nil { panic(err.Error()) } return createPodObjects(pods) } func createPodObjects(pods *v1.PodList) []*Pod { ps := []*Pod{} for i := 0; i < len(pods.Items); i++ { p := createPodObject(&pods.Items[i]) ps = append(ps, &p) } return ps } func createPodObject(pod *v1.Pod) Pod { return Pod{ name: pod.GetObjectMeta().GetName(), nodeName: pod.Spec.NodeName, pvcs: getPodVolumes(pod), podMetrics: metrics.CalculatePodStatsFromContainers([]v1.Pod{*pod}), startTime: *pod.Status.StartTime, } } func getPodVolumes(pod *v1.Pod) []*string { podVolumes := []*string{} for j := 0; j < len(pod.Spec.Volumes); j++ { vol := pod.Spec.Volumes[j] if vol.PersistentVolumeClaim != nil { podVolumes = append(podVolumes, &vol.PersistentVolumeClaim.ClaimName) } } return podVolumes } func printPodsVerbose(pods []*Pod) { fmt.Printf("Cost Summary\n") totalCost := 0.0 totalCPUCost := 0.0 totalMemoryCost := 0.0 totalStorageCost := 0.0 for i := 0; i <= len(pods)-1; i++ { fmt.Printf("%-30s %s\n", "Pod Name:", pods[i].name) fmt.Printf("%-30s %s\n", "Node:", pods[i].nodeName) fmt.Printf("%-30s\n", "Persistent Volume Claims:") for j := 0; j <= len(pods[i].pvcs)-1; j++ { fmt.Printf(" %s\n", *pods[i].pvcs[j]) } fmt.Printf("%-30s\n", "Cost:") fmt.Printf(" %-30s%f$\n", "Total Cost:", pods[i].cost.TotalCost) fmt.Printf(" %-30s%f$\n", "Compute Cost:", pods[i].cost.CPUCost+pods[i].cost.MemoryCost) fmt.Printf(" %-30s%f$\n", "Storage Cost:", pods[i].cost.StorageCost) fmt.Printf("\n") totalCost += pods[i].cost.TotalCost totalCPUCost += pods[i].cost.CPUCost totalMemoryCost += pods[i].cost.MemoryCost totalStorageCost += pods[i].cost.StorageCost } fmt.Printf("%-30s\n", "Total Cost Summary:") fmt.Printf(" %-30s%f$\n", "Total Cost:", totalCost) fmt.Printf(" %-30s%f$\n", "Compute Cost:", totalCPUCost+totalMemoryCost) fmt.Printf(" %-30s%f$\n", "Storage Cost:", totalStorageCost) } ================================================ FILE: pkg/plugin/pricing.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( "fmt" "strconv" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( namespace = "default" userCostsConfigMap = "purser-user-costs" defaultCPUCostPerCPUPerHour = 0.024 defaultMemCostPerGBPerHour = 0.01 defaultStorageCostPerGBPerHour = 0.00013888888 ) // Price information. // NOTE: All fields are Per unit resource per hour type Price struct { CPU float64 Memory float64 Storage float64 } // SaveUserCosts stores the cpu, memory and storage cost per unit per hour in the cluster as config maps. func SaveUserCosts(cpuCostPerCPUPerHour, memCostPerGBPerHour, storageCostPerGBPerHour string) bool { cm, err := ClientSetInstance.CoreV1().ConfigMaps(namespace).Get(userCostsConfigMap, metav1.GetOptions{}) if err != nil { // no configmap so create new one costMap := map[string]string{} costMap["cpuCostPerCPUPerHour"] = cpuCostPerCPUPerHour costMap["memCostPerGBPerHour"] = memCostPerGBPerHour costMap["storageCostPerGBPerHour"] = storageCostPerGBPerHour cm = &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: userCostsConfigMap, }, Data: costMap, } _, err2 := ClientSetInstance.CoreV1().ConfigMaps(namespace).Create(cm) if err2 != nil { fmt.Printf("Error in createing config map: %s", err2) return false } } else { // update configmap cm.Data["cpuCostPerCPUPerHour"] = cpuCostPerCPUPerHour cm.Data["memCostPerGBPerHour"] = memCostPerGBPerHour cm.Data["storageCostPerGBPerHour"] = storageCostPerGBPerHour _, err2 := ClientSetInstance.CoreV1().ConfigMaps(namespace).Update(cm) if err2 != nil { fmt.Printf("Error in updating config map: %s", err2) return false } fmt.Printf("Updated config map\n") } return true } // GetUserCosts gives the cpu, memory and storage cost per unit per hour which are stored in the cluster as config maps. func GetUserCosts() *Price { var cpuCostPerCPUPerHour, memCostPerGBPerHour, storageCostPerGBPerHour float64 cm, err := ClientSetInstance.CoreV1().ConfigMaps(namespace).Get(userCostsConfigMap, metav1.GetOptions{}) if err != nil { // no user configed costs. so return default values cpuCostPerCPUPerHour = defaultCPUCostPerCPUPerHour memCostPerGBPerHour = defaultMemCostPerGBPerHour storageCostPerGBPerHour = defaultStorageCostPerGBPerHour } else { cpuCostPerCPUPerHour, err = strconv.ParseFloat(cm.Data["cpuCostPerCPUPerHour"], 64) if err != nil { fmt.Printf("Error converting cpuCostPerCPUPerHour string %s to float, rolling back to default value\n", cm.Data["cpuCostPerCPUPerHour"]) cpuCostPerCPUPerHour = defaultCPUCostPerCPUPerHour } memCostPerGBPerHour, err = strconv.ParseFloat(cm.Data["memCostPerGBPerHour"], 64) if err != nil { fmt.Printf("Error converting memCostPerGBPerHour string %s to float, rolling back to default value\n", cm.Data["memCostPerGBPerHour"]) memCostPerGBPerHour = defaultMemCostPerGBPerHour } storageCostPerGBPerHour, err = strconv.ParseFloat(cm.Data["storageCostPerGBPerHour"], 64) if err != nil { fmt.Printf("Error converting storageCostPerGBPerHour string %s to float, rolling back to default value\n", cm.Data["storageCostPerGBPerHour"]) storageCostPerGBPerHour = defaultStorageCostPerGBPerHour } } return &Price{ CPU: cpuCostPerCPUPerHour, Memory: memCostPerGBPerHour, Storage: storageCostPerGBPerHour, } } ================================================ FILE: pkg/plugin/utils.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // getCurrentTime returns the current time as k8s apimachinery Time object func getCurrentTime() metav1.Time { return metav1.Now() } // getCurrentMonthStartTime returns month start time as k8s apimachinery Time object func getCurrentMonthStartTime() metav1.Time { now := time.Now() monthStart := metav1.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.Local) return monthStart } /* currentMonthActiveTimeInHours returns active time (endTime - startTime) in the current month. 1. If startTime is before month start then it is set as month start 2. If endTime is not set(isZero) then it is set as current time These two conditions ensures that the active time we compute is within the current month. */ func currentMonthActiveTimeInHours(startTime, endTime metav1.Time) float64 { currentTime := getCurrentTime() monthStart := getCurrentMonthStartTime() return currentMonthActiveTimeInHoursMulti(startTime, endTime, currentTime, monthStart) } /* currentMonthActiveTimeInHoursMulti is same as currentMonthActiveTimeInHours but it needs extra inputs: currentTime and monthStart. Use this method(currentMonthActiveTimeInHoursMulti) if you want to caclculate active time multiple times (ex: inside a loop). */ func currentMonthActiveTimeInHoursMulti(startTime, endTime, currentTime, monthStart metav1.Time) float64 { if startTime.Time.Before(monthStart.Time) { startTime = monthStart } if endTime.IsZero() { endTime = currentTime } duration := endTime.Time.Sub(startTime.Time) durationInHours := duration.Hours() return durationInHours } // totalHoursTillNow return number of hours from month start to current time. func totalHoursTillNow() float64 { monthStart := getCurrentMonthStartTime() currentTime := getCurrentTime() return currentMonthActiveTimeInHours(monthStart, currentTime) } func projectToMonth(val float64) float64 { // TODO: enhance this. return (val * 31 * 24) / totalHoursTillNow() } func bytesToGB(val int64) float64 { return float64(val) / (1024.0 * 1024.0 * 1024.0) } ================================================ FILE: pkg/plugin/volume.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package plugin import ( "fmt" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // PersistentVolumeClaim details type PersistentVolumeClaim struct { name string volumeName string requestSizeInGB float64 capacityAllotedInGB float64 storageClass *string } // GetClusterVolumes returns list of persistent volumes for the cluster. func GetClusterVolumes() []v1.PersistentVolume { pvs, err := ClientSetInstance.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) if err != nil { panic(err.Error()) } return pvs.Items } // GetClusterPersistentVolumeClaims returns the list of persistent volume claims for the cluster. func GetClusterPersistentVolumeClaims() []v1.PersistentVolumeClaim { pvcs, err := ClientSetInstance.CoreV1().PersistentVolumeClaims("").List(metav1.ListOptions{}) if err != nil { panic(err.Error()) } return pvcs.Items } func collectPersistentVolumeClaims(pvcs map[string]*PersistentVolumeClaim) map[string]*PersistentVolumeClaim { for key := range pvcs { pvc := collectPersistentVolumeClaim(key) pvcs[key] = pvc } return pvcs } func collectPersistentVolumeClaim(claimName string) *PersistentVolumeClaim { pvc, err := ClientSetInstance.CoreV1().PersistentVolumeClaims("default").Get(claimName, metav1.GetOptions{}) if errors.IsNotFound(err) { fmt.Printf("Persistent Volume Claim %s not found\n", claimName) return nil } else if statusError, isStatus := err.(*errors.StatusError); isStatus { fmt.Printf("Error getting persistence volume Claim %s : %v\n", claimName, statusError.ErrStatus.Message) return nil } else if err != nil { panic(err.Error()) } else { request := pvc.Spec.Resources.Requests["storage"].DeepCopy() capacity := pvc.Status.Capacity["storage"].DeepCopy() return &PersistentVolumeClaim{ name: pvc.GetObjectMeta().GetName(), volumeName: pvc.Spec.VolumeName, storageClass: pvc.Spec.StorageClassName, requestSizeInGB: bytesToGB(request.Value()), capacityAllotedInGB: bytesToGB(capacity.Value()), } } } ================================================ FILE: pkg/pricing/aws/aws.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package aws import ( "net/http" "time" "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/utils" ) const ( httpTimeout = 100 * time.Second ) // Pricing structure type Pricing struct { Products map[string]Product Terms PlanList } // PlanList structure type PlanList struct { OnDemand map[string]map[string]TermAttributes } // TermAttributes structure type TermAttributes struct { PriceDimensions map[string]PricingData } // PricingData structure type PricingData struct { Unit string PricePerUnit map[string]string } // Product structure type Product struct { Sku string ProductFamily string Attributes ProductAttributes } // ProductAttributes structure type ProductAttributes struct { InstanceType string InstanceFamily string OperatingSystem string PreInstalledSW string VolumeType string UsageType string Vcpu string Memory string } // GetAWSPricing function details // input: region // retrieves data from http get to the corresponding url for that region func GetAWSPricing(region string) (*Pricing, error) { var myClient = &http.Client{Timeout: httpTimeout} rateCard := Pricing{} err := utils.GetJSONResponse(myClient, getURLForRegion(region), &rateCard) if err != nil { logrus.Errorf("Unable to get aws pricing. Reason: %v", err) return nil, err } return &rateCard, nil } func getURLForRegion(region string) string { return "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/" + region + "/index.json" } ================================================ FILE: pkg/pricing/aws/convert.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package aws import ( "strconv" "strings" "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/dgraph/models" ) // AWS specific constants const ( na = "NA" gbMonth = "GB-Mo" deliminator = "-" storageInstance = "Storage" computeInstance = "Compute Instance" // TODO: Determine priceSplitRatio according to instance type i.e, compute optimized or memory optimized etc priceSplitRatio = 0.5 ) // GetRateCardForAWS takes region as input and returns RateCard and error if any func GetRateCardForAWS(region string) *models.RateCard { awsPricing, err := GetAWSPricing(region) if err == nil { return convertAWSPricingToPurserRateCard(region, awsPricing) } return nil } func convertAWSPricingToPurserRateCard(region string, awsPricing *Pricing) *models.RateCard { nodePrices, storagePrices := getResourcePricesFromAWSPricing(awsPricing) return &models.RateCard{ ID: dgraph.ID{Xid: models.RateCardXID}, IsRateCard: true, CloudProvider: models.AWS, Region: region, NodePrices: nodePrices, StoragePrices: storagePrices, } } func getResourcePricesFromAWSPricing(awsPricing *Pricing) ([]*models.NodePrice, []*models.StoragePrice) { var nodePrices []*models.NodePrice var storagePrices []*models.StoragePrice products := awsPricing.Products planList := awsPricing.Terms duplicateComputeInstanceChecker := make(map[string]bool) for _, product := range products { priceInFloat64, unit := getResourcePrice(product, planList) switch product.ProductFamily { case computeInstance: nodePrices = updateComputeInstancePrices(product, priceInFloat64, duplicateComputeInstanceChecker, nodePrices) case storageInstance: storagePrices = updateStorageInstancePrices(product, priceInFloat64, unit, storagePrices) } } return nodePrices, storagePrices } func getResourcePrice(product Product, planList PlanList) (float64, string) { for _, pricingAttributes := range planList.OnDemand[product.Sku] { for _, pricingData := range pricingAttributes.PriceDimensions { for _, pricePerUnit := range pricingData.PricePerUnit { priceInFloat64, err := strconv.ParseFloat(pricePerUnit, 64) if err != nil { logrus.Errorf("unable to parse string: %s to float. err: %v", pricePerUnit, err) return models.PriceError, "" // negative price means error } return priceInFloat64, pricingData.Unit } } } return models.PriceError, "" } func updateComputeInstancePrices(product Product, priceInFloat64 float64, duplicateComputeInstanceChecker map[string]bool, nodePrices []*models.NodePrice) []*models.NodePrice { key := product.Sku + product.Attributes.InstanceType + product.Attributes.OperatingSystem if _, isPresent := duplicateComputeInstanceChecker[key]; !isPresent && product.Attributes.PreInstalledSW == na { // Unit of Compute price USD-perHour productXID := product.Attributes.InstanceType + deliminator + product.Attributes.OperatingSystem pricePerCPU, pricePerGB := getPriceForUnitResource(product, priceInFloat64) nodePrice := &models.NodePrice{ ID: dgraph.ID{Xid: productXID}, IsNodePrice: true, InstanceType: product.Attributes.InstanceType, InstanceFamily: product.Attributes.InstanceFamily, OperatingSystem: product.Attributes.OperatingSystem, Price: priceInFloat64, PricePerCPU: pricePerCPU, PricePerMemory: pricePerGB, } duplicateComputeInstanceChecker[key] = true uid := models.StoreNodePrice(nodePrice, productXID) if uid != "" { nodePrice.ID = dgraph.ID{UID: uid, Xid: productXID} nodePrices = append(nodePrices, nodePrice) } } return nodePrices } func updateStorageInstancePrices(product Product, priceInFloat64 float64, unit string, storagePrices []*models.StoragePrice) []*models.StoragePrice { if priceInFloat64 == models.PriceError { priceInFloat64 = models.DefaultStorageCostInFloat64 } else if unit == gbMonth { // convert to GBHour priceInFloat64 = priceInFloat64 / models.HoursInMonth } productXID := product.Attributes.VolumeType + deliminator + product.Attributes.UsageType storagePrice := &models.StoragePrice{ ID: dgraph.ID{Xid: productXID}, IsStoragePrice: true, VolumeType: product.Attributes.VolumeType, UsageType: product.Attributes.UsageType, Price: priceInFloat64, } uid := models.StoreStoragePrice(storagePrice, productXID) if uid != "" { storagePrice.ID = dgraph.ID{UID: uid, Xid: productXID} storagePrices = append(storagePrices, storagePrice) } return storagePrices } func getPriceForUnitResource(product Product, priceInFloat64 float64) (float64, float64) { pricePerCPU := models.DefaultCPUCostInFloat64 pricePerGB := models.DefaultMemCostInFloat64 // priceInFloat64 should be greater than 0 otherwise this function returns default pricing if priceInFloat64 != models.PriceError && priceInFloat64 != 0 { cpu, err := strconv.ParseFloat(product.Attributes.Vcpu, 64) if err == nil { pricePerCPU = priceSplitRatio * priceInFloat64 / cpu } memWithUnits := product.Attributes.Memory // memWithUnits format: "3,126 GiB" mem, err := strconv.ParseFloat(strings.Join(strings.Split(strings.Split(memWithUnits, " GiB")[0], ","), ""), 64) if err == nil { pricePerGB = (1 - priceSplitRatio) * priceInFloat64 / mem } } return pricePerCPU, pricePerGB } ================================================ FILE: pkg/pricing/cloud.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package pricing import ( "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/vmware/purser/pkg/pricing/aws" "k8s.io/client-go/kubernetes" ) // Cloud structure used for pricing type Cloud struct { CloudProvider string Region string Kubeclient *kubernetes.Clientset } // GetClusterProviderAndRegion returns cluster provider(ex: aws) and region(ex: us-east-1) func GetClusterProviderAndRegion() (string, string) { // TODO: https://github.com/vmware/purser/issues/143 cloudProvider := models.AWS region := "us-east-1" logrus.Infof("CloudProvider: %s, Region: %s", cloudProvider, region) return cloudProvider, region } // PopulateRateCard given a cloud (cloudProvider and region) it populates corresponding rate card in dgraph func (c *Cloud) PopulateRateCard() { switch c.CloudProvider { case models.AWS: rateCard := aws.GetRateCardForAWS(c.Region) models.StoreRateCard(rateCard) } } ================================================ FILE: pkg/utils/fileutils.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "os" "os/user" log "github.com/Sirupsen/logrus" ) // OpenFile handles opening file in Read/Write mode, creating and appending to it as needed. func OpenFile(filename string) *os.File { f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600) if err != nil { log.Errorf("failed to open file %s, %v", filename, err) } return f } // GetUsrHomeDir returns the current user's Home Directory func GetUsrHomeDir() string { usr, err := user.Current() if err != nil { log.Errorf("failed to fetch current user %v", err) } return usr.HomeDir } ================================================ FILE: pkg/utils/k8sutil.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "github.com/Sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) // GetKubeclient returns a k8s clientset from the kubeconfig, if nil fallback to // client from inCluster config. func GetKubeclient(config *rest.Config) *kubernetes.Clientset { clientset, err := kubernetes.NewForConfig(config) if err != nil { logrus.Fatalf("failed to create kubernetes clientset: %v", err) } return clientset } // GetKubeconfig builds config from the kubeconfig path, if nil fallback to // inCluster config. func GetKubeconfig(kubeconfigPath string) (*rest.Config, error) { return clientcmd.BuildConfigFromFlags("", kubeconfigPath) } ================================================ FILE: pkg/utils/logutil.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "io" "os" log "github.com/Sirupsen/logrus" ) const logFile = "purser.log" // InitializeLogger sets and configures logger options. func InitializeLogger(logLevel string) { logFile := OpenFile(logFile) log.SetOutput(io.MultiWriter(os.Stdout, logFile)) log.SetFormatter(&log.TextFormatter{ForceColors: true}) if logLevel == "debug" { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } } ================================================ FILE: plugin.yaml ================================================ name: "purser" shortDesc: "Cost Insight integration with kubernetes" longDesc: > Purser gives cost insights of kubernetes deployments. command: purser_plugin $@ flags: - name: info desc: Show more details about the plugin. - name: version desc: Show plugin version ================================================ FILE: test/controller/buffering/ring_buffer_test.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package buffering_test import ( "sync" "testing" "github.com/vmware/purser/pkg/controller/buffering" "github.com/vmware/purser/test/utils" ) func TestPut(t *testing.T) { // use Put to add one more, return from Put should be True r := &buffering.RingBuffer{Size: 2, Mutex: &sync.Mutex{}} testValue1 := 1 ret1 := r.Put(testValue1) utils.Assert(t, ret1, "inserting into not full buffer") testValue2 := 38 ret2 := r.Put(testValue2) utils.Assert(t, !ret2, "inserting into full buffer") } func TestGet(t *testing.T) { // use Put to add one more, return from Put should be True r := &buffering.RingBuffer{Size: 2, Mutex: &sync.Mutex{}} ret1 := r.Get() utils.Assert(t, ret1 == nil, "get elements of empty buffer") testValue := 1 r.Put(testValue) ret2 := r.Get() utils.Assert(t, (*ret2).(int) == testValue, "get elements from non empty buffer") } ================================================ FILE: test/pricing/pricing_aws_test.go ================================================ package pricing import ( "testing" "github.com/vmware/purser/test/utils" "github.com/Sirupsen/logrus" "github.com/vmware/purser/pkg/controller/dgraph" "github.com/vmware/purser/pkg/controller/dgraph/models" "github.com/vmware/purser/pkg/pricing/aws" ) // TestAWSPricingFlow it should populate your dgraph running at localhost 9080 port with aws compute and storage prices // The following dgraph query will give the rate card data // { // rateCard(func: has(isRateCard)) { // cloudProvider // region // nodePrices { // instanceType // operatingSystem // price // instanceFamily // } // storagePrices { // volumeType // usageType // price // } // } // } func TestAWSPricingFlow(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) dgraph.Start("localhost", "9080") rateCard := aws.GetRateCardForAWS("us-east-1") models.StoreRateCard(rateCard) defer dgraph.Close() utils.Assert(t, rateCard != nil, "rate card is nil") } ================================================ FILE: test/utils/checkUtil.go ================================================ /* * Copyright (c) 2018 VMware Inc. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package utils import ( "fmt" "path/filepath" "reflect" "runtime" "testing" ) // Assert fails the test if the condition is false. func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) { if !condition { _, file, line, _ := runtime.Caller(1) fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) tb.FailNow() } } // Ok fails the test if an err is not nil. func Ok(tb testing.TB, err error) { if err != nil { _, file, line, _ := runtime.Caller(1) fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) tb.FailNow() } } // Equals fails the test if exp is not equal to act. func Equals(tb testing.TB, exp, act interface{}) { if !reflect.DeepEqual(exp, act) { _, file, line, _ := runtime.Caller(1) fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) tb.FailNow() } } ================================================ FILE: ui/Dockerfile.deploy.purser ================================================ FROM node:9.6.1 as builder LABEL maintainer = "VMware " LABEL author = "Krishna Karthik " # set working directory RUN mkdir /usr/src/app WORKDIR /usr/src/app # add `/usr/src/app/node_modules/.bin` to $PATH ENV PATH /usr/src/app/node_modules/.bin:$PATH # install and cache app dependencies COPY package.json package-lock.json ./ RUN npm install RUN npm install -g @angular/cli@6.2.1 # add purser application to the working directory COPY . . # start purser application RUN npm run build # Build a small nginx image FROM nginx:latest COPY nginx.conf /etc/nginx/conf.d/default.conf COPY --from=builder /usr/src/app/dist /usr/share/nginx/html ================================================ FILE: ui/README.md ================================================ # Purser UI Purser UI is designed to provide a visual representation to a host of features provided by Purser such as **cluster hierarchy**, **Pod and Service interactions** and **capacity allocations** for CPU, memory, disk space and other resources. It has been generated with [Angular CLI](https://github.com/angular/angular-cli) version 6.2.1 and [Clarity Design System](https://clarity.design/). ## Installing Dependencies Use "npm" or "yarn" to install/manage dependencies. Run `npm install` inside this directory to install all the needed dependencies. ## Development server Run `npm start` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files. ## Code scaffolding Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. ## Build Run `npm build` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `--prod` flag for a production build. ## Further help To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI README](https://github.com/angular/angular-cli/blob/master/README.md). ================================================ FILE: ui/angular.json ================================================ { "$schema": "./node_modules/@angular/cli/lib/config/schema.json", "version": 1, "newProjectRoot": "projects", "projects": { "purser": { "root": "", "sourceRoot": "src", "projectType": "application", "prefix": "app", "schematics": {}, "architect": { "build": { "builder": "@angular-devkit/build-angular:browser", "options": { "outputPath": "dist/purser", "index": "src/index.html", "main": "src/main.ts", "polyfills": "src/polyfills.ts", "tsConfig": "src/tsconfig.app.json", "assets": [ "src/favicon.ico", "src/assets", "src/json" ], "styles": [ "node_modules/@clr/icons/clr-icons.min.css", "node_modules/@clr/ui/clr-ui.min.css", "src/styles.css", "node_modules/vis/dist/vis.css" ], "scripts": [ "node_modules/@webcomponents/custom-elements/custom-elements.min.js", "node_modules/@clr/icons/clr-icons.min.js" ] }, "configurations": { "production": { "fileReplacements": [ { "replace": "src/environments/environment.ts", "with": "src/environments/environment.prod.ts" } ], "optimization": true, "outputHashing": "all", "sourceMap": false, "extractCss": true, "namedChunks": false, "aot": true, "extractLicenses": true, "vendorChunk": false, "buildOptimizer": true } } }, "serve": { "builder": "@angular-devkit/build-angular:dev-server", "options": { "browserTarget": "purser:build" }, "configurations": { "production": { "browserTarget": "purser:build:production" } } }, "extract-i18n": { "builder": "@angular-devkit/build-angular:extract-i18n", "options": { "browserTarget": "purser:build" } }, "test": { "builder": "@angular-devkit/build-angular:karma", "options": { "main": "src/test.ts", "polyfills": "src/polyfills.ts", "tsConfig": "src/tsconfig.spec.json", "karmaConfig": "src/karma.conf.js", "styles": [ "src/styles.css" ], "scripts": [], "assets": [ "src/favicon.ico", "src/assets" ] } }, "lint": { "builder": "@angular-devkit/build-angular:tslint", "options": { "tsConfig": [ "src/tsconfig.app.json", "src/tsconfig.spec.json" ], "exclude": [ "**/node_modules/**" ] } } } }, "purser-e2e": { "root": "e2e/", "projectType": "application", "architect": { "e2e": { "builder": "@angular-devkit/build-angular:protractor", "options": { "protractorConfig": "e2e/protractor.conf.js", "devServerTarget": "purser:serve" }, "configurations": { "production": { "devServerTarget": "purser:serve:production" } } }, "lint": { "builder": "@angular-devkit/build-angular:tslint", "options": { "tsConfig": "e2e/tsconfig.e2e.json", "exclude": [ "**/node_modules/**" ] } } } } }, "defaultProject": "purser" } ================================================ FILE: ui/e2e/protractor.conf.js ================================================ // Protractor configuration file, see link for more information // https://github.com/angular/protractor/blob/master/lib/config.ts const { SpecReporter } = require('jasmine-spec-reporter'); exports.config = { allScriptsTimeout: 11000, specs: [ './src/**/*.e2e-spec.ts' ], capabilities: { 'browserName': 'chrome' }, directConnect: true, baseUrl: 'http://localhost:4200/', framework: 'jasmine', jasmineNodeOpts: { showColors: true, defaultTimeoutInterval: 30000, print: function() {} }, onPrepare() { require('ts-node').register({ project: require('path').join(__dirname, './tsconfig.e2e.json') }); jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); } }; ================================================ FILE: ui/e2e/src/app.e2e-spec.ts ================================================ import { AppPage } from './app.po'; describe('workspace-project App', () => { let page: AppPage; beforeEach(() => { page = new AppPage(); }); it('should display welcome message', () => { page.navigateTo(); expect(page.getParagraphText()).toEqual('Welcome to purser!'); }); }); ================================================ FILE: ui/e2e/src/app.po.ts ================================================ import { browser, by, element } from 'protractor'; export class AppPage { navigateTo() { return browser.get('/'); } getParagraphText() { return element(by.css('app-root h1')).getText(); } } ================================================ FILE: ui/e2e/tsconfig.e2e.json ================================================ { "extends": "../tsconfig.json", "compilerOptions": { "outDir": "../out-tsc/app", "module": "commonjs", "target": "es5", "types": [ "jasmine", "jasminewd2", "node" ] } } ================================================ FILE: ui/nginx.conf ================================================ upstream purser { server purser.purser.svc.cluster.local:3030; } server { listen 4200; location /auth { proxy_pass http://purser; } location /api { proxy_pass http://purser; } location / { root /usr/share/nginx/html/purser; index index.html index.htm; try_files $uri $uri/ /index.html =404; } } ================================================ FILE: ui/package.json ================================================ { "name": "app-dapp", "version": "0.0.0", "scripts": { "ng": "ng", "start": "ng serve", "startdev": "ng serve --proxy-config proxy.conf.json", "build": "ng build", "test": "ng test", "lint": "ng lint", "e2e": "ng e2e" }, "private": true, "dependencies": { "@angular-devkit/build-angular": "~0.13.9", "@angular/animations": "^6.1.0", "@angular/common": "^6.1.0", "@angular/compiler": "^6.1.0", "@angular/core": "^6.1.0", "@angular/forms": "^6.1.0", "@angular/http": "^6.1.0", "@angular/platform-browser": "^6.1.0", "@angular/platform-browser-dynamic": "^6.1.0", "@angular/router": "^6.1.0", "@clr/angular": "^0.13.1-patch.1", "@clr/icons": "^0.13.1-patch.1", "@clr/ui": "^0.13.1-patch.1", "@types/vis": "^4.21.8", "@webcomponents/custom-elements": "^1.0.0", "angular-google-charts": "^0.1.0", "core-js": "^2.5.4", "ngx-cookie-service": "^2.1.0", "rxjs": "~6.2.0", "vis": "^4.21.0", "zone.js": "~0.8.26" }, "devDependencies": { "@angular/cli": "~6.2.1", "@angular/compiler-cli": "^6.1.0", "@angular/language-service": "^6.1.0", "@types/jasmine": "~2.8.8", "@types/jasminewd2": "~2.0.3", "@types/node": "~8.9.4", "codelyzer": "~4.3.0", "jasmine-core": "~2.99.1", "jasmine-spec-reporter": "~4.2.1", "karma": "~3.0.0", "karma-chrome-launcher": "~2.2.0", "karma-coverage-istanbul-reporter": "~2.0.1", "karma-jasmine": "~1.1.2", "karma-jasmine-html-reporter": "^0.2.2", "protractor": "~5.4.0", "ts-node": "~7.0.0", "tslint": "~5.11.0", "typescript": "~2.9.2" } } ================================================ FILE: ui/proxy.conf.json ================================================ { "/api": { "target": "http://localhost:3030/", "changeOrigin": true, "secure":false } } ================================================ FILE: ui/src/app/app.component.html ================================================
{{messages && messages.common.appHeader}}
Custom Groups Capacity Hierarchy Interactions
GitHub GitHub New Issue
================================================ FILE: ui/src/app/app.component.scss ================================================ .main-container{ .appHeader{ font-size: 20px; align-items: center; } } .content-container { position: relative; height: 100%; display: flex; display: -webkit-flex; display: -moz-flex; display: -ms-flex; flex-direction: column; -webkit-box-direction: normal; -webkit-box-orient: vertical; .header { -webkit-box-flex: 0; box-flex: 0; flex: 0 0 60px; display: flex; } .webpageSpinner { position: absolute; top: 0; bottom: 0; right: 0; left: 0; z-index: 100; background: white; .spinner { position: absolute; margin: auto; top: 0; bottom: 0; right: 0; left: 0; } } .main-body { display: flex; display: -webkit-flex; display: -moz-flex; display: -ms-flex; overflow-x: hidden; -webkit-box-flex: 1; -ms-flex: 1 1 auto; flex: 1 1 auto; .navigation-area { /* -webkit-box-flex: 0; -ms-flex: 0 0 auto; flex: 0 0 auto; -webkit-box-ordinal-group: 0; order: -1; overflow: hidden; display: flex; -webkit-box-orient: vertical; -webkit-box-direction: normal; flex-direction: column;*/ background-color: #eee; } .content-area { background-color: #FAFAFA; display: flex; display: -webkit-flex; display: -moz-flex; display: -ms-flex; -webkit-box-flex: 1; -ms-flex: 1 1 auto; flex: 1 1 auto; -webkit-flex-direction: column; flex-direction: column; overflow-x: hidden; padding: 20px 24px 80px 24px; .bread-crumb { border-style: solid; border-width: 0px; border-color: grey; max-height: 40px; font-size: 12px; z-index: 10; .synctime-div { float: right; font-size: 12px; } } .page-area { flex: auto; position: relative; } .app-loader { height: 100%; display: flex; justify-content: center; align-items: center; flex-direction: column; } } } } ================================================ FILE: ui/src/app/app.component.spec.ts ================================================ import { TestBed, async } from '@angular/core/testing'; import { AppComponent } from './app.component'; describe('AppComponent', () => { beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ AppComponent ], }).compileComponents(); })); it('should create the app', async(() => { const fixture = TestBed.createComponent(AppComponent); const app = fixture.debugElement.componentInstance; expect(app).toBeTruthy(); })); it(`should have as title 'purser'`, async(() => { const fixture = TestBed.createComponent(AppComponent); const app = fixture.debugElement.componentInstance; expect(app.title).toEqual('purser'); })); it('should render title in a h1 tag', async(() => { const fixture = TestBed.createComponent(AppComponent); fixture.detectChanges(); const compiled = fixture.debugElement.nativeElement; expect(compiled.querySelector('h1').textContent).toContain('Welcome to purser!'); })); }); ================================================ FILE: ui/src/app/app.component.ts ================================================ import { Component, OnInit } from '@angular/core'; import { NavigationCancel, NavigationEnd, NavigationError, NavigationStart, Router, RouterEvent } from '@angular/router'; import { MCommon } from './common/messages/common.messages'; @Component({ selector: 'app-root', templateUrl: './app.component.html', styleUrls: ['./app.component.scss'] }) export class AppComponent implements OnInit { public routeLoading: boolean = false; public messages: any = {}; public IS_LOGEDIN = true; constructor(public router: Router) { this.messages = { 'common': MCommon } } private loadApp() { this.router.events.subscribe((event: RouterEvent) => { this.navigationEventHandler(event); }); } private navigationEventHandler(event: RouterEvent): void { if (event instanceof NavigationStart) { this.routeLoading = true; } if (event instanceof NavigationEnd) { this.routeLoading = false; } // Set loading state to false in both of the below events to hide the spinner in case a request fails. if (event instanceof NavigationCancel) { this.routeLoading = false; } if (event instanceof NavigationError) { this.routeLoading = false; } } ngOnInit() { this.loadApp(); } } ================================================ FILE: ui/src/app/app.constants.ts ================================================ const BACKEND_BASE_URL = "http://10.112.141.194"; export const BACKEND_URL = BACKEND_BASE_URL + '/api/' export const BACKEND_AUTH_URL = BACKEND_BASE_URL + '/auth/' ================================================ FILE: ui/src/app/app.module.ts ================================================ import { HttpClientModule } from '@angular/common/http'; import { CUSTOM_ELEMENTS_SCHEMA, NgModule } from '@angular/core'; import { BrowserModule } from '@angular/platform-browser'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { RouterModule } from '@angular/router'; import { ClarityModule } from '@clr/angular'; import { GoogleChartsModule } from 'angular-google-charts'; import { CookieService } from 'ngx-cookie-service'; import { AppComponent } from './app.component'; import { ROUTING } from "./app.routing"; import { CapacityGraphModule } from './modules/capacity-graph/capacity-graph.module'; import { ChangepasswordModule } from './modules/changepassword/changepassword.module'; import { LogicalGroupModule } from './modules/logical-group/logical-group.module'; import { LoginModule } from './modules/login/login.module'; import { LogoutModule } from './modules/logout/logout.module'; import { OptionsModule } from './modules/options/options.module'; import { TopoGraphModule } from './modules/topo-graph/modules'; import { TopologyGraphModule } from './modules/topologyGraph/modules'; @NgModule({ declarations: [ AppComponent, ], imports: [ BrowserModule, ClarityModule, BrowserAnimationsModule, RouterModule, HttpClientModule, ROUTING, CapacityGraphModule, TopologyGraphModule, TopoGraphModule, LoginModule, LogoutModule, LogicalGroupModule, ChangepasswordModule, OptionsModule, GoogleChartsModule.forRoot() ], providers: [CookieService], schemas: [CUSTOM_ELEMENTS_SCHEMA], bootstrap: [AppComponent] }) export class AppModule { } ================================================ FILE: ui/src/app/app.routing.ts ================================================ /*Framework imports, 3rd party imports */ import { ModuleWithProviders } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; import { LogicalGroupComponent } from './modules/logical-group/components/logical-group.component' import { TopologyGraphComponent } from './modules/topologyGraph/components/topologyGraph.component' import { TopoGraphComponent } from './modules/topo-graph/components/topo-graph.component' import { CapactiyGraphComponent } from './modules/capacity-graph/components/capactiy-graph.component' import { OptionsComponent } from './modules/options/components/options.component' import { ChangepasswordComponent } from './modules/changepassword/components/changepassword.component' export const ROUTES: Routes = [ { path: 'group', component: LogicalGroupComponent }, { path: 'network', component: TopologyGraphComponent }, { path: 'hierarchy', component: TopoGraphComponent }, { path: 'capacity', component: CapactiyGraphComponent }, { path: 'changepassword', component: ChangepasswordComponent }, { path: 'options', component: OptionsComponent }, { path: '**', redirectTo: 'group', pathMatch: 'full' } ]; export const ROUTING: ModuleWithProviders = RouterModule.forRoot(ROUTES); ================================================ FILE: ui/src/app/common/messages/common.messages.ts ================================================ export const MCommon: any = Object.freeze({ appHeader: 'PURSER' }); ================================================ FILE: ui/src/app/common/messages/left-navigation.messages.ts ================================================ export const MLeftNav: any = Object.freeze({ homeText: 'Home' }); ================================================ FILE: ui/src/app/modules/capacity-graph/capacity-graph.module.ts ================================================ import { CommonModule } from '@angular/common'; import { CUSTOM_ELEMENTS_SCHEMA, NgModule } from '@angular/core'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { GoogleChartsModule } from 'angular-google-charts'; import { CapactiyGraphComponent } from './components/capactiy-graph.component'; import { CapacityGraphService } from './services/capacity-graph.service'; @NgModule({ imports: [ CommonModule, ClarityModule, FormsModule, GoogleChartsModule ], exports: [CapactiyGraphComponent], declarations: [CapactiyGraphComponent], providers: [CapacityGraphService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class CapacityGraphModule { } ================================================ FILE: ui/src/app/modules/capacity-graph/components/capactiy-graph.component.html ================================================
Capacity
Name Type {{item.displayValue + (item.units ? '('+item.units+')' : '')}} {{item.displayValue + ' Cost ($)'}} Total Cost ($)
{{rootItem.name || '-'}} {{rootItem.type || '-'}} {{rootItem[item.value] | number: '1.0-2' || 0}} {{rootItem[item.value+'Cost'] | number: '1.0-2' || 0}} {{rootItem['cpuCost'] + rootItem['memoryCost'] + rootItem['storageCost'] | number: '1.0-2' || 0}}

{{resourceType + ' Resource Allocation vs Capacity'}}

CPU

Allocated: {{ cpuAllocated }} vCPU
Capacity: {{ cpuCapacity }} vCPU

{{ cpuRatio }}%

Memory

Allocated: {{ memoryAllocated }} GB
Capacity: {{ memoryCapacity }} GB

{{ memoryRatio }}%

Storage

Allocated: {{ storageAllocated }} GB
Capacity: {{ storageCapacity }} GB

{{ storageRatio }}%
================================================ FILE: ui/src/app/modules/capacity-graph/components/capactiy-graph.component.scss ================================================ .graphCardBlock{ ::ng-deep .googleChart{ width: 100%; } .headerBlock{ display: flex; .headerText{ font-size: 18px; } .card-title{ flex: 1; } .toggleDiv{ .viewSwitchLeftLabel{ padding-right: 5px; } } } .card-text{ text-align: center; } .radioWrapper{ padding: 5px; .radioLabel{ padding-left: 5px; } } .googleChartDiv{ padding-top: 10px; } .selectDiv{ display: flex; .selectDropdownDiv{ padding-left: 60px; } } } ================================================ FILE: ui/src/app/modules/capacity-graph/components/capactiy-graph.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { CapactiyGraphComponent } from './capactiy-graph.component'; describe('CapactiyGraphComponent', () => { let component: CapactiyGraphComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ CapactiyGraphComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(CapactiyGraphComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/capacity-graph/components/capactiy-graph.component.ts ================================================ import { Component, OnInit, ViewChild, ElementRef } from '@angular/core'; import { Router } from '@angular/router'; import { Observable } from 'rxjs'; import { CapacityGraphService } from '../services/capacity-graph.service'; const STATUS_WAIT = 'WAIT', STATUS_READY = 'READY', STATUS_NODATA = 'NO_DATA'; @Component({ selector: 'app-capactiy-graph', templateUrl: './capactiy-graph.component.html', styleUrls: ['./capactiy-graph.component.scss'] }) export class CapactiyGraphComponent implements OnInit { public cpuAllocated = 100.0; public cpuCapacity = 100.0; public cpuRatio = 100; public memoryAllocated = 100.0; public memoryCapacity = 100.0; public memoryRatio = 100; public storageAllocated = 100.0; public storageCapacity = 100.0; public storageRatio = 100; public resourceType = 'Cluster'; //PUBLIC public CAPA_STATUS = STATUS_WAIT; public graphData = []; public colNames = ['Child', 'Parent', 'Metrics']; public chartOptions = { nodeClass: 'customNode', allowHtml: true, animation: { startup: true, duration: 1000, easing: 'out', }, minColor: '#009688', midColor: '#f7f7f7', maxColor: '#ee8100', headerHeight: 40, }; public selectedMetric: string = 'cpu'; public metricOptions: any = [ { displayValue: 'CPU', value: 'cpu', units: 'vCPU' }, { displayValue: 'Memory', value: 'memory', units: 'GB' }, { displayValue: 'Storage', value: 'storage', units: 'GB' } //{ displayValue: 'Network', value: 'network' } ]; public physicalView: boolean = false; public rootItem: any = {}; public filterItems: any = []; public selectedFilterItem: string = 'select'; //PRIVATE private orgCapaData: any = {}; private capaData: any = {}; private keysToConsider: any = ['service', 'pod', 'container', 'process', 'cluster', 'namespace', 'deployment', 'replicaset', 'node', 'daemonset', 'job', 'statefulset', 'children', 'pv', 'pvc']; private uniqNames: any = []; constructor(private router: Router, private capacityGraphService: CapacityGraphService) { } private getCapacityData() { let observableEntity: Observable = this.capacityGraphService.getCapacityData(this.physicalView); this.CAPA_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { return; } this.capaData = response && response.data || {}; this.orgCapaData = JSON.parse(JSON.stringify(this.capaData)); this.constructData(this.capaData); }, (err) => { this.CAPA_STATUS = STATUS_NODATA; }); } private computeAllocationRatios(data) { if (!!data.cpuCapacity) { this.cpuCapacity = data.cpuCapacity.toFixed(2); } else { this.cpuCapacity = 0; } if (!!data.cpuAllocated) { this.cpuAllocated = data.cpuAllocated.toFixed(2); } else { this.cpuAllocated = 0; } this.cpuRatio = Math.round(this.cpuAllocated * 100 / this.cpuCapacity); if (!!data.memoryCapacity) { this.memoryCapacity = data.memoryCapacity.toFixed(2); } else { this.memoryCapacity = 0; } if (!!data.memoryAllocated) { this.memoryAllocated = data.memoryAllocated.toFixed(2); } else { this.memoryAllocated = 0; } this.memoryRatio = Math.round(this.memoryAllocated * 100 / this.memoryCapacity); if (!!data.storageCapacity) { this.storageCapacity = data.storageCapacity.toFixed(2); } else { this.storageCapacity = 0; } if (!!data.storageAllocated) { this.storageAllocated = data.storageAllocated.toFixed(2); } else { this.storageAllocated = 0; } this.storageRatio = Math.round(this.storageAllocated * 100 / this.storageCapacity); if (data.type == 'node') { this.resourceType = 'Node'; this.storageAllocated = 0; this.storageCapacity = 0; this.storageRatio = 0; } else { if (data.type == 'pv') { this.resourceType = 'PersistentVolume'; this.cpuAllocated = 0; this.cpuCapacity = 0; this.cpuRatio = 0; this.memoryAllocated = 0; this.memoryCapacity = 0; this.memoryRatio = 0; } else { this.resourceType = 'Cluster'; } } } private constructRoot(capaData) { this.rootItem = capaData; let eachRow = []; let rootName = capaData && capaData.name; let metricValue = capaData[this.selectedMetric] || 0; let metricCostValue = capaData[this.selectedMetric + 'Cost'] || 0; if (rootName) { eachRow.push({ v: rootName, f: rootName + ', ' + this.selectedMetric + ': ' + metricValue.toFixed(2) + ', ' + this.selectedMetric + ' cost: ' + metricCostValue.toFixed(2) }); eachRow.push(null); eachRow.push(0); if (this.uniqNames.indexOf(rootName) === -1) { this.graphData.push(eachRow); this.uniqNames.push(rootName); } } } private pushToGraphData(item, parent) { let eachRow = []; let parentName = item.name === parent.name ? parent.type : parent.name; let metricValue = item[this.selectedMetric] || 0; let metricCostValue = item[this.selectedMetric + 'Cost'] || 0; eachRow.push({ v: item.name, f: item.name + ', ' + this.selectedMetric + ': ' + metricValue.toFixed(2) + ', ' + this.selectedMetric + ' cost: ' + metricCostValue.toFixed(2), t: item.type }); eachRow.push(parentName); eachRow.push(metricValue); if (this.uniqNames.indexOf(item.name) === -1) { this.graphData.push(eachRow); this.uniqNames.push(item.name); } } private collectFilterItems(item) { this.filterItems.push(item.name); } private constructData(capaData) { this.selectedFilterItem = 'select'; this.graphData = []; this.uniqNames = []; this.constructRoot(capaData); let data = JSON.parse(JSON.stringify(capaData)); for (let key in data) { if (this.keysToConsider.indexOf(key) > -1) { this.filterItems = []; for (let item of data[key]) { this.collectFilterItems(item); this.pushToGraphData(item, data); } } } this.computeAllocationRatios(data); this.CAPA_STATUS = STATUS_READY; //console.log(this.graphData); } public onSelect(element) { if (!element) { return; } if (!element[0]) { return; } let row = element[0].row; let selectedItem = this.graphData[row]; this.getAdditionalData(selectedItem); } private getAdditionalData(item) { let selectedItem = item; if (item && item[0] && item[0].v && item[0].t) { let name = item[0].v; let type = item[0].t; let observableEntity: Observable = this.capacityGraphService.getCapacityData(this.physicalView, type, name); this.CAPA_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { return; } let capaData = response && response.data || {}; this.constructData(capaData); }, (err) => { this.CAPA_STATUS = STATUS_NODATA; }); } else { return; } } public filterItemChange(evt) { for (let item of this.graphData) { for (let subItem of item) { if (subItem && subItem.v && subItem.v === this.selectedFilterItem) { this.getAdditionalData(item); } } } } public metricChange(evt) { this.constructData(this.orgCapaData); } public reset() { this.CAPA_STATUS = STATUS_WAIT; this.graphData = []; this.uniqNames = []; this.constructData(this.orgCapaData); } public viewChange() { this.getCapacityData() } ngOnInit() { this.getCapacityData(); } } ================================================ FILE: ui/src/app/modules/capacity-graph/services/capacity-graph.service.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { BACKEND_URL } from '../../../app.constants'; @Injectable() export class CapacityGraphService { constructor(private http: HttpClient) { } public getCapacityData(view?, type?, name?) { let _devUrl: string = './json/capacity.json'; let _url: string = BACKEND_URL + 'metrics'; if (type) { _url = _url + '/' + type; } if (view && !name) { _url = _url + '?view=physical'; } if (name) { _url = _url + '?name=' + name; _devUrl = './json/capacity1.json'; //testing purpose } //console.log(_url); return this.http.get(_url, { observe: 'body', responseType: 'json', withCredentials: true, }); } } ================================================ FILE: ui/src/app/modules/changepassword/changepassword.module.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { ChangepasswordComponent } from './components/changepassword.component'; import { ChangepasswordService } from './services/changepassword.service'; @NgModule({ imports: [ CommonModule, ClarityModule, FormsModule ], exports: [ChangepasswordComponent], declarations: [ChangepasswordComponent], providers: [ChangepasswordService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class ChangepasswordModule { } ================================================ FILE: ui/src/app/modules/changepassword/components/changepassword.component.html ================================================ ================================================ FILE: ui/src/app/modules/changepassword/components/changepassword.component.scss ================================================ ================================================ FILE: ui/src/app/modules/changepassword/components/changepassword.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { ChangepasswordComponent } from './changepassword.component'; describe('CapactiyGraphComponent', () => { let component: ChangepasswordComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ ChangepasswordComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(ChangepasswordComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/changepassword/components/changepassword.component.ts ================================================ import { Component, OnInit, ViewChild, ElementRef } from '@angular/core'; import { HttpClient } from "@angular/common/http"; import { Router } from '@angular/router'; import { Observable } from 'rxjs'; import { ChangepasswordService } from '../services/changepassword.service'; @Component({ selector: 'app-changepassword', templateUrl: './changepassword.component.html', styleUrls: ['./changepassword.component.scss'] }) export class ChangepasswordComponent implements OnInit { public form: any = {}; public notMatch = false; public LOGIN_STATUS = "wait"; ngOnInit() { this.notMatch = false; this.LOGIN_STATUS = "wait"; } constructor(private router: Router, private changepasswordService: ChangepasswordService) { } public submitChangePassword() { if (this.form.newPassword != this.form.newPasswordCheck) { this.notMatch = true; return; } this.notMatch = false; var credentials = JSON.stringify(this.form); let observableEntity: Observable = this.changepasswordService.sendLoginCredentials(credentials); observableEntity.subscribe((response) => { this.LOGIN_STATUS = "success"; }, (err) => { this.LOGIN_STATUS = "wrong"; }); } } ================================================ FILE: ui/src/app/modules/changepassword/services/changepassword.service.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { BACKEND_AUTH_URL } from '../../../app.constants'; @Injectable() export class ChangepasswordService { url: string; constructor(private http: HttpClient) { this.url = BACKEND_AUTH_URL + 'changePassword'; } public sendLoginCredentials(credentials) { return this.http.post(this.url, credentials); } } ================================================ FILE: ui/src/app/modules/logical-group/components/logical-group.component.css ================================================ .header-wrapper { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 15px; } .quick-filters { font-weight: bold; } ::ng-deep .datagrid-overlay-wrapper { overflow-x: hidden; } ================================================ FILE: ui/src/app/modules/logical-group/components/logical-group.component.html ================================================

Custom Groups

Quick Filters:
Group Name Pods CPU (vCPU) Memory (GB) Storage (GB) CPU Cost (US $) Memory Cost (US $) Storage Cost (US $) Total Cost (US $) Projected CPU Cost (US $) Projected Memory Cost (US $) Projected Storage Cost (US $) Projected Cost (US $) Last Month CPU Cost (US $) Last Month Memory Cost (US $) Last Month Storage Cost (US $) Last Month Cost (US $) Last three months CPU Cost (US $) Last three months Memory Cost (US $) Last three months Storage Cost (US $) Last three months Cost (US $) We couldn't find any custom groups! {{ group.podsCount }} {{ (group.cpu | number: '1.0-2') || 0 }} {{ (group.memory | number: '1.0-2') || 0 }} {{ (group.storage | number: '1.0-2') || 0 }} {{ (group.mtdCPUCost | number: '1.0-2') || 0 }} {{ (group.mtdMemoryCost | number: '1.0-2') || 0 }} {{ (group.mtdStorageCost | number: '1.0-2') || 0 }} {{ (group.mtdCost | number: '1.0-2') || 0 }} {{ (group.projectedCPUCost | number: '1.0-2') || 0 }} {{ (group.projectedMemoryCost | number: '1.0-2') || 0 }} {{ (group.projectedStorageCost | number: '1.0-2') || 0 }} {{ (group.projectedCost | number: '1.0-2') || 0 }} {{ (group.lastMonthCPUCost | number: '1.0-2') || 0 }} {{ (group.lastMonthMemoryCost | number: '1.0-2') || 0 }} {{ (group.lastMonthStorageCost | number: '1.0-2') || 0 }} {{ (group.lastMonthCost | number: '1.0-2') || 0 }} {{ (((group.mtdCPUCost || 0) + (group.lastMonthCPUCost || 0) + (group.lastLastMonthCPUCost || 0)) | number: '1.0-2') || 0 }} {{ (((group.mtdMemoryCost || 0) + (group.lastMonthMemoryCost || 0) + (group.lastLastMonthMemoryCost || 0))| number: '1.0-2') || 0 }} {{ (((group.mtdStorageCost || 0) + (group.lastMonthStorageCost || 0) + (group.lastLastMonthStorageCost || 0)) | number: '1.0-2') || 0 }} {{ (((group.mtdCost || 0) + (group.lastMonthCost || 0) + (group.lastLastMonthCost || 0)) | number: '1.0-2') || 0 }} Hide/Show Features Select All {{pagination.firstItem + 1}} - {{pagination.lastItem + 1}} of {{pagination.totalItems}} groups

New Custom Group

For group template click here.
To read more about groups check this.

Delete group {{ toBeDeletedGroup }}?
Group creation failed

Error: {{ creationError }}
Successfully created group.
Successfully deleted group.
Group deletion failed

Error: {{ deletionError }}
Resource Details for {{ groupToFocus.name }}
Pods : {{ groupToFocus.podsCount }}
CPU : {{ groupToFocus.cpu }}
Memory : {{ groupToFocus.memory }}
Storage : {{ groupToFocus.storage }}
CurrentMonth Cost : {{ groupToFocus.mtdCost }}
Last Month Cost : {{ groupToFocus.lastMonthCost || 0 }}
Last 3 Months Cost : {{ (groupToFocus.mtdCost || 0) + (groupToFocus.lastMonthCost || 0) + (groupToFocus.lastLastMonthCost || 0) }}
{{ costRatio }}%
MTD Costs
Projected Costs
================================================ FILE: ui/src/app/modules/logical-group/components/logical-group.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { LogicalGroupComponent } from './logical-group.component'; describe('LogicalGroupComponent', () => { let component: LogicalGroupComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ LogicalGroupComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(LogicalGroupComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/logical-group/components/logical-group.component.ts ================================================ import { Component, OnInit } from '@angular/core'; import { Router } from '@angular/router'; import { Observable } from 'rxjs'; import { LogicalGroupService } from '../services/logical-group.service'; const STATUS_WAIT = 'WAIT', STATUS_READY = 'READY', STATUS_NODATA = 'NO_DATA'; @Component({ selector: 'app-logical-group', templateUrl: './logical-group.component.html', styleUrls: ['./logical-group.component.css'] }) export class LogicalGroupComponent implements OnInit { public groups: Object[]; public GROUP_STATUS = STATUS_WAIT; public isCreateGroup = false; public isDeleteGroup = false; public isShowGroupDetails = false; public toBeDeletedGroup = "Custom Group"; public groupToFocus: any; public groupCreation = 'wait'; public groupDeletion = 'wait'; public creationError = null; public deletionError = null; public isShowMTD = false; public isShowProjected = false; public donutOptions = {}; public donutData = { "data": [] }; public group: any; public costRatio = 100; public isViewCurrentMonth = true; public isViewLastMonth = false; public isViewLastThreeMonth = false; public isViewProjected = false; constructor(private router: Router, private logicalGroupService: LogicalGroupService) { } public getCurrentMonthStatus() { if (this.isViewCurrentMonth) { return 'label-info' } else { return 'label-purple' } } public getLastMonthStatus() { if (this.isViewLastMonth) { return 'label-info' } else { return 'label-purple' } } public getLast3MonthStatus() { if (this.isViewLastThreeMonth) { return 'label-info' } else { return 'label-purple' } } public getProjectedMonthStatus() { if (this.isViewProjected) { return 'label-info' } else { return 'label-purple' } } public changeCurrentMonthView() { this.isViewCurrentMonth = !this.isViewCurrentMonth; } public changeLastMonthView() { this.isViewLastMonth = !this.isViewLastMonth; } public changeLast3MonthView() { this.isViewLastThreeMonth = !this.isViewLastThreeMonth; } public changeProjectedMonthView() { this.isViewProjected = !this.isViewProjected; } private getLogicalGroupData() { let observableEntity: Observable = this.logicalGroupService.getLogicalGroupData(); this.GROUP_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { console.log("empty response") return; } this.groups = JSON.parse(JSON.stringify(response)); }, (err) => { this.GROUP_STATUS = STATUS_NODATA; }); } public fillGroupData() { this.isCreateGroup = true; this.group = null; } public deleteGroupData() { this.toBeDeletedGroup = "Custom Group"; this.isDeleteGroup = true; } public createGroup() { let observableEntity: Observable = this.logicalGroupService.createCustomGroup(this.group); observableEntity.subscribe((response) => { console.log("successfully created group"); this.groupCreation = 'success'; setTimeout(() => this.ngOnInit(), 6000); }, (err) => { console.log("failed to create group", err); this.groupCreation = 'fail'; this.creationError = err["error"]; ; }); this.isCreateGroup = false; } public deleteGroup() { console.log("deleting group ", this.toBeDeletedGroup) let observableEntity: Observable = this.logicalGroupService.deleteCustomGroup(this.toBeDeletedGroup); observableEntity.subscribe((response) => { console.log("successfully deleted group"); this.groupDeletion = 'success'; setTimeout(() => this.ngOnInit(), 6000); }, (err) => { console.log("failed to delete group", err); this.groupDeletion = 'fail'; this.deletionError = err["error"]; }); this.isDeleteGroup = false; } public setToBeDeletedGroup(grpName) { this.toBeDeletedGroup = grpName; this.isDeleteGroup = true; } public showGroupDetails(group) { console.log("group: ", group); this.groupToFocus = group; this.isShowGroupDetails = true; this.costRatio = Math.round(this.groupToFocus.mtdCost * 100 / this.groupToFocus.projectedCost); } public reset() { this.isCreateGroup = false; this.getLogicalGroupData(); this.isDeleteGroup = false; this.isShowGroupDetails = false; this.toBeDeletedGroup = "Custom Group"; this.group = null; this.groupCreation = 'wait'; this.groupDeletion = 'wait'; } public showMTD() { this.isShowMTD = true; this.isShowProjected = false; this.donutData = { "data": [ ['CPU', this.groupToFocus.mtdCPUCost], ['Memory', this.groupToFocus.mtdMemoryCost], ['Storage', this.groupToFocus.mtdStorageCost] ] }; this.donutOptions = { title: 'Total MTD Cost for ' + this.groupToFocus.name + ': ' + this.groupToFocus.mtdCost.toFixed(2), pieHole: 0.3, pieSliceText: 'value-and-percentage', width: 750, height: 400, chartArea: { left: "10%", top: "10%", height: "80%", width: "80%" } }; } public showProjected() { this.isShowProjected = true; this.isShowMTD = false; this.donutData = { "data": [ ['CPU', this.groupToFocus.projectedCPUCost], ['Memory', this.groupToFocus.projectedMemoryCost], ['Storage', this.groupToFocus.projectedStorageCost] ] }; this.donutOptions = { title: 'Total Projected Cost for ' + this.groupToFocus.name + ': ' + this.groupToFocus.projectedCost.toFixed(2), pieHole: 0.3, pieSliceText: 'value-and-percentage', width: 750, height: 400, chartArea: { left: "10%", top: "10%", height: "80%", width: "80%" } }; } ngOnInit() { this.reset(); } } ================================================ FILE: ui/src/app/modules/logical-group/logical-group.module.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { GoogleChartsModule } from 'angular-google-charts'; import { LogicalGroupComponent } from './components/logical-group.component'; import { LogicalGroupService } from './services/logical-group.service'; @NgModule({ imports: [ CommonModule, ClarityModule, FormsModule, GoogleChartsModule ], exports: [LogicalGroupComponent], declarations: [LogicalGroupComponent], providers: [LogicalGroupService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class LogicalGroupModule { } ================================================ FILE: ui/src/app/modules/logical-group/services/logical-group.service.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { BACKEND_URL } from '../../../app.constants'; @Injectable() export class LogicalGroupService { constructor(private http: HttpClient) { } public getLogicalGroupData(name?) { let _devUrl: string = './json/logicalGroup.json'; let _url: string = BACKEND_URL + 'groups'; if (name) { _url = _url + '?name=' + name; _devUrl = './json/logicalGroup1.json'; //testing purpose } //console.log(_url); return this.http.get(_url, { observe: 'body', responseType: 'json', withCredentials: true, }); } public deleteCustomGroup(name) { let _url: string = BACKEND_URL + 'group/delete?name=' + name; return this.http.post(_url, null, { withCredentials: true }) } public createCustomGroup(groupDef) { let _url: string = BACKEND_URL + 'group/create'; return this.http.post(_url, groupDef, { withCredentials: true }) } } ================================================ FILE: ui/src/app/modules/login/components/login.component.html ================================================ ================================================ FILE: ui/src/app/modules/login/components/login.component.scss ================================================ ================================================ FILE: ui/src/app/modules/login/components/login.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { LoginComponent } from './login.component'; describe('LoginComponent', () => { let component: LoginComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ LoginComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(LoginComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/login/components/login.component.ts ================================================ import { Component, OnInit } from '@angular/core'; import { Router } from '@angular/router'; import { Observable } from 'rxjs'; import { AppComponent } from '../../../app.component'; import { LoginService } from '../services/login.service'; @Component({ selector: 'app-login', templateUrl: './login.component.html', styleUrls: ['./login.component.scss'] }) export class LoginComponent implements OnInit { public form: any = {}; public LOGIN_STATUS = "wait"; ngOnInit() { this.LOGIN_STATUS = "wait"; this.appComponent.IS_LOGEDIN = false; } constructor(private router: Router, private loginService: LoginService, private appComponent: AppComponent) { } public submitLogin() { var credentials = JSON.stringify(this.form); let observableEntity: Observable = this.loginService.sendLoginCredential(credentials); observableEntity.subscribe((response) => { this.LOGIN_STATUS = "success"; this.appComponent.IS_LOGEDIN = true; this.router.navigateByUrl('/group'); }, (err) => { this.LOGIN_STATUS = "wrong"; this.appComponent.IS_LOGEDIN = false; }); } } ================================================ FILE: ui/src/app/modules/login/login.module.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { LoginComponent } from './components/login.component'; import { LoginService } from './services/login.service'; @NgModule({ imports: [ CommonModule, ClarityModule, FormsModule ], exports: [LoginComponent], declarations: [LoginComponent], providers: [LoginService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class LoginModule { } ================================================ FILE: ui/src/app/modules/login/services/login.service.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { BACKEND_AUTH_URL } from '../../../app.constants'; @Injectable() export class LoginService { url: string; private sessionID: string; constructor(private http: HttpClient) { this.url = BACKEND_AUTH_URL + 'login'; } public sendLoginCredential(credentials) { const httpPostOptions = { withCredentials: true, } return this.http.post(this.url, credentials, httpPostOptions); } } ================================================ FILE: ui/src/app/modules/logout/components/logout.component.html ================================================

Logging out

================================================ FILE: ui/src/app/modules/logout/components/logout.component.scss ================================================ ================================================ FILE: ui/src/app/modules/logout/components/logout.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { LogoutComponent } from './logout.component'; describe('LogoutComponent', () => { let component: LogoutComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ LogoutComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(LogoutComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/logout/components/logout.component.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Component, OnInit } from '@angular/core'; import { Router } from '@angular/router'; import { AppComponent } from '../../../app.component'; import { BACKEND_AUTH_URL } from '../../../app.constants'; @Component({ selector: 'app-logout', templateUrl: './logout.component.html', styleUrls: ['./logout.component.scss'] }) export class LogoutComponent implements OnInit { public form: any = {}; public LOGIN_STATUS = "wait"; ngOnInit() { this.handleLogout(); this.LOGIN_STATUS = "wait"; } constructor(private router: Router, private http: HttpClient, private appComponent: AppComponent) { } public handleLogout() { let logoutURL = BACKEND_AUTH_URL + 'logout'; const logoutOptions = { withCredentials: true }; this.http.post(logoutURL, JSON.stringify({}), logoutOptions).subscribe((response) => { this.appComponent.IS_LOGEDIN = false; }, (err) => { console.log("Error", err); } ); this.router.navigateByUrl("./login"); } } ================================================ FILE: ui/src/app/modules/logout/logout.module.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { LogoutComponent } from './components/logout.component'; @NgModule({ imports: [ CommonModule, ClarityModule, FormsModule ], exports: [LogoutComponent], declarations: [LogoutComponent], providers: [], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class LogoutModule { } ================================================ FILE: ui/src/app/modules/options/components/options.component.html ================================================ ================================================ FILE: ui/src/app/modules/options/components/options.component.scss ================================================ ================================================ FILE: ui/src/app/modules/options/components/options.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { OptionsComponent } from './options.component'; describe('OptionsComponent', () => { let component: OptionsComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ OptionsComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(OptionsComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/options/components/options.component.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Component, OnInit } from '@angular/core'; import { BACKEND_URL } from '../../../app.constants'; @Component({ selector: 'app-options', templateUrl: './options.component.html', styleUrls: ['./options.component.scss'] }) export class OptionsComponent implements OnInit { public SYNC_STATUS = "wait"; ngOnInit() { this.SYNC_STATUS = "wait"; } constructor(private http: HttpClient) { } public initiateSync() { let syncURL = BACKEND_URL + 'sync'; const syncOptions = { withCredentials: true }; this.http.get(syncURL, syncOptions).subscribe((response) => { this.SYNC_STATUS = "requested"; console.log("sync status", this.SYNC_STATUS); }, (err) => { console.log("Error", err); this.SYNC_STATUS = "failed"; console.log("sync request status", this.SYNC_STATUS); }); } } ================================================ FILE: ui/src/app/modules/options/options.module.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { OptionsComponent } from './components/options.component'; @NgModule({ imports: [ CommonModule, ClarityModule, FormsModule ], exports: [OptionsComponent], declarations: [OptionsComponent], providers: [], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class OptionsModule { } ================================================ FILE: ui/src/app/modules/topo-graph/components/topo-graph.component.html ================================================
Hierarchy
{{item.displayText}}

================================================ FILE: ui/src/app/modules/topo-graph/components/topo-graph.component.scss ================================================ .graphCardBlock{ ::ng-deep .googleChart{ display: block; margin: 0 auto; } ::ng-deep .customNode{ border: 1px solid #2B7CE9; border-radius: 5%; background-color: whitesmoke; font-size: 14px; font-weight: 800; } .headerBlock{ display: flex; .headerText{ font-size: 18px; } .card-title{ flex: 1; } .filterDiv{ label{ padding-right: 10px; } padding-right: 60px; } .toggleDiv{ .viewSwitchLeftLabel{ padding-right: 5px; } } } .card-text{ text-align: center; overflow-x: auto; .legendDiv{ display: flex; .legend{ display: flex; align-items: center; padding: 5px; .fakeLegend{ width: 10px; height: 10px; border-radius: 50%; } .fakeLegendText{ padding-left: 5px; } } } } ::ng-deep .namespace{ color: red; } ::ng-deep .service{ color: yellow; } ::ng-deep .pod{ color: green; } ::ng-deep .container{ color: blue } ::ng-deep .process{ color: orange; } ::ng-deep .cluster{ color: orangered; } ::ng-deep .deployment{ color: purple; } ::ng-deep .replicaset{ color: palevioletred; } ::ng-deep .node{ color: royalblue; } ::ng-deep .daemonset{ color: brown; } ::ng-deep .job{ color: black; } ::ng-deep .statefulset{ color: goldenrod; } } ================================================ FILE: ui/src/app/modules/topo-graph/components/topo-graph.component.spec.ts ================================================ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { TopoGraphComponent } from './topo-graph.component'; describe('TopoGraphComponent', () => { let component: TopoGraphComponent; let fixture: ComponentFixture; beforeEach(async(() => { TestBed.configureTestingModule({ declarations: [ TopoGraphComponent ] }) .compileComponents(); })); beforeEach(() => { fixture = TestBed.createComponent(TopoGraphComponent); component = fixture.componentInstance; fixture.detectChanges(); }); it('should create', () => { expect(component).toBeTruthy(); }); }); ================================================ FILE: ui/src/app/modules/topo-graph/components/topo-graph.component.ts ================================================ import { Component, OnInit, ViewChild, ElementRef } from '@angular/core'; import { Router } from '@angular/router'; import { Observable } from 'rxjs'; import { TopoGraphService } from '../services/topo-graph.service'; const STATUS_WAIT = 'WAIT', STATUS_READY = 'READY', STATUS_NODATA = 'NO_DATA'; @Component({ selector: 'app-topo-graph', templateUrl: './topo-graph.component.html', styleUrls: ['./topo-graph.component.scss'] }) export class TopoGraphComponent implements OnInit { //PUBLIC public TOPO_STATUS = STATUS_WAIT; public graphData = []; public colNames = ['Child', 'Parent', 'Tooltip'] public chartOptions = { nodeClass: 'customNode', allowHtml: true }; public physicalView: boolean = false; public legendArr: any = [ { displayText: 'namespace', color: 'red' }, { displayText: 'service', color: 'yellow' }, { displayText: 'pod', color: 'green' }, { displayText: 'container', color: 'blue' }, { displayText: 'process', color: 'orange' }, { displayText: 'cluster', color: 'orangered' }, { displayText: 'deployment', color: 'purple' }, { displayText: 'replicaset', color: 'palevioletred' }, { displayText: 'node', color: 'royalblue' }, { displayText: 'daemonset', color: 'brown' }, { displayText: 'job', color: 'black' }, { displayText: 'statefulset', color: 'goldenrod' } ]; public filterItems: any = []; public selectedFilterItem: string = 'select'; //PRIVATE private orgTopoData: any = {}; private topoData: any = {}; private keysToConsider: any = ['service', 'pod', 'container', 'process', 'cluster', 'namespace', 'deployment', 'replicaset', 'node', 'daemonset', 'job', 'statefulset', 'children']; private uniqNames: any = []; constructor(private router: Router, private topoService: TopoGraphService) { } private getTopoData() { this.graphData = []; this.uniqNames = []; let observableEntity: Observable = this.topoService.getTopoData(this.physicalView); this.TOPO_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { return; } this.topoData = response && response.data || {}; this.orgTopoData = JSON.parse(JSON.stringify(this.topoData)); this.constructData(this.topoData); //console.log(this.topoData); }, (err) => { this.TOPO_STATUS = STATUS_NODATA; }); } private getIcon(type) { if (!type) { return 'host'; } switch (type) { case 'service': return 'cluster'; case 'pod': return 'storage'; case 'container': return 'host'; default: return 'host'; } } private pushToGraphData(item, parent) { let eachRow = []; let iconShape = this.getIcon(item.type); let parentName = item.name === parent.name ? parent.type : parent.name; eachRow.push({ v: item.name, f: '' + item.name + '', t: item.type }); eachRow.push(parentName); eachRow.push(item.type); if (this.uniqNames.indexOf(item.name) === -1) { this.graphData.push(eachRow); this.uniqNames.push(item.name); } } /*private traverse(data){ for (let key in data) { if (this.keysToConsider.indexOf(key) > -1) { for (let item of data[key]) { this.pushToGraphData(item, item); } } } }*/ private collectFilterItems(item) { this.filterItems.push(item.name); } private constructData(topoData) { let data = JSON.parse(JSON.stringify(topoData)); for (let key in data) { if (this.keysToConsider.indexOf(key) > -1) { this.filterItems = []; for (let item of data[key]) { this.collectFilterItems(item); this.pushToGraphData(item, data); /*for (let subKey in item) { if (this.keysToConsider.indexOf(subKey) > -1) { for (let subItem of item[subKey]) { this.pushToGraphData(subItem, item); for (let supSubKey in subItem) { if (this.keysToConsider.indexOf(supSubKey) > -1) { for (let supSubItem of subItem[supSubKey]) { this.pushToGraphData(supSubItem, subItem); } } } } } }*/ } } } this.TOPO_STATUS = STATUS_READY; //console.log(this.graphData); } public onSelect(element) { if (!element) { return; } if (!element[0]) { return; } let row = element[0].row; let selectedItem = this.graphData[row]; this.getAdditionalData(selectedItem); } private getAdditionalData(item) { this.selectedFilterItem = 'select'; if (item && item[0] && item[0].v && item[0].t) { let name = item[0].v; let type = item[0].t; let observableEntity: Observable = this.topoService.getTopoData(this.physicalView, type, name); this.TOPO_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { return; } let topoData = response && response.data || {}; this.constructData(topoData); }, (err) => { this.TOPO_STATUS = STATUS_NODATA; }); } else { return; } } public filterItemChange(evt) { for (let item of this.graphData) { for (let subItem of item) { if (subItem && subItem.v && subItem.v === this.selectedFilterItem) { this.uniqNames = []; this.graphData = []; this.getAdditionalData(item); } } } } public reset() { this.TOPO_STATUS = STATUS_WAIT; this.graphData = []; this.uniqNames = []; this.constructData(this.orgTopoData); } public viewChange() { this.getTopoData(); } ngOnInit() { this.getTopoData(); } } ================================================ FILE: ui/src/app/modules/topo-graph/modules.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { Routes, RouterModule } from '@angular/router'; import { TopoGraphComponent } from './components/topo-graph.component'; import { TopoGraphService } from './services/topo-graph.service'; import { GoogleChartsModule } from 'angular-google-charts'; @NgModule({ imports: [RouterModule, CommonModule, ClarityModule, FormsModule, GoogleChartsModule], declarations: [TopoGraphComponent], exports: [TopoGraphComponent], providers: [TopoGraphService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class TopoGraphModule { } ================================================ FILE: ui/src/app/modules/topo-graph/services/topo-graph.service.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { CookieService } from 'ngx-cookie-service'; import { BACKEND_URL } from '../../../app.constants'; @Injectable() export class TopoGraphService { constructor(private http: HttpClient, private cookieService: CookieService) { } public getTopoData(view?, type?, name?) { let _devUrl: string = './json/topology.json'; let _url: string = BACKEND_URL + 'hierarchy'; if (type) { _url = _url + '/' + type; } if (view && !name) { _url = _url + '?view=physical'; } if (name) { _url = _url + '?name=' + name; _devUrl = './json/topology1.json'; //testing purpose } return this.http.get(_url, { observe: 'body', responseType: 'json', withCredentials: true, }); } } ================================================ FILE: ui/src/app/modules/topologyGraph/components/index.ts ================================================ export * from './topologyGraph.component'; ================================================ FILE: ui/src/app/modules/topologyGraph/components/topologyGraph.component.html ================================================

Interactions

Scroll in to cluster. Press on the cluster to open up.
================================================ FILE: ui/src/app/modules/topologyGraph/components/topologyGraph.component.scss ================================================ .mainDiv{ height: 500px; width: 100%; border: 1px solid #ddd; } .serviceSelect{ width: 30%; } .serviceSelectLabel{ padding-right: 15px; } .actionDiv{ display: flex; } .buttonDiv{ width: 100%; text-align: right; } .spinnerDiv{ width: 100%; text-align: center; } ================================================ FILE: ui/src/app/modules/topologyGraph/components/topologyGraph.component.ts ================================================ import { Component, OnInit, ViewChild, ElementRef } from '@angular/core'; import { Router } from '@angular/router'; import { DataSet, Network } from 'vis'; import { Observable } from 'rxjs'; import { TopologyGraphService } from '../services/topologyGraph.service'; const STATUS_WAIT = 'WAIT', STATUS_READY = 'READY', STATUS_NODATA = 'NO_DATA'; @Component({ selector: 'topology-graph', templateUrl: './topologyGraph.component.html', styleUrls: ['./topologyGraph.component.scss'] }) export class TopologyGraphComponent implements OnInit { private clusterIndex = 0; private clusters = []; private lastClusterZoomLevel = 0; private clusterFactor = 0.9; private nodes: any; private edges: any; private nodesDataset: any; private edgesDataset: any; public NODE_STATUS = STATUS_WAIT; public EDGE_STATUS = STATUS_WAIT; public serviceList: any = []; public enableClustering: boolean = false; public serviceName: string = 'ALL'; @ViewChild('networkContainer') container: ElementRef; data: any = {}; options = { nodes: { shape: 'dot', size: 16 }, physics: { enabled: false, /*forceAtlas2Based: { gravitationalConstant: -26, centralGravity: 0.005, springLength: 230, springConstant: 0.18 }, maxVelocity: 146, solver: 'forceAtlas2Based', timestep: 0.35, stabilization: { iterations: 150 }*/ }, layout: { improvedLayout: false } }; network: any; constructor(private router: Router, private topologyService: TopologyGraphService) { } private getServiceList() { let observableEntity: Observable = this.topologyService.getServiceList(); this.NODE_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { return; } this.serviceList = response; }, (err) => { }); } private getNodes() { this.serviceList = []; let observableEntity: Observable = this.topologyService.getNodes(this.serviceName); this.NODE_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { this.NODE_STATUS = STATUS_NODATA; return; } this.nodes = response; for (let item of this.nodes) { if (item.cid && this.serviceList.indexOf(item.cid) === -1) { for (let cid of item.cid) { if (this.serviceList.indexOf(cid) === -1) { this.serviceList.push(cid); } } } } this.NODE_STATUS = STATUS_READY; this.initNetwork(); }, (err) => { this.NODE_STATUS = STATUS_NODATA; }); } private getEdges() { let observableEntity: Observable = this.topologyService.getEdges(this.serviceName); this.EDGE_STATUS = STATUS_WAIT; observableEntity.subscribe((response) => { if (!response) { this.EDGE_STATUS = STATUS_NODATA; return; } this.edges = response; this.EDGE_STATUS = STATUS_READY; this.initNetwork(); }, (err) => { this.EDGE_STATUS = STATUS_NODATA; }); } private initNetwork() { let filteredNodes = []; let filteredEdges = []; if (this.EDGE_STATUS === STATUS_READY && this.NODE_STATUS === STATUS_READY) { if (this.serviceName && this.serviceName !== 'ALL') { let self = this; filteredNodes = this.nodes.filter(function (item) { return item.cid.indexOf(self.serviceName) > -1; }); let idsArr = []; for (let item of filteredNodes) { idsArr.push(item.id); } //console.log(idsArr); filteredEdges = this.edges.filter(function (item) { return idsArr.indexOf(item.from) > -1 || idsArr.indexOf(item.to) > -1; }); let leftOutIdsArr = []; for (let item of filteredEdges) { if (leftOutIdsArr.indexOf(item.from) === -1) { leftOutIdsArr.push(item.from); } if (leftOutIdsArr.indexOf(item.to) === -1) { leftOutIdsArr.push(item.to); } } filteredNodes = this.nodes.filter(function (item) { return leftOutIdsArr.indexOf(item.id) > -1; }) } else { filteredNodes = this.nodes; filteredEdges = this.edges; } //console.log(filteredNodes); //console.log(filteredEdges); this.nodesDataset = new DataSet(filteredNodes); this.edgesDataset = new DataSet(filteredEdges); this.data = { nodes: this.nodesDataset, edges: this.edgesDataset }; this.network = new Network(this.container.nativeElement, this.data, this.options); this.network.stabilize(100); // if we click on a node, we want to open it up! let self = this; this.network.on("selectNode", function (params) { if (params.nodes.length === 1 && self.network.isCluster(params.nodes[0])) { self.network.openCluster(params.nodes[0]); } }); if (this.serviceName && this.enableClustering) { this.clusterByCid(); } } } public reset() { this.serviceName = 'ALL'; this.enableClustering = false; this.reload(); } public reload() { this.clusterIndex = 0; this.clusters = []; this.lastClusterZoomLevel = 0; this.clusterFactor = 0.9; this.nodes = [] this.edges = [] this.nodesDataset = [] this.edgesDataset = [] this.NODE_STATUS = STATUS_WAIT; this.EDGE_STATUS = STATUS_WAIT; this.data = {}; this.options = { nodes: { shape: 'dot', size: 16 }, physics: { enabled: false, /*forceAtlas2Based: { gravitationalConstant: -26, centralGravity: 0.005, springLength: 230, springConstant: 0.18 }, maxVelocity: 146, solver: 'forceAtlas2Based', timestep: 0.35, stabilization: { iterations: 150 }*/ }, layout: { improvedLayout: false } }; this.network = {}; this.loadApp(); } private loadApp() { this.getNodes(); this.getEdges(); } ngOnInit() { //this.getServiceList(); this.loadApp(); } public clusterByCid() { this.enableClustering = true; this.network.setData(this.data); let nodeServices = []; for (let item of this.nodes) { for (let cid of item.cid) { if (cid && nodeServices.indexOf(cid) === -1) { nodeServices.push(cid); } } } /*for (let i = 0; i < this.nodes.length; i++) { nodeServices[i] = (this.nodes[i].cid); }*/ let uniqServices = ([...nodeServices]); let clusterOptionsByData = new Array(uniqServices.length); for (let i = 0; i < uniqServices.length; i++) { clusterOptionsByData[i] = { joinCondition: function (childOptions) { //return childOptions.cid == uniqServices[i]; return childOptions.cid && childOptions.cid.indexOf(uniqServices[i]) > -1; }, clusterNodeProperties: { allowSingleNodeCluster: true, id: 'cidCluster' + i, borderWidth: 2, shape: 'star', label: uniqServices[i] } }; //console.log(clusterOptionsByData[i]); this.network.cluster(clusterOptionsByData[i]); } } } ================================================ FILE: ui/src/app/modules/topologyGraph/modules.ts ================================================ import { NgModule, CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { ClarityModule } from '@clr/angular'; import { Routes, RouterModule } from '@angular/router'; import { TopologyGraphComponent } from './components/topologyGraph.component'; import { TopologyGraphService } from './services/topologyGraph.service'; @NgModule({ imports: [RouterModule, CommonModule, ClarityModule, FormsModule], declarations: [TopologyGraphComponent], exports: [TopologyGraphComponent], providers: [TopologyGraphService], schemas: [CUSTOM_ELEMENTS_SCHEMA] }) export class TopologyGraphModule { } ================================================ FILE: ui/src/app/modules/topologyGraph/services/topologyGraph.service.ts ================================================ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { BACKEND_URL } from '../../../app.constants'; @Injectable() export class TopologyGraphService { constructor(private http: HttpClient) { } public getNodes(serviceName) { let _devUrl: string = './json/nodes.json'; let _url: string = BACKEND_URL + 'nodes'; if (serviceName && serviceName !== 'ALL') { _url = _url + '?service=' + serviceName; } return this.http.get(_url, { observe: 'body', responseType: 'json', withCredentials: true }); } public getEdges(serviceName) { let _devUrl: string = './json/edges.json'; let _url: string = BACKEND_URL + 'edges'; if (serviceName && serviceName !== 'ALL') { _url = _url + '?service=' + serviceName; } return this.http.get(_url, { observe: 'body', responseType: 'json', withCredentials: true }); } public getServiceList() { let _devUrl: string = './json/serviceList.json'; let _url: string = BACKEND_URL + 'services'; return this.http.get(_url, { observe: 'body', responseType: 'json', withCredentials: true }); } } ================================================ FILE: ui/src/assets/.gitkeep ================================================ ================================================ FILE: ui/src/browserslist ================================================ # This file is currently used by autoprefixer to adjust CSS to support the below specified browsers # For additional information regarding the format and rule options, please see: # https://github.com/browserslist/browserslist#queries # # For IE 9-11 support, please remove 'not' from the last line of the file and adjust as needed > 0.5% last 2 versions Firefox ESR not dead not IE 9-11 ================================================ FILE: ui/src/environments/environment.prod.ts ================================================ export const environment = { production: true }; ================================================ FILE: ui/src/environments/environment.ts ================================================ // This file can be replaced during build by using the `fileReplacements` array. // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. // The list of file replacements can be found in `angular.json`. export const environment = { production: false }; /* * For easier debugging in development mode, you can import the following file * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. * * This import should be commented out in production mode because it will have a negative impact * on performance if an error is thrown. */ // import 'zone.js/dist/zone-error'; // Included with Angular CLI. ================================================ FILE: ui/src/index.html ================================================ Purser
================================================ FILE: ui/src/json/logicalGroup.json ================================================ [ { "name": "vrbc", "podsCount": 882, "mtdCPU": 662.475759, "cpu": 56.9, "mtdCPUCost": 15.899418, "mtdCost": 306.798539, "mtdMemory": 29043.16396, "mtdStorage": 3365.862511, "memory": 1951.792969, "storage": 275, "mtdMemoryCost": 290.43164, "mtdStorageCost": 0.467481 }, { "name": "tango", "podsCount": 6, "mtdCPU": 223.35569, "mtdMemory": 589.78287, "cpu": 14.55, "memory": 38.406295, "mtdCPUCost": 5.360537, "mtdMemoryCost": 5.897829, "mtdCost": 11.258366 }, { "name": "symphony", "podsCount": 3, "mtdCPU": 206.529147, "mtdMemory": 812.197643, "mtdStorage": 49.76606, "cpu": 12.45, "memory": 48.960938, "storage": 3, "mtdCPUCost": 4.9567, "mtdMemoryCost": 8.121976, "mtdStorageCost": 0.006912, "mtdCost": 13.085588 }, { "name": "ops-all", "podsCount": 19, "mtdCPU": 1336.188258, "mtdMemory": 6132.373898, "mtdStorage": 289910.168038, "cpu": 88.75, "memory": 399.859913, "storage": 24380, "mtdCPUCost": 32.068518, "mtdMemoryCost": 61.323739, "mtdStorageCost": 40.265299, "mtdCost": 133.657556 } ] ================================================ FILE: ui/src/karma.conf.js ================================================ // Karma configuration file, see link for more information // https://karma-runner.github.io/1.0/config/configuration-file.html module.exports = function (config) { config.set({ basePath: '', frameworks: ['jasmine', '@angular-devkit/build-angular'], plugins: [ require('karma-jasmine'), require('karma-chrome-launcher'), require('karma-jasmine-html-reporter'), require('karma-coverage-istanbul-reporter'), require('@angular-devkit/build-angular/plugins/karma') ], client: { clearContext: false // leave Jasmine Spec Runner output visible in browser }, coverageIstanbulReporter: { dir: require('path').join(__dirname, '../coverage'), reports: ['html', 'lcovonly'], fixWebpackSourcePaths: true }, reporters: ['progress', 'kjhtml'], port: 9876, colors: true, logLevel: config.LOG_INFO, autoWatch: true, browsers: ['Chrome'], singleRun: false }); }; ================================================ FILE: ui/src/main.ts ================================================ import { enableProdMode } from '@angular/core'; import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; import { AppModule } from './app/app.module'; import { environment } from './environments/environment'; if (environment.production) { enableProdMode(); } platformBrowserDynamic().bootstrapModule(AppModule) .catch(err => console.log(err)); ================================================ FILE: ui/src/polyfills.ts ================================================ /** * This file includes polyfills needed by Angular and is loaded before the app. * You can add your own extra polyfills to this file. * * This file is divided into 2 sections: * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. * 2. Application imports. Files imported after ZoneJS that should be loaded before your main * file. * * The current setup is for so-called "evergreen" browsers; the last versions of browsers that * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. * * Learn more in https://angular.io/docs/ts/latest/guide/browser-support.html */ /*************************************************************************************************** * BROWSER POLYFILLS */ /** IE9, IE10 and IE11 requires all of the following polyfills. **/ // import 'core-js/es6/symbol'; // import 'core-js/es6/object'; // import 'core-js/es6/function'; // import 'core-js/es6/parse-int'; // import 'core-js/es6/parse-float'; // import 'core-js/es6/number'; // import 'core-js/es6/math'; // import 'core-js/es6/string'; // import 'core-js/es6/date'; // import 'core-js/es6/array'; // import 'core-js/es6/regexp'; // import 'core-js/es6/map'; // import 'core-js/es6/weak-map'; // import 'core-js/es6/set'; /** IE10 and IE11 requires the following for NgClass support on SVG elements */ // import 'classlist.js'; // Run `npm install --save classlist.js`. /** IE10 and IE11 requires the following for the Reflect API. */ // import 'core-js/es6/reflect'; /** Evergreen browsers require these. **/ // Used for reflect-metadata in JIT. If you use AOT (and only Angular decorators), you can remove. import 'core-js/es7/reflect'; /** * Web Animations `@angular/platform-browser/animations` * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). **/ // import 'web-animations-js'; // Run `npm install --save web-animations-js`. /** * By default, zone.js will patch all possible macroTask and DomEvents * user can disable parts of macroTask/DomEvents patch by setting following flags */ // (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame // (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick // (window as any).__zone_symbol__BLACK_LISTED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames /* * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js * with the following flag, it will bypass `zone.js` patch for IE/Edge */ // (window as any).__Zone_enable_cross_context_check = true; /*************************************************************************************************** * Zone JS is required by default for Angular itself. */ import 'zone.js/dist/zone'; // Included with Angular CLI. /*************************************************************************************************** * APPLICATION IMPORTS */ ================================================ FILE: ui/src/styles.css ================================================ .loading-container { position: absolute; top: 0; bottom: 0; right: 0; left: 0; } .loading-container .spinner { position: absolute; top: 0; bottom: 0; right: 0; left: 0; margin: auto; } ================================================ FILE: ui/src/test.ts ================================================ // This file is required by karma.conf.js and loads recursively all the .spec and framework files import 'zone.js/dist/zone-testing'; import { getTestBed } from '@angular/core/testing'; import { BrowserDynamicTestingModule, platformBrowserDynamicTesting } from '@angular/platform-browser-dynamic/testing'; declare const require: any; // First, initialize the Angular testing environment. getTestBed().initTestEnvironment( BrowserDynamicTestingModule, platformBrowserDynamicTesting() ); // Then we find all the tests. const context = require.context('./', true, /\.spec\.ts$/); // And load the modules. context.keys().map(context); ================================================ FILE: ui/src/tsconfig.app.json ================================================ { "extends": "../tsconfig.json", "compilerOptions": { "outDir": "../out-tsc/app", "types": [] }, "exclude": [ "test.ts", "**/*.spec.ts" ] } ================================================ FILE: ui/src/tsconfig.spec.json ================================================ { "extends": "../tsconfig.json", "compilerOptions": { "outDir": "../out-tsc/spec", "types": [ "jasmine", "node" ] }, "files": [ "test.ts", "polyfills.ts" ], "include": [ "**/*.spec.ts", "**/*.d.ts" ] } ================================================ FILE: ui/src/tslint.json ================================================ { "extends": "../tslint.json", "rules": { "directive-selector": [ true, "attribute", "app", "camelCase" ], "component-selector": [ true, "element", "app", "kebab-case" ] } } ================================================ FILE: ui/tsconfig.json ================================================ { "compileOnSave": false, "compilerOptions": { "baseUrl": "./", "outDir": "./dist/out-tsc", "sourceMap": true, "declaration": false, "module": "es2015", "moduleResolution": "node", "emitDecoratorMetadata": true, "experimentalDecorators": true, "target": "es5", "typeRoots": [ "node_modules/@types" ], "lib": [ "es2017", "dom" ] } } ================================================ FILE: ui/tslint.json ================================================ { "rulesDirectory": [ "node_modules/codelyzer" ], "rules": { "arrow-return-shorthand": true, "callable-types": true, "class-name": true, "comment-format": [ true, "check-space" ], "curly": true, "deprecation": { "severity": "warn" }, "eofline": true, "forin": true, "import-blacklist": [ true, "rxjs/Rx" ], "import-spacing": true, "indent": [ true, "spaces" ], "interface-over-type-literal": true, "label-position": true, "max-line-length": [ true, 140 ], "member-access": false, "member-ordering": [ true, { "order": [ "static-field", "instance-field", "static-method", "instance-method" ] } ], "no-arg": true, "no-bitwise": true, "no-console": [ true, "debug", "info", "time", "timeEnd", "trace" ], "no-construct": true, "no-debugger": true, "no-duplicate-super": true, "no-empty": false, "no-empty-interface": true, "no-eval": true, "no-inferrable-types": [ true, "ignore-params" ], "no-misused-new": true, "no-non-null-assertion": true, "no-redundant-jsdoc": true, "no-shadowed-variable": true, "no-string-literal": false, "no-string-throw": true, "no-switch-case-fall-through": true, "no-trailing-whitespace": true, "no-unnecessary-initializer": true, "no-unused-expression": true, "no-use-before-declare": true, "no-var-keyword": true, "object-literal-sort-keys": false, "one-line": [ true, "check-open-brace", "check-catch", "check-else", "check-whitespace" ], "prefer-const": true, "quotemark": [ true, "single" ], "radix": true, "semicolon": [ true, "always" ], "triple-equals": [ true, "allow-null-check" ], "typedef-whitespace": [ true, { "call-signature": "nospace", "index-signature": "nospace", "parameter": "nospace", "property-declaration": "nospace", "variable-declaration": "nospace" } ], "unified-signatures": true, "variable-name": false, "whitespace": [ true, "check-branch", "check-decl", "check-operator", "check-separator", "check-type" ], "no-output-on-prefix": true, "use-input-property-decorator": true, "use-output-property-decorator": true, "use-host-property-decorator": true, "no-input-rename": true, "no-output-rename": true, "use-life-cycle-interface": true, "use-pipe-transform-interface": true, "component-class-suffix": true, "directive-class-suffix": true } }