Repository: nathanielc/morgoth Branch: master Commit: c9874e8c4225 Files: 352 Total size: 2.0 MB Directory structure: gitextract_y5n2h024/ ├── .gitignore ├── .gitmodules ├── .travis.yml ├── Gopkg.toml ├── LICENSE ├── README.md ├── cmd/ │ └── morgoth/ │ └── main.go ├── counter/ │ ├── counter.go │ ├── lossy_counter.go │ └── lossy_counter_test.go ├── detector.go ├── fingerprint.go ├── fingerprinters/ │ ├── jsdiv/ │ │ ├── jsdiv.go │ │ └── jsdiv_test.go │ ├── kstest/ │ │ ├── kstest.go │ │ └── kstest_test.go │ └── sigma/ │ └── sigma.go ├── vendor/ │ └── github.com/ │ ├── beorn7/ │ │ └── perks/ │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ └── quantile/ │ │ ├── bench_test.go │ │ ├── example_test.go │ │ ├── exampledata.txt │ │ ├── stream.go │ │ └── stream_test.go │ ├── davecgh/ │ │ └── go-spew/ │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── cov_report.sh │ │ ├── spew/ │ │ │ ├── bypass.go │ │ │ ├── bypasssafe.go │ │ │ ├── common.go │ │ │ ├── common_test.go │ │ │ ├── config.go │ │ │ ├── doc.go │ │ │ ├── dump.go │ │ │ ├── dump_test.go │ │ │ ├── dumpcgo_test.go │ │ │ ├── dumpnocgo_test.go │ │ │ ├── example_test.go │ │ │ ├── format.go │ │ │ ├── format_test.go │ │ │ ├── internal_test.go │ │ │ ├── internalunsafe_test.go │ │ │ ├── spew.go │ │ │ └── spew_test.go │ │ └── test_coverage.txt │ ├── golang/ │ │ └── protobuf/ │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── AUTHORS │ │ ├── CONTRIBUTORS │ │ ├── LICENSE │ │ ├── Make.protobuf │ │ ├── Makefile │ │ ├── README.md │ │ └── proto/ │ │ ├── Makefile │ │ ├── all_test.go │ │ ├── any_test.go │ │ ├── clone.go │ │ ├── clone_test.go │ │ ├── decode.go │ │ ├── decode_test.go │ │ ├── encode.go │ │ ├── encode_test.go │ │ ├── equal.go │ │ ├── equal_test.go │ │ ├── extensions.go │ │ ├── extensions_test.go │ │ ├── lib.go │ │ ├── map_test.go │ │ ├── message_set.go │ │ ├── message_set_test.go │ │ ├── pointer_reflect.go │ │ ├── pointer_unsafe.go │ │ ├── properties.go │ │ ├── proto3_test.go │ │ ├── size2_test.go │ │ ├── size_test.go │ │ ├── text.go │ │ ├── text_parser.go │ │ ├── text_parser_test.go │ │ └── text_test.go │ ├── influxdata/ │ │ ├── kapacitor/ │ │ │ ├── .dockerignore │ │ │ ├── .gitattributes │ │ │ ├── .gitignore │ │ │ ├── BLOB_STORE_DESIGN.md │ │ │ ├── CHANGELOG.md │ │ │ ├── CONTRIBUTING.md │ │ │ ├── DESIGN.md │ │ │ ├── Dockerfile_build_ubuntu32 │ │ │ ├── Dockerfile_build_ubuntu64 │ │ │ ├── Gopkg.toml │ │ │ ├── LICENSE │ │ │ ├── LICENSE_OF_DEPENDENCIES.md │ │ │ ├── README.md │ │ │ ├── alert.go │ │ │ ├── autoscale.go │ │ │ ├── batch.go │ │ │ ├── build.py │ │ │ ├── build.sh │ │ │ ├── circle-test.sh │ │ │ ├── circle.yml │ │ │ ├── combine.go │ │ │ ├── combine_test.go │ │ │ ├── default.go │ │ │ ├── delete.go │ │ │ ├── derivative.go │ │ │ ├── doc.go │ │ │ ├── edge.go │ │ │ ├── eval.go │ │ │ ├── expr.go │ │ │ ├── flatten.go │ │ │ ├── gobuild.sh │ │ │ ├── group_by.go │ │ │ ├── http_out.go │ │ │ ├── http_post.go │ │ │ ├── influxdb_out.go │ │ │ ├── influxql.gen.go │ │ │ ├── influxql.gen.go.tmpl │ │ │ ├── influxql.go │ │ │ ├── join.go │ │ │ ├── kapacitor_loopback.go │ │ │ ├── list-deps │ │ │ ├── log.go │ │ │ ├── metaclient.go │ │ │ ├── node.go │ │ │ ├── noop.go │ │ │ ├── output.go │ │ │ ├── query.go │ │ │ ├── query_test.go │ │ │ ├── replay.go │ │ │ ├── result.go │ │ │ ├── sample.go │ │ │ ├── shift.go │ │ │ ├── state_tracking.go │ │ │ ├── stats.go │ │ │ ├── stream.go │ │ │ ├── task.go │ │ │ ├── task_master.go │ │ │ ├── template.go │ │ │ ├── test.sh │ │ │ ├── tmpldata.json │ │ │ ├── udf/ │ │ │ │ ├── agent/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── agent.go │ │ │ │ │ ├── io.go │ │ │ │ │ ├── io_test.go │ │ │ │ │ ├── server.go │ │ │ │ │ ├── udf.pb.go │ │ │ │ │ └── udf.proto │ │ │ │ ├── server.go │ │ │ │ ├── server_test.go │ │ │ │ └── udf.go │ │ │ ├── udf.go │ │ │ ├── udf_test.go │ │ │ ├── union.go │ │ │ ├── update_tick_docs.sh │ │ │ ├── where.go │ │ │ ├── window.go │ │ │ └── window_test.go │ │ └── wlog/ │ │ ├── LICENSE │ │ ├── README.md │ │ └── writer.go │ ├── matttproud/ │ │ └── golang_protobuf_extensions/ │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── NOTICE │ │ ├── README.md │ │ └── pbutil/ │ │ ├── all_test.go │ │ ├── decode.go │ │ ├── decode_test.go │ │ ├── doc.go │ │ ├── encode.go │ │ ├── encode_test.go │ │ └── fixtures_test.go │ ├── pkg/ │ │ └── errors/ │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── appveyor.yml │ │ ├── bench_test.go │ │ ├── errors.go │ │ ├── errors_test.go │ │ ├── example_test.go │ │ ├── format_test.go │ │ ├── stack.go │ │ └── stack_test.go │ ├── pmezard/ │ │ └── go-difflib/ │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ └── difflib/ │ │ ├── difflib.go │ │ └── difflib_test.go │ ├── prometheus/ │ │ ├── client_golang/ │ │ │ ├── .gitignore │ │ │ ├── .travis.yml │ │ │ ├── CHANGELOG.md │ │ │ ├── CONTRIBUTING.md │ │ │ ├── LICENSE │ │ │ ├── MAINTAINERS.md │ │ │ ├── NOTICE │ │ │ ├── README.md │ │ │ ├── VERSION │ │ │ └── prometheus/ │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── benchmark_test.go │ │ │ ├── collector.go │ │ │ ├── counter.go │ │ │ ├── counter_test.go │ │ │ ├── desc.go │ │ │ ├── desc_test.go │ │ │ ├── doc.go │ │ │ ├── example_clustermanager_test.go │ │ │ ├── example_timer_complex_test.go │ │ │ ├── example_timer_gauge_test.go │ │ │ ├── example_timer_test.go │ │ │ ├── examples_test.go │ │ │ ├── expvar_collector.go │ │ │ ├── expvar_collector_test.go │ │ │ ├── fnv.go │ │ │ ├── gauge.go │ │ │ ├── gauge_test.go │ │ │ ├── go_collector.go │ │ │ ├── go_collector_test.go │ │ │ ├── histogram.go │ │ │ ├── histogram_test.go │ │ │ ├── http.go │ │ │ ├── http_test.go │ │ │ ├── labels.go │ │ │ ├── metric.go │ │ │ ├── metric_test.go │ │ │ ├── observer.go │ │ │ ├── process_collector.go │ │ │ ├── process_collector_test.go │ │ │ ├── promhttp/ │ │ │ │ ├── delegator.go │ │ │ │ ├── delegator_1_8.go │ │ │ │ ├── delegator_pre_1_8.go │ │ │ │ ├── http.go │ │ │ │ ├── http_test.go │ │ │ │ ├── instrument_client.go │ │ │ │ ├── instrument_client_1_8.go │ │ │ │ ├── instrument_client_1_8_test.go │ │ │ │ ├── instrument_server.go │ │ │ │ └── instrument_server_test.go │ │ │ ├── registry.go │ │ │ ├── registry_test.go │ │ │ ├── summary.go │ │ │ ├── summary_test.go │ │ │ ├── timer.go │ │ │ ├── timer_test.go │ │ │ ├── untyped.go │ │ │ ├── value.go │ │ │ ├── value_test.go │ │ │ ├── vec.go │ │ │ └── vec_test.go │ │ ├── client_model/ │ │ │ ├── .gitignore │ │ │ ├── CONTRIBUTING.md │ │ │ ├── LICENSE │ │ │ ├── MAINTAINERS.md │ │ │ ├── Makefile │ │ │ ├── NOTICE │ │ │ ├── README.md │ │ │ ├── go/ │ │ │ │ └── metrics.pb.go │ │ │ ├── metrics.proto │ │ │ ├── pom.xml │ │ │ └── setup.py │ │ ├── common/ │ │ │ ├── .travis.yml │ │ │ ├── CONTRIBUTING.md │ │ │ ├── LICENSE │ │ │ ├── MAINTAINERS.md │ │ │ ├── NOTICE │ │ │ ├── README.md │ │ │ ├── expfmt/ │ │ │ │ ├── bench_test.go │ │ │ │ ├── decode.go │ │ │ │ ├── decode_test.go │ │ │ │ ├── encode.go │ │ │ │ ├── expfmt.go │ │ │ │ ├── fuzz.go │ │ │ │ ├── text_create.go │ │ │ │ ├── text_create_test.go │ │ │ │ ├── text_parse.go │ │ │ │ └── text_parse_test.go │ │ │ ├── internal/ │ │ │ │ └── bitbucket.org/ │ │ │ │ └── ww/ │ │ │ │ └── goautoneg/ │ │ │ │ ├── README.txt │ │ │ │ ├── autoneg.go │ │ │ │ └── autoneg_test.go │ │ │ └── model/ │ │ │ ├── alert.go │ │ │ ├── alert_test.go │ │ │ ├── fingerprinting.go │ │ │ ├── fnv.go │ │ │ ├── labels.go │ │ │ ├── labels_test.go │ │ │ ├── labelset.go │ │ │ ├── metric.go │ │ │ ├── metric_test.go │ │ │ ├── model.go │ │ │ ├── signature.go │ │ │ ├── signature_test.go │ │ │ ├── silence.go │ │ │ ├── silence_test.go │ │ │ ├── time.go │ │ │ ├── time_test.go │ │ │ ├── value.go │ │ │ └── value_test.go │ │ └── procfs/ │ │ ├── .travis.yml │ │ ├── CONTRIBUTING.md │ │ ├── LICENSE │ │ ├── MAINTAINERS.md │ │ ├── Makefile │ │ ├── NOTICE │ │ ├── README.md │ │ ├── buddyinfo.go │ │ ├── buddyinfo_test.go │ │ ├── doc.go │ │ ├── fs.go │ │ ├── fs_test.go │ │ ├── ipvs.go │ │ ├── ipvs_test.go │ │ ├── mdstat.go │ │ ├── mdstat_test.go │ │ ├── mountstats.go │ │ ├── mountstats_test.go │ │ ├── proc.go │ │ ├── proc_io.go │ │ ├── proc_io_test.go │ │ ├── proc_limits.go │ │ ├── proc_limits_test.go │ │ ├── proc_stat.go │ │ ├── proc_stat_test.go │ │ ├── proc_test.go │ │ ├── stat.go │ │ ├── stat_test.go │ │ ├── ttar │ │ ├── xfrm.go │ │ ├── xfrm_test.go │ │ └── xfs/ │ │ ├── parse.go │ │ ├── parse_test.go │ │ └── xfs.go │ └── stretchr/ │ └── testify/ │ ├── .gitignore │ ├── .travis.yml │ ├── LICENCE.txt │ ├── LICENSE │ ├── README.md │ ├── assert/ │ │ ├── assertion_forward.go │ │ ├── assertion_forward.go.tmpl │ │ ├── assertions.go │ │ ├── assertions_test.go │ │ ├── doc.go │ │ ├── errors.go │ │ ├── forward_assertions.go │ │ ├── forward_assertions_test.go │ │ ├── http_assertions.go │ │ └── http_assertions_test.go │ ├── doc.go │ └── package_test.go └── window.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ ================================================ FILE: .gitmodules ================================================ [submodule "docs"] path = docs url = https://github.com/nvcook42/morgoth.git branch = gh-pages ================================================ FILE: .travis.yml ================================================ language: go sudo: false go: - 1.7 install: true script: - go test -v $(go list ./... | grep -v vendor/) before_deploy: # Build binary - go get -u github.com/mitchellh/gox - CGO_ENABLED=0 gox ./cmd/morgoth deploy: provider: releases api_key: secure: cK0/w0ggcVawInO+b++4qqwMGrVoP7PzNkTxaNq+tbFc9cu8CSUS2+PlgRkdoRYfD+fohri3lqdzHUH0rcC7gOfx/Dvtw7mcs14gzfvuNL+/xMlKn/mBoPkOGPXQeh86qsmGbsrgDzv/BwGNgxqqEimxdiev2ZDlssTD16RQZR8= file: - morgoth_darwin_386 - morgoth_darwin_amd64 - morgoth_freebsd_386 - morgoth_freebsd_amd64 - morgoth_freebsd_arm - morgoth_linux_386 - morgoth_linux_amd64 - morgoth_linux_arm - morgoth_netbsd_386 - morgoth_netbsd_amd64 - morgoth_netbsd_arm - morgoth_openbsd_386 - morgoth_openbsd_amd64 - morgoth_windows_386.exe - morgoth_windows_amd64.exe skip_cleanup: true on: tags: true ================================================ FILE: Gopkg.toml ================================================ [[constraint]] branch = "master" name = "github.com/influxdata/kapacitor" [[constraint]] branch = "master" name = "github.com/prometheus/client_golang" [[constraint]] branch = "master" name = "github.com/prometheus/client_model" ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: README.md ================================================ Morgoth [![Build Status](https://travis-ci.org/nathanielc/morgoth.svg?branch=master)](https://travis-ci.org/nathanielc/morgoth) ======= Morgoth is a framework for flexible anomaly detection algorithms packaged to be used with [Kapacitor](https://github.com/influxdata/kapacitor/) Morgoth provides a framework for implementing the smaller pieces of an anomaly detection problem. The basic framework is that Morgoth maintains a dictionary of normal behaviors and compares new windows of data to the normal dictionary. If the new window of data is not found in the dictionary then it is considered anomalous. Morgoth uses algorithms, called fingerprinters, to compare windows of data to determine if they are similar. The [Lossy Counting Algorithm](http://www.vldb.org/conf/2002/S10P03.pdf)(LCA) is used to maintain the dictionary of normal windows. The LCA is a space efficient algorithm that can account for drift in the normal dictionary, more on LCA below. Morgoth uses a consensus model where each fingerprinter votes for whether it thinks the current window is anomalous. If the total votes percentage is greater than a consensus threshold then the window is considered anomalous. ## Getting started ### Install Morgoth can be installed via go: ```sh go get github.com/nathanielc/morgoth/cmd/morgoth ``` ### Configuring Morgoth can run as either a child process of Kapacitor or as a standalone daemon that listens on a socket. #### Child Process Morgoth is a UDF for [Kapacitor](https://github.com/influxdata/kapacitor). Add this configuration to Kapacitor in order to enable using Morgoth. ``` [udf] [udf.functions] [udf.functions.morgoth] prog = "/path/to/bin/morgoth" timeout = "10s" ``` Restart Kapacitor and you are ready to start using Morgoth within Kapacitor. #### Socket To use Morgoth as a socket UDF start the morgoth process with the `-socket` option. ``` morgoth -socket /path/to/morgoth/socket ``` Next you will need to configure Kapacitor to use the morgoth socket. ``` [udf] [udf.functions] [udf.functions.morgoth] socket = "/path/to/morgoth/socket" timeout = "10s" ``` Restart Kapacitor and you are ready to start using Morgoth within Kapacitor. ### TICKscript Here is an example TICKscript for detecting anomalies in cpu data from [Telegraf](https://github.com/influxdata/telegraf). ```javascript stream |from() .measurement('cpu') .where(lambda: "cpu" == 'cpu-total') .groupBy(*) |window() .period(1m) .every(1m) @morgoth() // track the 'usage_idle' field .field('usage_idle') // label output data as anomalous using the 'anomalous' boolean field. .anomalousField('anomalous') .errorTolerance(0.01) // The window is anomalous if it occurs less the 5% of the time. .minSupport(0.05) // Use the sigma fingerprinter .sigma(3.0) // Multiple fingerprinters can be defined... |alert() // Trigger a critical alert when the window is marked as anomalous. .crit(lambda: "anomalous") ``` ## Fingerprinters A fingerprinter is a method that can determine if a window of data is similar to a previous window of data. In effect the fingerprinters take fingerprints of the incoming data and can compare fingerprints of new data to see if they match. These fingerprinting algorithms provide the core of Morgoth as they are the means by which Morgoth determines if a new window of data is new or something already observed. An example fingerprinting algorithm is a *sigma* algorithm that computes the mean and standard deviation of a window and store them as the fingerprint for the window. When a new window arrives it compares the fingerprint (mean, stddev) of the new window to the previous window. If the windows are too far apart then they are not considered at match. By defining several fingerprinting algorithms Morgoth can decide whether new data is anomalous or normal. ## Lossy Counting Algorithm The LCA counts frequent items in a stream of data. It is *lossy* because to conserve space it will drop less frequent items. The result is that the algorithm will find frequent items but may loose track of less frequent items. More on the specific mathematical properties of the algorithm can be found below. There are two parameters to the algorithm, error tolerance (e) and minimum support (m). First e is in the range [0, 1] and is an error bound, interpreted as a percentage value. For example given and e = 0.01 (1%), items less the 1% frequent in the data set can be dropped. Decreasing e will require more space but will keep track of less frequent items. Increasing e will require less space but will loose track of less frequent items. Second m is in the range [0, 1] and is a minimum support such that items that are considered frequent have at least m% frequency. For example if m = 0.05 (5%) then if an item has a support less than 5% it is not considered frequent, aka normal. The minimum support becomes the threshold for when items are considered anomalous. Notice that m > e, this is so that we reduce the number of false positives. For example say we set e = 5% and m = 5%. If a *normal* behavior X, has a true frequency of 6% than based on variations in the true frequency, X might fall below 5% for a small interval and be dropped. This will cause X's frequency to be underestimated, which will cause it to be flagged as an anomaly, triggering a false positive. By setting e < m we have a buffer to help mitigate creating false positives. ### Properties The Lossy Counting algorithm has three properties: 1. there are no false negatives, 2. false positives are guaranteed to have a frequency of at least (m - e)*N, 3. the frequency of an item can underestimated by at most e*N, where N is the number of items encountered. The space requirements for the algorithm are at most (1 / e) * log(e*N). It has also been show that if the item with low frequency are uniformly random than the space requirements are no more than 7 / e. This means that as Morgoth continues to processes windows of data its memory usage will grow as the log of the number of windows and can reach a stable upper bound. ## Metrics Morgoth exposes metrics about each detector and fingerprinter. The metrics are exposed as a promethues `/metrics` endpoint over HTTP. By default the metrics HTTP endpoint binds to `:6767`. >NOTE: Using the metrics HTTP endpoint only makes sense if you are using Morgoth in socket mode as otherwise each new process would collide on the bind port. Metrics will have some or all of these labels: * task - the Kapacitor task ID. * node - the ID of the morgoth node within the Kapacitor task. * group - the Kapacitor group ID. * fingerprinter - the unique name for the specific fingerprinter, i.e. `sigma-0`. The most useful metric for debugging why Morgoth is not behaving as expected is likely to be the `morgoth_unique_fingerprints` gauge. The metric reports the number of unique fingerprints each fingerprinter is tracking. This is useful because if the number is large or growing with each new window its likely that the fingerprinter is erroneously marking every window as anomalous. By providing visibility into each fingerprinter, Morgoth can be tuned as needed. Using Kapacitor's scraping service you can scrape the Morgoth UDF process for these metrics and consume them within Kapacitor. See this [tutorial](https://docs.influxdata.com/kapacitor/latest/pull_metrics/scraping-and-discovery/) for more information. ================================================ FILE: cmd/morgoth/main.go ================================================ package main import ( "flag" "fmt" "log" "net" "net/http" "os" "strings" "syscall" "github.com/influxdata/kapacitor/udf/agent" "github.com/influxdata/wlog" "github.com/nathanielc/morgoth" "github.com/nathanielc/morgoth/counter" "github.com/nathanielc/morgoth/fingerprinters/jsdiv" "github.com/nathanielc/morgoth/fingerprinters/kstest" "github.com/nathanielc/morgoth/fingerprinters/sigma" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( defaultMinSupport = 0.05 defaultErrorTolerance = 0.01 defaultConsensus = 0.5 defaultMetricsBindAddr = ":6767" defaultAnomalousField = "anomalous" ) var socket = flag.String("socket", "", "Optional listen socket. If set then Morgoth will run in UDF socket mode, otherwise it will expect communication over STDIN/STDOUT.") var logLevel = flag.String("log-level", "info", "Default log level, one of debug, info, warn or error.") var metricsBind = flag.String("metrics-bind", defaultMetricsBindAddr, "Bind address of the metrics HTTP server. The metrics server will only start if also using the socket mode of operation.") var detectorGauge = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "morgoth_detectors", Help: "Current number of active detectors.", }) func init() { prometheus.MustRegister(detectorGauge) } func main() { // Parse flags flag.Parse() // Setup logging log.SetOutput(wlog.NewWriter(os.Stderr)) if err := wlog.SetLevelFromName(*logLevel); err != nil { log.Fatal("E! ", err) } // Create error channels metricsErr := make(chan error, 1) errC := make(chan error, 1) if *socket == "" { a := agent.New(os.Stdin, os.Stdout) h := newHandler(a) a.Handler = h defer h.Stop() log.Println("I! Starting agent using STDIN/STDOUT") a.Start() go func() { errC <- a.Wait() }() } else { // Start the metrics server. // Only start the metrics server in socket mode or the bind address would conflict with each new process. go func() { log.Println("I! Starting metrics HTTP server on", *metricsBind) http.Handle("/metrics", promhttp.Handler()) metricsErr <- http.ListenAndServe(*metricsBind, nil) }() // Create unix socket addr, err := net.ResolveUnixAddr("unix", *socket) if err != nil { log.Fatal("E! ", err) } l, err := net.ListenUnix("unix", addr) if err != nil { log.Fatal("E! ", err) } // Create server that listens on the socket s := agent.NewServer(l, &accepter{}) defer s.Stop() // this closes the listener // Setup signal handler to stop Server on various signals s.StopOnSignals(os.Interrupt, syscall.SIGTERM) go func() { log.Println("I! Starting socket server on", addr.String()) errC <- s.Serve() }() } select { case err := <-metricsErr: if err != nil { log.Println("E!", err) } case err := <-errC: if err != nil { log.Println("E!", err) } } log.Println("I! Stopping") } // Simple connection accepter type accepter struct { count int64 } // Create a new agent/handler for each new connection. // Count and log each new connection and termination. func (acc *accepter) Accept(conn net.Conn) { count := acc.count acc.count++ a := agent.New(conn, conn) h := newHandler(a) a.Handler = h log.Println("I! Starting agent for connection", count) a.Start() go func() { err := a.Wait() if err != nil { log.Printf("E! Agent for connection %d terminated with error: %s", count, err) } else { log.Printf("I! Agent for connection %d finished", count) } h.Close() }() } type fingerprinterInfo struct { init initFingerprinterFunc options *agent.OptionInfo } // Function that creates a new instance of a fingerprinter type createFingerprinterFunc func() morgoth.Fingerprinter // Init createFingerprinterFunc from agent.OptionValues type initFingerprinterFunc func(opts []*agent.OptionValue) (createFingerprinterFunc, error) var fingerprinters = map[string]fingerprinterInfo{ "sigma": { options: &agent.OptionInfo{ValueTypes: []agent.ValueType{agent.ValueType_DOUBLE}}, init: func(args []*agent.OptionValue) (createFingerprinterFunc, error) { deviations := args[0].Value.(*agent.OptionValue_DoubleValue).DoubleValue if deviations <= 0 { return nil, fmt.Errorf("sigma: deviations must be > 0, got %f", deviations) } return func() morgoth.Fingerprinter { return sigma.New(deviations) }, nil }, }, "kstest": { options: &agent.OptionInfo{ValueTypes: []agent.ValueType{agent.ValueType_INT}}, init: func(args []*agent.OptionValue) (createFingerprinterFunc, error) { confidence := args[0].Value.(*agent.OptionValue_IntValue).IntValue if confidence < 0 || confidence > 5 { return nil, fmt.Errorf("kstest: confidence must be in range [0,5], got %d", confidence) } return func() morgoth.Fingerprinter { return kstest.New(uint(confidence)) }, nil }, }, "jsdiv": { options: &agent.OptionInfo{ValueTypes: []agent.ValueType{ agent.ValueType_DOUBLE, agent.ValueType_DOUBLE, agent.ValueType_DOUBLE, agent.ValueType_DOUBLE, }}, init: func(args []*agent.OptionValue) (createFingerprinterFunc, error) { min := args[0].Value.(*agent.OptionValue_DoubleValue).DoubleValue max := args[1].Value.(*agent.OptionValue_DoubleValue).DoubleValue binWidth := args[2].Value.(*agent.OptionValue_DoubleValue).DoubleValue pValue := args[3].Value.(*agent.OptionValue_DoubleValue).DoubleValue if binWidth <= 0 { return nil, fmt.Errorf("jsdiv: binWidth, arg 3, must be > 0, got %f", binWidth) } if pValue <= 0 || pValue > 1 { return nil, fmt.Errorf("jsdiv: pValue, arg 4, must be in range (0,1], got %f", pValue) } if (max-min)/binWidth < 3 { return nil, fmt.Errorf("jsdiv: more than 3 bins should fit in the range [min,max]") } return func() morgoth.Fingerprinter { return jsdiv.New(min, max, binWidth, pValue) }, nil }, }, } // A Kapacitor UDF Handler type Handler struct { taskID string nodeID string field string scoreField string anomalousField string minSupport float64 errorTolerance float64 consensus float64 agent *agent.Agent currentWindow *morgoth.Window beginBatch *agent.BeginBatch batchPoints []*agent.Point detectors map[string]*morgoth.Detector fingerprinters []fingerprinterCreator } type fingerprinterCreator struct { Kind string Create createFingerprinterFunc } func newHandler(a *agent.Agent) *Handler { return &Handler{ agent: a, minSupport: defaultMinSupport, errorTolerance: defaultErrorTolerance, consensus: defaultConsensus, detectors: make(map[string]*morgoth.Detector), anomalousField: defaultAnomalousField, } } func (h *Handler) Close() { for _, d := range h.detectors { detectorGauge.Dec() d.Close() } } func (h *Handler) detectorName(group string) string { return fmt.Sprintf("%s:%s,group=%s", h.taskID, h.nodeID, group) } // Return the InfoResponse. Describing the properties of this Handler func (h *Handler) Info() (*agent.InfoResponse, error) { options := map[string]*agent.OptionInfo{ "field": {ValueTypes: []agent.ValueType{agent.ValueType_STRING}}, "scoreField": {ValueTypes: []agent.ValueType{agent.ValueType_STRING}}, "anomalousField": {ValueTypes: []agent.ValueType{agent.ValueType_STRING}}, "minSupport": {ValueTypes: []agent.ValueType{agent.ValueType_DOUBLE}}, "errorTolerance": {ValueTypes: []agent.ValueType{agent.ValueType_DOUBLE}}, "consensus": {ValueTypes: []agent.ValueType{agent.ValueType_DOUBLE}}, "logLevel": {ValueTypes: []agent.ValueType{agent.ValueType_STRING}}, } // Add in options from fingerprinters for name, info := range fingerprinters { options[name] = info.options } info := &agent.InfoResponse{ Wants: agent.EdgeType_BATCH, Provides: agent.EdgeType_BATCH, Options: options, } return info, nil } // Initialize the Handler with the provided options. func (h *Handler) Init(r *agent.InitRequest) (*agent.InitResponse, error) { h.taskID = r.TaskID h.nodeID = r.NodeID init := &agent.InitResponse{ Success: true, } var errors []string for _, opt := range r.Options { switch opt.Name { case "field": h.field = opt.Values[0].Value.(*agent.OptionValue_StringValue).StringValue case "scoreField": h.scoreField = opt.Values[0].Value.(*agent.OptionValue_StringValue).StringValue case "anomalousField": h.anomalousField = opt.Values[0].Value.(*agent.OptionValue_StringValue).StringValue case "minSupport": h.minSupport = opt.Values[0].Value.(*agent.OptionValue_DoubleValue).DoubleValue case "errorTolerance": h.errorTolerance = opt.Values[0].Value.(*agent.OptionValue_DoubleValue).DoubleValue case "consensus": h.consensus = opt.Values[0].Value.(*agent.OptionValue_DoubleValue).DoubleValue case "logLevel": level := opt.Values[0].Value.(*agent.OptionValue_StringValue).StringValue err := wlog.SetLevelFromName(level) if err != nil { init.Success = false errors = append(errors, err.Error()) } default: if info, ok := fingerprinters[opt.Name]; ok { createFn, err := info.init(opt.Values) if err != nil { init.Success = false errors = append(errors, err.Error()) } else { h.fingerprinters = append(h.fingerprinters, fingerprinterCreator{ Kind: opt.Name, Create: createFn, }) } } else { return nil, fmt.Errorf("received unknown init option %q", opt.Name) } } } if h.field == "" { errors = append(errors, "field must not be empty") } if h.anomalousField == "" { errors = append(errors, "anomalousField must not be empty") } if h.minSupport < 0 || h.minSupport > 1 { errors = append(errors, "minSupport must be in the range [0,1)") } if h.errorTolerance < 0 || h.errorTolerance > 1 { errors = append(errors, "errorTolerance must be in the range [0,1)") } if (h.consensus != -1 && h.consensus < 0) || h.consensus > 1 { errors = append(errors, "consensus must be in the range [0,1) or equal to -1") } if h.minSupport <= h.errorTolerance { errors = append(errors, "invalid minSupport or errorTolerance: minSupport must be greater than errorTolerance") } init.Success = len(errors) == 0 init.Error = strings.Join(errors, "\n") return init, nil } // Create a snapshot of the running state of the handler. func (h *Handler) Snapshot() (*agent.SnapshotResponse, error) { return &agent.SnapshotResponse{}, nil } // Restore a previous snapshot. func (h *Handler) Restore(*agent.RestoreRequest) (*agent.RestoreResponse, error) { return &agent.RestoreResponse{}, nil } // A batch has begun. func (h *Handler) BeginBatch(b *agent.BeginBatch) error { h.currentWindow = &morgoth.Window{} h.beginBatch = b h.batchPoints = h.batchPoints[0:0] return nil } // A point has arrived. func (h *Handler) Point(p *agent.Point) error { // Keep point around h.batchPoints = append(h.batchPoints, p) var value float64 if f, ok := p.FieldsDouble[h.field]; ok { value = f } else { if i, ok := p.FieldsInt[h.field]; ok { value = float64(i) } else { return fmt.Errorf("field %q is not a float or int", h.field) } } h.currentWindow.Data = append(h.currentWindow.Data, value) return nil } // The batch is complete. func (h *Handler) EndBatch(b *agent.EndBatch) error { detector, ok := h.detectors[b.Group] if !ok { metrics := h.createDetectorMetrics(b.Group) if err := metrics.Register(); err != nil { return errors.Wrapf(err, "failed to register metrics for group: %q", b.Group) } // We validated the args ourselves, ignore the error here detector, _ = morgoth.NewDetector( metrics, h.consensus, h.minSupport, h.errorTolerance, h.newFingerprinters(), ) h.detectors[b.Group] = detector detectorGauge.Inc() } anomalous, avgSupport := detector.IsAnomalous(h.currentWindow) // Send batch back to Kapacitor h.agent.Responses <- &agent.Response{ Message: &agent.Response_Begin{ Begin: h.beginBatch, }, } for _, p := range h.batchPoints { if p.FieldsBool == nil { p.FieldsBool = make(map[string]bool, 1) } p.FieldsBool[h.anomalousField] = anomalous if h.scoreField != "" { if p.FieldsDouble == nil { p.FieldsDouble = make(map[string]float64, 1) } p.FieldsDouble[h.scoreField] = 1 - avgSupport } h.agent.Responses <- &agent.Response{ Message: &agent.Response_Point{ Point: p, }, } } h.agent.Responses <- &agent.Response{ Message: &agent.Response_End{ End: b, }, } return nil } func (h *Handler) createDetectorMetrics(group string) *morgoth.DetectorMetrics { labels := prometheus.Labels{ "task": h.taskID, "node": h.nodeID, "group": group, } metrics := &morgoth.DetectorMetrics{ WindowCount: prometheus.NewCounter( prometheus.CounterOpts{ Name: "morgoth_windows_total", Help: "Number of windows processed.", ConstLabels: labels, }, ), PointCount: prometheus.NewCounter(prometheus.CounterOpts{ Name: "morgoth_points_total", Help: "Number of points processed.", ConstLabels: labels, }), AnomalousCount: prometheus.NewCounter(prometheus.CounterOpts{ Name: "morgoth_anomalies_total", Help: "Number of anomalies detected.", ConstLabels: labels, }), FingerprinterMetrics: make([]*counter.Metrics, len(h.fingerprinters)), } for i, creator := range h.fingerprinters { fingerprinterLabel := fmt.Sprintf("%s-%d", creator.Kind, i) fLabels := prometheus.Labels{ "task": h.taskID, "node": h.nodeID, "group": group, "fingerprinter": fingerprinterLabel, } metrics.FingerprinterMetrics[i] = &counter.Metrics{ UniqueFingerprints: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "morgoth_unique_fingerprints", Help: "Current number of unique fingerprints.", ConstLabels: fLabels, }), Distribution: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "morgoth_fingerprints_distribution", Help: "Distribution of counts per unique fingerprint. The label \"fp\" is an arbitrary index to identify the fingerprint and it may change.", ConstLabels: fLabels, }, []string{"fp"}, ), } // Unregistering a metric does not forget the last value. // We need to explicitly reset the value. metrics.FingerprinterMetrics[i].UniqueFingerprints.Set(0) } return metrics } // Gracefully stop the Handler. // No other methods will be called. func (h *Handler) Stop() { close(h.agent.Responses) } func (h *Handler) newFingerprinters() []morgoth.Fingerprinter { f := make([]morgoth.Fingerprinter, len(h.fingerprinters)) for i, creator := range h.fingerprinters { f[i] = creator.Create() } return f } ================================================ FILE: counter/counter.go ================================================ package counter import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) type Counter interface { // Count a fingerprint and return the support for the item. // support = count / total Count(Countable) float64 } type Countable interface { IsMatch(other Countable) bool } type Metrics struct { UniqueFingerprints prometheus.Gauge Distribution *prometheus.GaugeVec } func (m *Metrics) Register() error { if err := prometheus.Register(m.UniqueFingerprints); err != nil { return errors.Wrap(err, "unique fingerprints metric") } if err := prometheus.Register(m.Distribution); err != nil { return errors.Wrap(err, "distribution metric") } return nil } func (m *Metrics) Unregister() { prometheus.Unregister(m.UniqueFingerprints) prometheus.Unregister(m.Distribution) } ================================================ FILE: counter/lossy_counter.go ================================================ package counter import ( "math" "strconv" "sync" "github.com/prometheus/client_golang/prometheus" ) type lossyCounter struct { mu sync.RWMutex errorTolerance float64 frequencies []*entry distributionGauges []prometheus.Gauge width int total int bucket int metrics *Metrics } type entry struct { countable Countable count int delta int } //Create a new lossycounter with specified errorTolerance func NewLossyCounter(metrics *Metrics, errorTolerance float64) *lossyCounter { return &lossyCounter{ metrics: metrics, errorTolerance: errorTolerance, width: int(math.Ceil(1.0 / errorTolerance)), total: 0, bucket: 1, } } // Count a countable and return the support for the countable. func (self *lossyCounter) Count(countable Countable) float64 { self.mu.Lock() defer self.mu.Unlock() self.total++ count := 0 for i, existing := range self.frequencies { if existing.countable.IsMatch(countable) { //Found match, count it existing.count++ count = existing.count // Keep new countable to allow for drift self.frequencies[i].countable = countable self.distributionGauges[i].Set(float64(count)) break } } if count == 0 { // No matches create new entry count = 1 // Create new gauge g := self.metrics.Distribution.WithLabelValues(strconv.Itoa(len(self.distributionGauges))) g.Set(float64(count)) // Count new unique fingerprint self.metrics.UniqueFingerprints.Inc() // append self.frequencies = append(self.frequencies, &entry{ countable: countable, count: count, delta: self.bucket - 1, }) self.distributionGauges = append(self.distributionGauges, g) } if self.total%self.width == 0 { self.prune() self.bucket++ } return float64(count) / float64(self.total) } //Remove infrequent items from the list func (self *lossyCounter) prune() { filteredFreqs := self.frequencies[0:0] filteredGauges := self.distributionGauges[0:0] self.metrics.Distribution.Reset() for i, entry := range self.frequencies { if entry.count+entry.delta > self.bucket { g := self.metrics.Distribution.WithLabelValues(strconv.Itoa(i)) g.Set(float64(entry.count)) filteredFreqs = append(filteredFreqs, entry) filteredGauges = append(filteredGauges, g) } } self.frequencies = filteredFreqs self.distributionGauges = filteredGauges self.metrics.UniqueFingerprints.Set(float64(len(self.frequencies))) } ================================================ FILE: counter/lossy_counter_test.go ================================================ package counter import ( "math" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" ) //Simple fingerprint implementation type fp struct { id int } func (self *fp) IsMatch(other Countable) bool { fp, ok := other.(*fp) return ok && self.id == fp.id } var metrics = &Metrics{ UniqueFingerprints: prometheus.NewGauge( prometheus.GaugeOpts{ Name: "unique", Help: "help", }, ), Distribution: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "distribution", Help: "help", }, []string{"fp"}, ), } func TestLossyCounterShouldCountAllItems(t *testing.T) { assert := assert.New(t) lc := NewLossyCounter(metrics, 0.01) fp1 := &fp{1} fp2 := &fp{2} assert.NotEqual(fp1, fp2) assert.Equal(1.0/1.0, lc.Count(fp1)) assert.Equal(2.0/2.0, lc.Count(fp1)) assert.Equal(1.0/3.0, lc.Count(fp2)) assert.Equal(2.0/4.0, lc.Count(fp2)) assert.Equal(3.0/5.0, lc.Count(fp1)) assert.Equal(4.0/6.0, lc.Count(fp1)) } func TestLossyCounterShouldByLossy(t *testing.T) { assert := assert.New(t) //Create Lossy Counter that will drop items less than 10% frequent lc := NewLossyCounter(metrics, 0.10) fp1 := &fp{1} fp2 := &fp{2} // Count fp1 10 times: 10% for i := 0; i < 10; i++ { assert.Equal(1.0, lc.Count(fp1)) } // Count fp2 90 times: 90% for i := 0; i < 90; i++ { assert.Equal(float64(i+1)/float64(11+i), lc.Count(fp2)) } // Count fp1 once more, should have been dropped and // now is counted only once assert.Equal(1.0/101.0, lc.Count(fp1)) } //Benchmark the worst case scenario for the lossy counter: // every item is errorTolerance frequent func BenchmarkCounting(b *testing.B) { e := 0.01 lc := NewLossyCounter(metrics, 0.01) unique := int(math.Ceil(1.0 / e)) fps := make([]*fp, unique) for i := 0; i < unique; i++ { fps[i] = &fp{i} } b.ResetTimer() for i := 0; i < b.N; i++ { id := i % unique lc.Count(fps[id]) } } ================================================ FILE: detector.go ================================================ package morgoth import ( "log" "sync" "github.com/nathanielc/morgoth/counter" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) type Detector struct { mu sync.RWMutex consensus float64 minSupport float64 errorTolerance float64 counters []fingerprinterCounter metrics *DetectorMetrics } type DetectorMetrics struct { WindowCount prometheus.Counter PointCount prometheus.Counter AnomalousCount prometheus.Counter FingerprinterMetrics []*counter.Metrics } func (m *DetectorMetrics) Register() error { if err := prometheus.Register(m.WindowCount); err != nil { return errors.Wrap(err, "window count metric") } if err := prometheus.Register(m.PointCount); err != nil { return errors.Wrap(err, "point count metric") } if err := prometheus.Register(m.AnomalousCount); err != nil { return errors.Wrap(err, "anomalous count metric") } for i, f := range m.FingerprinterMetrics { if err := f.Register(); err != nil { return errors.Wrapf(err, "fingerprinter %d", i) } } return nil } func (m *DetectorMetrics) Unregister() { prometheus.Unregister(m.WindowCount) prometheus.Unregister(m.PointCount) prometheus.Unregister(m.AnomalousCount) for _, f := range m.FingerprinterMetrics { f.Unregister() } } // Pair of fingerprinter and counter type fingerprinterCounter struct { Fingerprinter counter.Counter } // Create a new Lossy couting based detector // The consensus is a percentage of the fingerprinters that must agree in order to flag a window as anomalous. // If the consensus is -1 then the average support from each fingerprinter is compared to minSupport instead of using a consensus. // The minSupport defines a minimum frequency as a percentage for a window to be considered normal. // The errorTolerance defines a frequency as a precentage for the smallest frequency that will be retained in memory. // The errorTolerance must be less than the minSupport. func NewDetector(metrics *DetectorMetrics, consensus, minSupport, errorTolerance float64, fingerprinters []Fingerprinter) (*Detector, error) { if (consensus != -1 && consensus < 0) || consensus > 1 { return nil, errors.New("consensus must be in the range [0,1) or equal to -1") } if minSupport <= errorTolerance { return nil, errors.New("minSupport must be greater than errorTolerance") } if len(metrics.FingerprinterMetrics) != len(fingerprinters) { return nil, errors.New("must provide the same number of fingerprinter metrics as fingerprinters") } counters := make([]fingerprinterCounter, len(fingerprinters)) for i, fingerprinter := range fingerprinters { counters[i] = fingerprinterCounter{ Fingerprinter: fingerprinter, Counter: counter.NewLossyCounter(metrics.FingerprinterMetrics[i], errorTolerance), } } return &Detector{ metrics: metrics, consensus: consensus, minSupport: minSupport, errorTolerance: errorTolerance, counters: counters, }, nil } // Determine if the window is anomalous func (self *Detector) IsAnomalous(window *Window) (bool, float64) { self.mu.Lock() defer self.mu.Unlock() self.metrics.WindowCount.Inc() self.metrics.PointCount.Add(float64(len(window.Data))) vote := 0.0 avgSupport := 0.0 n := 0.0 for _, fc := range self.counters { fingerprint := fc.Fingerprint(window.Copy()) support := fc.Count(fingerprint) anomalous := support <= self.minSupport if anomalous { vote++ } log.Printf("D! %T anomalous? %v support: %f", fc.Fingerprinter, anomalous, support) avgSupport = ((avgSupport * n) + support) / (n + 1) n++ } anomalous := false if self.consensus != -1 { // Use voting consensus vote /= float64(len(self.counters)) anomalous = vote >= self.consensus } else { // Use average suppport anomalous = avgSupport <= self.minSupport } if anomalous { self.metrics.AnomalousCount.Inc() } return anomalous, avgSupport } func (self *Detector) Close() { self.metrics.Unregister() } ================================================ FILE: fingerprint.go ================================================ package morgoth import "github.com/nathanielc/morgoth/counter" type Fingerprint interface { IsMatch(other counter.Countable) bool } type Fingerprinter interface { Fingerprint(window *Window) Fingerprint } ================================================ FILE: fingerprinters/jsdiv/jsdiv.go ================================================ package jsdiv import ( "math" "github.com/nathanielc/morgoth" "github.com/nathanielc/morgoth/counter" ) const iterations = 20 type histogram map[int]float64 var ln2 = math.Log(2) // Jensen-Shannon Divergence // // Fingerprints store the histogram of the window. // Fingerprints are compared to see their JS divergence distance is less than a critical threshold. // // Configuration: // min: Excpected minimum value of the window data. // max: Excpected maximum value of the window data. // binwidth: Size of a bin for the histogram // pValue: Standard p-value statistical threshold. Typical value is 0.05 type JSDiv struct { minIndex int maxIndex int binWidth float64 pValue float64 } func New(min, max, binWidth, pValue float64) *JSDiv { return &JSDiv{ minIndex: int(math.Floor(min / binWidth)), maxIndex: int(math.Floor(max / binWidth)), binWidth: binWidth, pValue: pValue, } } func (self *JSDiv) Fingerprint(window *morgoth.Window) morgoth.Fingerprint { hist, count := calcHistogram(window.Data, self.binWidth) return &JSDivFingerprint{ hist, count, self.pValue, self.minIndex, self.maxIndex, } } func calcHistogram(xs []float64, binWidth float64) (hist histogram, count int) { count = len(xs) c := float64(count) hist = make(histogram) for _, x := range xs { i := int(math.Floor(x / binWidth)) hist[i] += 1.0 / c } return } type JSDivFingerprint struct { histogram histogram count int pValue float64 minIndex int maxIndex int } func (self *JSDivFingerprint) IsMatch(other counter.Countable) bool { othr, ok := other.(*JSDivFingerprint) if !ok { return false } s := self.calcSignificance(othr) return s < self.pValue } func (self *JSDivFingerprint) calcSignificance(other *JSDivFingerprint) float64 { p := self.histogram q := other.histogram m := make(histogram, len(p)+len(q)) min := self.minIndex max := self.maxIndex for i := range p { if i < min { min = i } if i > max { max = i } m[i] = 0.5 * p[i] } for i := range q { if i < min { min = i } if i > max { max = i } m[i] += 0.5 * q[i] } k := max - min v := 0.5 * float64(k-1) D := calcS(m) - (0.5*calcS(p) + 0.5*calcS(q)) inc := apporxIncompleteGamma(v, float64(self.count+other.count)*ln2*D) gamma := math.Gamma(v) return inc / gamma } // Calculate the Shannon measure for a histogram func calcS(hist histogram) float64 { s := 0.0 for _, v := range hist { if v != 0 { s += v * math.Log2(v) } } return -s } // This is a work in progress. Need to update. func apporxIncompleteGamma(s, x float64) float64 { g := 0.0 xs := math.Pow(x, s) ex := math.Exp(-x) for k := 0; k < iterations; k++ { denominator := s for i := 1; i <= k; i++ { denominator *= s + float64(i) } g += (xs * ex * math.Pow(x, float64(k))) / denominator } return g } ================================================ FILE: fingerprinters/jsdiv/jsdiv_test.go ================================================ package jsdiv ================================================ FILE: fingerprinters/kstest/kstest.go ================================================ package kstest import ( "math" "sort" "github.com/nathanielc/morgoth" "github.com/nathanielc/morgoth/counter" ) var confidenceMappings = []float64{ 1.22, 1.36, 1.48, 1.63, 1.73, 1.95, } // Kolmogorov–Smirnov test. // https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test // // The fingerprint is the cummulative distribution of the window. // The fingerprints are compared by computing the largest distance between the cummulative distribution functions and comparing to a critical value. // // Configuration: // The only parameter is a confidence level. // Valid values are from 0-5. // The level maps to a list of predefined critical values for the KS test. // Increasing 'confidence' decreases the number of anomalies detected. // type KSTest struct { confidence uint } func New(confidence uint) *KSTest { return &KSTest{ confidence: confidence, } } func (self *KSTest) Fingerprint(window *morgoth.Window) morgoth.Fingerprint { sort.Float64s(window.Data) return &KSTestFingerprint{self.confidence, window.Data} } type KSTestFingerprint struct { confidence uint edf []float64 } func (self *KSTestFingerprint) IsMatch(other counter.Countable) bool { othr, ok := other.(*KSTestFingerprint) if !ok { return false } if self.confidence != othr.confidence { return false } threshold := self.calcThreshold(othr) D := calcD(self.edf, othr.edf) return D < threshold } // Calculate the critical threshold for this comparision func (self *KSTestFingerprint) calcThreshold(othr *KSTestFingerprint) float64 { c := confidenceMappings[self.confidence] n := float64(len(self.edf)) m := float64(len(othr.edf)) return c * math.Sqrt((n+m)/(n*m)) } // Calculate maximum distance between cummulative distributions func calcD(f1, f2 []float64) float64 { D := 0.0 n := len(f1) m := len(f2) i := 0 j := 0 for i < n && j < m { for i < n && j < m && f1[i] < f2[j] { i++ } for i < n && j < m && f1[i] > f2[j] { j++ } for i < n && j < m && f1[i] == f2[j] { i++ j++ } cdf1 := float64(i) / float64(n) cdf2 := float64(j) / float64(m) if d := math.Abs(cdf1 - cdf2); d > D { D = d } } return D } ================================================ FILE: fingerprinters/kstest/kstest_test.go ================================================ package kstest import ( "testing" "github.com/nathanielc/morgoth" "github.com/stretchr/testify/assert" ) func TestCalcDShouldBe0(t *testing.T) { assert := assert.New(t) data := make([]float64, 10) for i := range data { data[i] = float64(i+1) / float64(len(data)) } expectedD := 0.0 d := calcD(data, data) assert.InDelta(expectedD, d, 1e-5) } func TestCalcDShouldBeSmall(t *testing.T) { assert := assert.New(t) data1 := make([]float64, 10) for i := range data1 { data1[i] = float64(i+1) / float64(len(data1)) } data2 := make([]float64, 10) for i := range data2 { data2[i] = float64(i) / float64(len(data2)) } expectedD := 0.1 d := calcD(data1, data2) assert.InDelta(expectedD, d, 1e-5) d = calcD(data2, data1) assert.InDelta(expectedD, d, 1e-5) } func TestCalcDShouldBe1(t *testing.T) { assert := assert.New(t) data1 := make([]float64, 10) for i := range data1 { data1[i] = 0.0 } data2 := make([]float64, 10) for i := range data2 { data2[i] = float64(i+1) / float64(len(data2)) } expectedD := 1.0 d := calcD(data1, data2) assert.InDelta(expectedD, d, 1e-5) d = calcD(data2, data1) assert.InDelta(expectedD, d, 1e-5) } func BenchmarkCalcD(b *testing.B) { data := make([]float64, 100) for i := range data { data[i] = float64(i+1) / float64(len(data)) } for i := 0; i < b.N; i++ { calcD(data, data) } } func BenchmarkIsMatch(b *testing.B) { data1 := make([]float64, 100) for i := range data1 { data1[i] = float64(-i) / float64(len(data1)) } data2 := make([]float64, 100) for i := range data2 { data2[i] = float64(i+1) / float64(len(data2)) } ks := KSTest{ confidence: 4, } f1 := ks.Fingerprint(&morgoth.Window{ Data: data1, }) f2 := ks.Fingerprint(&morgoth.Window{ Data: data2, }) for i := 0; i < b.N; i++ { f1.IsMatch(f2) } } func BenchmarkFingerprint(b *testing.B) { data := make([]float64, 100) for i := range data { data[i] = float64(i) / float64(len(data)) } ks := KSTest{ confidence: 4, } w := &morgoth.Window{ Data: data, } for i := 0; i < b.N; i++ { ks.Fingerprint(w) } } ================================================ FILE: fingerprinters/sigma/sigma.go ================================================ package sigma import ( "math" "github.com/nathanielc/morgoth" "github.com/nathanielc/morgoth/counter" ) // Simple fingerprinter that computes both mean and standard deviation of a window. // Fingerprints are compared to see if the means are more than n deviations apart. type Sigma struct { deviations float64 } func New(deviations float64) *Sigma { return &Sigma{ deviations: deviations, } } func (self *Sigma) Fingerprint(window *morgoth.Window) morgoth.Fingerprint { mean, std := calcStats(window.Data) return SigmaFingerprint{ mean: mean, threshold: self.deviations * std, } } func calcStats(xs []float64) (mean, std float64) { n := 0.0 M2 := 0.0 for _, x := range xs { n++ delta := x - mean mean = mean + delta/n M2 += delta * (x - mean) } std = math.Sqrt(M2 / n) return } type SigmaFingerprint struct { mean float64 threshold float64 } func (self SigmaFingerprint) IsMatch(other counter.Countable) bool { o, ok := other.(SigmaFingerprint) if !ok { return false } return math.Abs(self.mean-o.mean) <= self.threshold } ================================================ FILE: vendor/github.com/beorn7/perks/.gitignore ================================================ *.test *.prof ================================================ FILE: vendor/github.com/beorn7/perks/LICENSE ================================================ Copyright (C) 2013 Blake Mizerany Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/beorn7/perks/README.md ================================================ # Perks for Go (golang.org) Perks contains the Go package quantile that computes approximate quantiles over an unbounded data stream within low memory and CPU bounds. For more information and examples, see: http://godoc.org/github.com/bmizerany/perks A very special thank you and shout out to Graham Cormode (Rutgers University), Flip Korn (AT&T Labs–Research), S. Muthukrishnan (Rutgers University), and Divesh Srivastava (AT&T Labs–Research) for their research and publication of [Effective Computation of Biased Quantiles over Data Streams](http://www.cs.rutgers.edu/~muthu/bquant.pdf) Thank you, also: * Armon Dadgar (@armon) * Andrew Gerrand (@nf) * Brad Fitzpatrick (@bradfitz) * Keith Rarick (@kr) FAQ: Q: Why not move the quantile package into the project root? A: I want to add more packages to perks later. Copyright (C) 2013 Blake Mizerany Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/beorn7/perks/quantile/bench_test.go ================================================ package quantile import ( "testing" ) func BenchmarkInsertTargeted(b *testing.B) { b.ReportAllocs() s := NewTargeted(Targets) b.ResetTimer() for i := float64(0); i < float64(b.N); i++ { s.Insert(i) } } func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) { s := NewTargeted(TargetsSmallEpsilon) b.ResetTimer() for i := float64(0); i < float64(b.N); i++ { s.Insert(i) } } func BenchmarkInsertBiased(b *testing.B) { s := NewLowBiased(0.01) b.ResetTimer() for i := float64(0); i < float64(b.N); i++ { s.Insert(i) } } func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) { s := NewLowBiased(0.0001) b.ResetTimer() for i := float64(0); i < float64(b.N); i++ { s.Insert(i) } } func BenchmarkQuery(b *testing.B) { s := NewTargeted(Targets) for i := float64(0); i < 1e6; i++ { s.Insert(i) } b.ResetTimer() n := float64(b.N) for i := float64(0); i < n; i++ { s.Query(i / n) } } func BenchmarkQuerySmallEpsilon(b *testing.B) { s := NewTargeted(TargetsSmallEpsilon) for i := float64(0); i < 1e6; i++ { s.Insert(i) } b.ResetTimer() n := float64(b.N) for i := float64(0); i < n; i++ { s.Query(i / n) } } ================================================ FILE: vendor/github.com/beorn7/perks/quantile/example_test.go ================================================ // +build go1.1 package quantile_test import ( "bufio" "fmt" "log" "os" "strconv" "time" "github.com/beorn7/perks/quantile" ) func Example_simple() { ch := make(chan float64) go sendFloats(ch) // Compute the 50th, 90th, and 99th percentile. q := quantile.NewTargeted(map[float64]float64{ 0.50: 0.005, 0.90: 0.001, 0.99: 0.0001, }) for v := range ch { q.Insert(v) } fmt.Println("perc50:", q.Query(0.50)) fmt.Println("perc90:", q.Query(0.90)) fmt.Println("perc99:", q.Query(0.99)) fmt.Println("count:", q.Count()) // Output: // perc50: 5 // perc90: 16 // perc99: 223 // count: 2388 } func Example_mergeMultipleStreams() { // Scenario: // We have multiple database shards. On each shard, there is a process // collecting query response times from the database logs and inserting // them into a Stream (created via NewTargeted(0.90)), much like the // Simple example. These processes expose a network interface for us to // ask them to serialize and send us the results of their // Stream.Samples so we may Merge and Query them. // // NOTES: // * These sample sets are small, allowing us to get them // across the network much faster than sending the entire list of data // points. // // * For this to work correctly, we must supply the same quantiles // a priori the process collecting the samples supplied to NewTargeted, // even if we do not plan to query them all here. ch := make(chan quantile.Samples) getDBQuerySamples(ch) q := quantile.NewTargeted(map[float64]float64{0.90: 0.001}) for samples := range ch { q.Merge(samples) } fmt.Println("perc90:", q.Query(0.90)) } func Example_window() { // Scenario: We want the 90th, 95th, and 99th percentiles for each // minute. ch := make(chan float64) go sendStreamValues(ch) tick := time.NewTicker(1 * time.Minute) q := quantile.NewTargeted(map[float64]float64{ 0.90: 0.001, 0.95: 0.0005, 0.99: 0.0001, }) for { select { case t := <-tick.C: flushToDB(t, q.Samples()) q.Reset() case v := <-ch: q.Insert(v) } } } func sendStreamValues(ch chan float64) { // Use your imagination } func flushToDB(t time.Time, samples quantile.Samples) { // Use your imagination } // This is a stub for the above example. In reality this would hit the remote // servers via http or something like it. func getDBQuerySamples(ch chan quantile.Samples) {} func sendFloats(ch chan<- float64) { f, err := os.Open("exampledata.txt") if err != nil { log.Fatal(err) } sc := bufio.NewScanner(f) for sc.Scan() { b := sc.Bytes() v, err := strconv.ParseFloat(string(b), 64) if err != nil { log.Fatal(err) } ch <- v } if sc.Err() != nil { log.Fatal(sc.Err()) } close(ch) } ================================================ FILE: vendor/github.com/beorn7/perks/quantile/exampledata.txt ================================================ 8 5 26 12 5 235 13 6 28 30 3 3 3 3 5 2 33 7 2 4 7 12 14 5 8 3 10 4 5 3 6 6 209 20 3 10 14 3 4 6 8 5 11 7 3 2 3 3 212 5 222 4 10 10 5 6 3 8 3 10 254 220 2 3 5 24 5 4 222 7 3 3 223 8 15 12 14 14 3 2 2 3 13 3 11 4 4 6 5 7 13 5 3 5 2 5 3 5 2 7 15 17 14 3 6 6 3 17 5 4 7 6 4 4 8 6 8 3 9 3 6 3 4 5 3 3 660 4 6 10 3 6 3 2 5 13 2 4 4 10 4 8 4 3 7 9 9 3 10 37 3 13 4 12 3 6 10 8 5 21 2 3 8 3 2 3 3 4 12 2 4 8 8 4 3 2 20 1 6 32 2 11 6 18 3 8 11 3 212 3 4 2 6 7 12 11 3 2 16 10 6 4 6 3 2 7 3 2 2 2 2 5 6 4 3 10 3 4 6 5 3 4 4 5 6 4 3 4 4 5 7 5 5 3 2 7 2 4 12 4 5 6 2 4 4 8 4 15 13 7 16 5 3 23 5 5 7 3 2 9 8 7 5 8 11 4 10 76 4 47 4 3 2 7 4 2 3 37 10 4 2 20 5 4 4 10 10 4 3 7 23 240 7 13 5 5 3 3 2 5 4 2 8 7 19 2 23 8 7 2 5 3 8 3 8 13 5 5 5 2 3 23 4 9 8 4 3 3 5 220 2 3 4 6 14 3 53 6 2 5 18 6 3 219 6 5 2 5 3 6 5 15 4 3 17 3 2 4 7 2 3 3 4 4 3 2 664 6 3 23 5 5 16 5 8 2 4 2 24 12 3 2 3 5 8 3 5 4 3 14 3 5 8 2 3 7 9 4 2 3 6 8 4 3 4 6 5 3 3 6 3 19 4 4 6 3 6 3 5 22 5 4 4 3 8 11 4 9 7 6 13 4 4 4 6 17 9 3 3 3 4 3 221 5 11 3 4 2 12 6 3 5 7 5 7 4 9 7 14 37 19 217 16 3 5 2 2 7 19 7 6 7 4 24 5 11 4 7 7 9 13 3 4 3 6 28 4 4 5 5 2 5 6 4 4 6 10 5 4 3 2 3 3 6 5 5 4 3 2 3 7 4 6 18 16 8 16 4 5 8 6 9 13 1545 6 215 6 5 6 3 45 31 5 2 2 4 3 3 2 5 4 3 5 7 7 4 5 8 5 4 749 2 31 9 11 2 11 5 4 4 7 9 11 4 5 4 7 3 4 6 2 15 3 4 3 4 3 5 2 13 5 5 3 3 23 4 4 5 7 4 13 2 4 3 4 2 6 2 7 3 5 5 3 29 5 4 4 3 10 2 3 79 16 6 6 7 7 3 5 5 7 4 3 7 9 5 6 5 9 6 3 6 4 17 2 10 9 3 6 2 3 21 22 5 11 4 2 17 2 224 2 14 3 4 4 2 4 4 4 4 5 3 4 4 10 2 6 3 3 5 7 2 7 5 6 3 218 2 2 5 2 6 3 5 222 14 6 33 3 2 5 3 3 3 9 5 3 3 2 7 4 3 4 3 5 6 5 26 4 13 9 7 3 221 3 3 4 4 4 4 2 18 5 3 7 9 6 8 3 10 3 11 9 5 4 17 5 5 6 6 3 2 4 12 17 6 7 218 4 2 4 10 3 5 15 3 9 4 3 3 6 29 3 3 4 5 5 3 8 5 6 6 7 5 3 5 3 29 2 31 5 15 24 16 5 207 4 3 3 2 15 4 4 13 5 5 4 6 10 2 7 8 4 6 20 5 3 4 3 12 12 5 17 7 3 3 3 6 10 3 5 25 80 4 9 3 2 11 3 3 2 3 8 7 5 5 19 5 3 3 12 11 2 6 5 5 5 3 3 3 4 209 14 3 2 5 19 4 4 3 4 14 5 6 4 13 9 7 4 7 10 2 9 5 7 2 8 4 6 5 5 222 8 7 12 5 216 3 4 4 6 3 14 8 7 13 4 3 3 3 3 17 5 4 3 33 6 6 33 7 5 3 8 7 5 2 9 4 2 233 24 7 4 8 10 3 4 15 2 16 3 3 13 12 7 5 4 207 4 2 4 27 15 2 5 2 25 6 5 5 6 13 6 18 6 4 12 225 10 7 5 2 2 11 4 14 21 8 10 3 5 4 232 2 5 5 3 7 17 11 6 6 23 4 6 3 5 4 2 17 3 6 5 8 3 2 2 14 9 4 4 2 5 5 3 7 6 12 6 10 3 6 2 2 19 5 4 4 9 2 4 13 3 5 6 3 6 5 4 9 6 3 5 7 3 6 6 4 3 10 6 3 221 3 5 3 6 4 8 5 3 6 4 4 2 54 5 6 11 3 3 4 4 4 3 7 3 11 11 7 10 6 13 223 213 15 231 7 3 7 228 2 3 4 4 5 6 7 4 13 3 4 5 3 6 4 6 7 2 4 3 4 3 3 6 3 7 3 5 18 5 6 8 10 3 3 3 2 4 2 4 4 5 6 6 4 10 13 3 12 5 12 16 8 4 19 11 2 4 5 6 8 5 6 4 18 10 4 2 216 6 6 6 2 4 12 8 3 11 5 6 14 5 3 13 4 5 4 5 3 28 6 3 7 219 3 9 7 3 10 6 3 4 19 5 7 11 6 15 19 4 13 11 3 7 5 10 2 8 11 2 6 4 6 24 6 3 3 3 3 6 18 4 11 4 2 5 10 8 3 9 5 3 4 5 6 2 5 7 4 4 14 6 4 4 5 5 7 2 4 3 7 3 3 6 4 5 4 4 4 3 3 3 3 8 14 2 3 5 3 2 4 5 3 7 3 3 18 3 4 4 5 7 3 3 3 13 5 4 8 211 5 5 3 5 2 5 4 2 655 6 3 5 11 2 5 3 12 9 15 11 5 12 217 2 6 17 3 3 207 5 5 4 5 9 3 2 8 5 4 3 2 5 12 4 14 5 4 2 13 5 8 4 225 4 3 4 5 4 3 3 6 23 9 2 6 7 233 4 4 6 18 3 4 6 3 4 4 2 3 7 4 13 227 4 3 5 4 2 12 9 17 3 7 14 6 4 5 21 4 8 9 2 9 25 16 3 6 4 7 8 5 2 3 5 4 3 3 5 3 3 3 2 3 19 2 4 3 4 2 3 4 4 2 4 3 3 3 2 6 3 17 5 6 4 3 13 5 3 3 3 4 9 4 2 14 12 4 5 24 4 3 37 12 11 21 3 4 3 13 4 2 3 15 4 11 4 4 3 8 3 4 4 12 8 5 3 3 4 2 220 3 5 223 3 3 3 10 3 15 4 241 9 7 3 6 6 23 4 13 7 3 4 7 4 9 3 3 4 10 5 5 1 5 24 2 4 5 5 6 14 3 8 2 3 5 13 13 3 5 2 3 15 3 4 2 10 4 4 4 5 5 3 5 3 4 7 4 27 3 6 4 15 3 5 6 6 5 4 8 3 9 2 6 3 4 3 7 4 18 3 11 3 3 8 9 7 24 3 219 7 10 4 5 9 12 2 5 4 4 4 3 3 19 5 8 16 8 6 22 3 23 3 242 9 4 3 3 5 7 3 3 5 8 3 7 5 14 8 10 3 4 3 7 4 6 7 4 10 4 3 11 3 7 10 3 13 6 8 12 10 5 7 9 3 4 7 7 10 8 30 9 19 4 3 19 15 4 13 3 215 223 4 7 4 8 17 16 3 7 6 5 5 4 12 3 7 4 4 13 4 5 2 5 6 5 6 6 7 10 18 23 9 3 3 6 5 2 4 2 7 3 3 2 5 5 14 10 224 6 3 4 3 7 5 9 3 6 4 2 5 11 4 3 3 2 8 4 7 4 10 7 3 3 18 18 17 3 3 3 4 5 3 3 4 12 7 3 11 13 5 4 7 13 5 4 11 3 12 3 6 4 4 21 4 6 9 5 3 10 8 4 6 4 4 6 5 4 8 6 4 6 4 4 5 9 6 3 4 2 9 3 18 2 4 3 13 3 6 6 8 7 9 3 2 16 3 4 6 3 2 33 22 14 4 9 12 4 5 6 3 23 9 4 3 5 5 3 4 5 3 5 3 10 4 5 5 8 4 4 6 8 5 4 3 4 6 3 3 3 5 9 12 6 5 9 3 5 3 2 2 2 18 3 2 21 2 5 4 6 4 5 10 3 9 3 2 10 7 3 6 6 4 4 8 12 7 3 7 3 3 9 3 4 5 4 4 5 5 10 15 4 4 14 6 227 3 14 5 216 22 5 4 2 2 6 3 4 2 9 9 4 3 28 13 11 4 5 3 3 2 3 3 5 3 4 3 5 23 26 3 4 5 6 4 6 3 5 5 3 4 3 2 2 2 7 14 3 6 7 17 2 2 15 14 16 4 6 7 13 6 4 5 6 16 3 3 28 3 6 15 3 9 2 4 6 3 3 22 4 12 6 7 2 5 4 10 3 16 6 9 2 5 12 7 5 5 5 5 2 11 9 17 4 3 11 7 3 5 15 4 3 4 211 8 7 5 4 7 6 7 6 3 6 5 6 5 3 4 4 26 4 6 10 4 4 3 2 3 3 4 5 9 3 9 4 4 5 5 8 2 4 2 3 8 4 11 19 5 8 6 3 5 6 12 3 2 4 16 12 3 4 4 8 6 5 6 6 219 8 222 6 16 3 13 19 5 4 3 11 6 10 4 7 7 12 5 3 3 5 6 10 3 8 2 5 4 7 2 4 4 2 12 9 6 4 2 40 2 4 10 4 223 4 2 20 6 7 24 5 4 5 2 20 16 6 5 13 2 3 3 19 3 2 4 5 6 7 11 12 5 6 7 7 3 5 3 5 3 14 3 4 4 2 11 1 7 3 9 6 11 12 5 8 6 221 4 2 12 4 3 15 4 5 226 7 218 7 5 4 5 18 4 5 9 4 4 2 9 18 18 9 5 6 6 3 3 7 3 5 4 4 4 12 3 6 31 5 4 7 3 6 5 6 5 11 2 2 11 11 6 7 5 8 7 10 5 23 7 4 3 5 34 2 5 23 7 3 6 8 4 4 4 2 5 3 8 5 4 8 25 2 3 17 8 3 4 8 7 3 15 6 5 7 21 9 5 6 6 5 3 2 3 10 3 6 3 14 7 4 4 8 7 8 2 6 12 4 213 6 5 21 8 2 5 23 3 11 2 3 6 25 2 3 6 7 6 6 4 4 6 3 17 9 7 6 4 3 10 7 2 3 3 3 11 8 3 7 6 4 14 36 3 4 3 3 22 13 21 4 2 7 4 4 17 15 3 7 11 2 4 7 6 209 6 3 2 2 24 4 9 4 3 3 3 29 2 2 4 3 3 5 4 6 3 3 2 4 ================================================ FILE: vendor/github.com/beorn7/perks/quantile/stream.go ================================================ // Package quantile computes approximate quantiles over an unbounded data // stream within low memory and CPU bounds. // // A small amount of accuracy is traded to achieve the above properties. // // Multiple streams can be merged before calling Query to generate a single set // of results. This is meaningful when the streams represent the same type of // data. See Merge and Samples. // // For more detailed information about the algorithm used, see: // // Effective Computation of Biased Quantiles over Data Streams // // http://www.cs.rutgers.edu/~muthu/bquant.pdf package quantile import ( "math" "sort" ) // Sample holds an observed value and meta information for compression. JSON // tags have been added for convenience. type Sample struct { Value float64 `json:",string"` Width float64 `json:",string"` Delta float64 `json:",string"` } // Samples represents a slice of samples. It implements sort.Interface. type Samples []Sample func (a Samples) Len() int { return len(a) } func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value } func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] } type invariant func(s *stream, r float64) float64 // NewLowBiased returns an initialized Stream for low-biased quantiles // (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but // error guarantees can still be given even for the lower ranks of the data // distribution. // // The provided epsilon is a relative error, i.e. the true quantile of a value // returned by a query is guaranteed to be within (1±Epsilon)*Quantile. // // See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error // properties. func NewLowBiased(epsilon float64) *Stream { ƒ := func(s *stream, r float64) float64 { return 2 * epsilon * r } return newStream(ƒ) } // NewHighBiased returns an initialized Stream for high-biased quantiles // (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but // error guarantees can still be given even for the higher ranks of the data // distribution. // // The provided epsilon is a relative error, i.e. the true quantile of a value // returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile). // // See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error // properties. func NewHighBiased(epsilon float64) *Stream { ƒ := func(s *stream, r float64) float64 { return 2 * epsilon * (s.n - r) } return newStream(ƒ) } // NewTargeted returns an initialized Stream concerned with a particular set of // quantile values that are supplied a priori. Knowing these a priori reduces // space and computation time. The targets map maps the desired quantiles to // their absolute errors, i.e. the true quantile of a value returned by a query // is guaranteed to be within (Quantile±Epsilon). // // See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties. func NewTargeted(targets map[float64]float64) *Stream { ƒ := func(s *stream, r float64) float64 { var m = math.MaxFloat64 var f float64 for quantile, epsilon := range targets { if quantile*s.n <= r { f = (2 * epsilon * r) / quantile } else { f = (2 * epsilon * (s.n - r)) / (1 - quantile) } if f < m { m = f } } return m } return newStream(ƒ) } // Stream computes quantiles for a stream of float64s. It is not thread-safe by // design. Take care when using across multiple goroutines. type Stream struct { *stream b Samples sorted bool } func newStream(ƒ invariant) *Stream { x := &stream{ƒ: ƒ} return &Stream{x, make(Samples, 0, 500), true} } // Insert inserts v into the stream. func (s *Stream) Insert(v float64) { s.insert(Sample{Value: v, Width: 1}) } func (s *Stream) insert(sample Sample) { s.b = append(s.b, sample) s.sorted = false if len(s.b) == cap(s.b) { s.flush() } } // Query returns the computed qth percentiles value. If s was created with // NewTargeted, and q is not in the set of quantiles provided a priori, Query // will return an unspecified result. func (s *Stream) Query(q float64) float64 { if !s.flushed() { // Fast path when there hasn't been enough data for a flush; // this also yields better accuracy for small sets of data. l := len(s.b) if l == 0 { return 0 } i := int(math.Ceil(float64(l) * q)) if i > 0 { i -= 1 } s.maybeSort() return s.b[i].Value } s.flush() return s.stream.query(q) } // Merge merges samples into the underlying streams samples. This is handy when // merging multiple streams from separate threads, database shards, etc. // // ATTENTION: This method is broken and does not yield correct results. The // underlying algorithm is not capable of merging streams correctly. func (s *Stream) Merge(samples Samples) { sort.Sort(samples) s.stream.merge(samples) } // Reset reinitializes and clears the list reusing the samples buffer memory. func (s *Stream) Reset() { s.stream.reset() s.b = s.b[:0] } // Samples returns stream samples held by s. func (s *Stream) Samples() Samples { if !s.flushed() { return s.b } s.flush() return s.stream.samples() } // Count returns the total number of samples observed in the stream // since initialization. func (s *Stream) Count() int { return len(s.b) + s.stream.count() } func (s *Stream) flush() { s.maybeSort() s.stream.merge(s.b) s.b = s.b[:0] } func (s *Stream) maybeSort() { if !s.sorted { s.sorted = true sort.Sort(s.b) } } func (s *Stream) flushed() bool { return len(s.stream.l) > 0 } type stream struct { n float64 l []Sample ƒ invariant } func (s *stream) reset() { s.l = s.l[:0] s.n = 0 } func (s *stream) insert(v float64) { s.merge(Samples{{v, 1, 0}}) } func (s *stream) merge(samples Samples) { // TODO(beorn7): This tries to merge not only individual samples, but // whole summaries. The paper doesn't mention merging summaries at // all. Unittests show that the merging is inaccurate. Find out how to // do merges properly. var r float64 i := 0 for _, sample := range samples { for ; i < len(s.l); i++ { c := s.l[i] if c.Value > sample.Value { // Insert at position i. s.l = append(s.l, Sample{}) copy(s.l[i+1:], s.l[i:]) s.l[i] = Sample{ sample.Value, sample.Width, math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1), // TODO(beorn7): How to calculate delta correctly? } i++ goto inserted } r += c.Width } s.l = append(s.l, Sample{sample.Value, sample.Width, 0}) i++ inserted: s.n += sample.Width r += sample.Width } s.compress() } func (s *stream) count() int { return int(s.n) } func (s *stream) query(q float64) float64 { t := math.Ceil(q * s.n) t += math.Ceil(s.ƒ(s, t) / 2) p := s.l[0] var r float64 for _, c := range s.l[1:] { r += p.Width if r+c.Width+c.Delta > t { return p.Value } p = c } return p.Value } func (s *stream) compress() { if len(s.l) < 2 { return } x := s.l[len(s.l)-1] xi := len(s.l) - 1 r := s.n - 1 - x.Width for i := len(s.l) - 2; i >= 0; i-- { c := s.l[i] if c.Width+x.Width+x.Delta <= s.ƒ(s, r) { x.Width += c.Width s.l[xi] = x // Remove element at i. copy(s.l[i:], s.l[i+1:]) s.l = s.l[:len(s.l)-1] xi -= 1 } else { x = c xi = i } r -= c.Width } } func (s *stream) samples() Samples { samples := make(Samples, len(s.l)) copy(samples, s.l) return samples } ================================================ FILE: vendor/github.com/beorn7/perks/quantile/stream_test.go ================================================ package quantile import ( "math" "math/rand" "sort" "testing" ) var ( Targets = map[float64]float64{ 0.01: 0.001, 0.10: 0.01, 0.50: 0.05, 0.90: 0.01, 0.99: 0.001, } TargetsSmallEpsilon = map[float64]float64{ 0.01: 0.0001, 0.10: 0.001, 0.50: 0.005, 0.90: 0.001, 0.99: 0.0001, } LowQuantiles = []float64{0.01, 0.1, 0.5} HighQuantiles = []float64{0.99, 0.9, 0.5} ) const RelativeEpsilon = 0.01 func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) { sort.Float64s(a) for quantile, epsilon := range Targets { n := float64(len(a)) k := int(quantile * n) if k < 1 { k = 1 } lower := int((quantile - epsilon) * n) if lower < 1 { lower = 1 } upper := int(math.Ceil((quantile + epsilon) * n)) if upper > len(a) { upper = len(a) } w, min, max := a[k-1], a[lower-1], a[upper-1] if g := s.Query(quantile); g < min || g > max { t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g) } } } func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) { sort.Float64s(a) for _, qu := range LowQuantiles { n := float64(len(a)) k := int(qu * n) lowerRank := int((1 - RelativeEpsilon) * qu * n) upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n)) w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1] if g := s.Query(qu); g < min || g > max { t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g) } } } func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) { sort.Float64s(a) for _, qu := range HighQuantiles { n := float64(len(a)) k := int(qu * n) lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n) upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n)) w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1] if g := s.Query(qu); g < min || g > max { t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g) } } } func populateStream(s *Stream) []float64 { a := make([]float64, 0, 1e5+100) for i := 0; i < cap(a); i++ { v := rand.NormFloat64() // Add 5% asymmetric outliers. if i%20 == 0 { v = v*v + 1 } s.Insert(v) a = append(a, v) } return a } func TestTargetedQuery(t *testing.T) { rand.Seed(42) s := NewTargeted(Targets) a := populateStream(s) verifyPercsWithAbsoluteEpsilon(t, a, s) } func TestTargetedQuerySmallSampleSize(t *testing.T) { rand.Seed(42) s := NewTargeted(TargetsSmallEpsilon) a := []float64{1, 2, 3, 4, 5} for _, v := range a { s.Insert(v) } verifyPercsWithAbsoluteEpsilon(t, a, s) // If not yet flushed, results should be precise: if !s.flushed() { for φ, want := range map[float64]float64{ 0.01: 1, 0.10: 1, 0.50: 3, 0.90: 5, 0.99: 5, } { if got := s.Query(φ); got != want { t.Errorf("want %f for φ=%f, got %f", want, φ, got) } } } } func TestLowBiasedQuery(t *testing.T) { rand.Seed(42) s := NewLowBiased(RelativeEpsilon) a := populateStream(s) verifyLowPercsWithRelativeEpsilon(t, a, s) } func TestHighBiasedQuery(t *testing.T) { rand.Seed(42) s := NewHighBiased(RelativeEpsilon) a := populateStream(s) verifyHighPercsWithRelativeEpsilon(t, a, s) } // BrokenTestTargetedMerge is broken, see Merge doc comment. func BrokenTestTargetedMerge(t *testing.T) { rand.Seed(42) s1 := NewTargeted(Targets) s2 := NewTargeted(Targets) a := populateStream(s1) a = append(a, populateStream(s2)...) s1.Merge(s2.Samples()) verifyPercsWithAbsoluteEpsilon(t, a, s1) } // BrokenTestLowBiasedMerge is broken, see Merge doc comment. func BrokenTestLowBiasedMerge(t *testing.T) { rand.Seed(42) s1 := NewLowBiased(RelativeEpsilon) s2 := NewLowBiased(RelativeEpsilon) a := populateStream(s1) a = append(a, populateStream(s2)...) s1.Merge(s2.Samples()) verifyLowPercsWithRelativeEpsilon(t, a, s2) } // BrokenTestHighBiasedMerge is broken, see Merge doc comment. func BrokenTestHighBiasedMerge(t *testing.T) { rand.Seed(42) s1 := NewHighBiased(RelativeEpsilon) s2 := NewHighBiased(RelativeEpsilon) a := populateStream(s1) a = append(a, populateStream(s2)...) s1.Merge(s2.Samples()) verifyHighPercsWithRelativeEpsilon(t, a, s2) } func TestUncompressed(t *testing.T) { q := NewTargeted(Targets) for i := 100; i > 0; i-- { q.Insert(float64(i)) } if g := q.Count(); g != 100 { t.Errorf("want count 100, got %d", g) } // Before compression, Query should have 100% accuracy. for quantile := range Targets { w := quantile * 100 if g := q.Query(quantile); g != w { t.Errorf("want %f, got %f", w, g) } } } func TestUncompressedSamples(t *testing.T) { q := NewTargeted(map[float64]float64{0.99: 0.001}) for i := 1; i <= 100; i++ { q.Insert(float64(i)) } if g := q.Samples().Len(); g != 100 { t.Errorf("want count 100, got %d", g) } } func TestUncompressedOne(t *testing.T) { q := NewTargeted(map[float64]float64{0.99: 0.01}) q.Insert(3.14) if g := q.Query(0.90); g != 3.14 { t.Error("want PI, got", g) } } func TestDefaults(t *testing.T) { if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 { t.Errorf("want 0, got %f", g) } } ================================================ FILE: vendor/github.com/davecgh/go-spew/.gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe ================================================ FILE: vendor/github.com/davecgh/go-spew/.travis.yml ================================================ language: go go: - 1.5.4 - 1.6.3 - 1.7 install: - go get -v golang.org/x/tools/cmd/cover script: - go test -v -tags=safe ./spew - go test -v -tags=testcgo ./spew -covermode=count -coverprofile=profile.cov after_success: - go get -v github.com/mattn/goveralls - export PATH=$PATH:$HOME/gopath/bin - goveralls -coverprofile=profile.cov -service=travis-ci ================================================ FILE: vendor/github.com/davecgh/go-spew/LICENSE ================================================ ISC License Copyright (c) 2012-2016 Dave Collins Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ================================================ FILE: vendor/github.com/davecgh/go-spew/README.md ================================================ go-spew ======= [![Build Status](https://img.shields.io/travis/davecgh/go-spew.svg)] (https://travis-ci.org/davecgh/go-spew) [![ISC License] (http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) [![Coverage Status] (https://img.shields.io/coveralls/davecgh/go-spew.svg)] (https://coveralls.io/r/davecgh/go-spew?branch=master) Go-spew implements a deep pretty printer for Go data structures to aid in debugging. A comprehensive suite of tests with 100% test coverage is provided to ensure proper functionality. See `test_coverage.txt` for the gocov coverage report. Go-spew is licensed under the liberal ISC license, so it may be used in open source or commercial projects. If you're interested in reading about how this package came to life and some of the challenges involved in providing a deep pretty printer, there is a blog post about it [here](https://web.archive.org/web/20160304013555/https://blog.cyphertite.com/go-spew-a-journey-into-dumping-go-data-structures/). ## Documentation [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)] (http://godoc.org/github.com/davecgh/go-spew/spew) Full `go doc` style documentation for the project can be viewed online without installing this package by using the excellent GoDoc site here: http://godoc.org/github.com/davecgh/go-spew/spew You can also view the documentation locally once the package is installed with the `godoc` tool by running `godoc -http=":6060"` and pointing your browser to http://localhost:6060/pkg/github.com/davecgh/go-spew/spew ## Installation ```bash $ go get -u github.com/davecgh/go-spew/spew ``` ## Quick Start Add this import line to the file you're working in: ```Go import "github.com/davecgh/go-spew/spew" ``` To dump a variable with full newlines, indentation, type, and pointer information use Dump, Fdump, or Sdump: ```Go spew.Dump(myVar1, myVar2, ...) spew.Fdump(someWriter, myVar1, myVar2, ...) str := spew.Sdump(myVar1, myVar2, ...) ``` Alternatively, if you would prefer to use format strings with a compacted inline printing style, use the convenience wrappers Printf, Fprintf, etc with %v (most compact), %+v (adds pointer addresses), %#v (adds types), or %#+v (adds types and pointer addresses): ```Go spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) ``` ## Debugging a Web Application Example Here is an example of how you can use `spew.Sdump()` to help debug a web application. Please be sure to wrap your output using the `html.EscapeString()` function for safety reasons. You should also only use this debugging technique in a development environment, never in production. ```Go package main import ( "fmt" "html" "net/http" "github.com/davecgh/go-spew/spew" ) func handler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") fmt.Fprintf(w, "Hi there, %s!", r.URL.Path[1:]) fmt.Fprintf(w, "") } func main() { http.HandleFunc("/", handler) http.ListenAndServe(":8080", nil) } ``` ## Sample Dump Output ``` (main.Foo) { unexportedField: (*main.Bar)(0xf84002e210)({ flag: (main.Flag) flagTwo, data: (uintptr) }), ExportedField: (map[interface {}]interface {}) { (string) "one": (bool) true } } ([]uint8) { 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| 00000020 31 32 |12| } ``` ## Sample Formatter Output Double pointer to a uint8: ``` %v: <**>5 %+v: <**>(0xf8400420d0->0xf8400420c8)5 %#v: (**uint8)5 %#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5 ``` Pointer to circular struct with a uint8 field and a pointer to itself: ``` %v: <*>{1 <*>} %+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)} %#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)} %#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)} ``` ## Configuration Options Configuration of spew is handled by fields in the ConfigState type. For convenience, all of the top-level functions use a global state available via the spew.Config global. It is also possible to create a ConfigState instance that provides methods equivalent to the top-level functions. This allows concurrent configuration options. See the ConfigState documentation for more details. ``` * Indent String to use for each indentation level for Dump functions. It is a single space by default. A popular alternative is "\t". * MaxDepth Maximum number of levels to descend into nested data structures. There is no limit by default. * DisableMethods Disables invocation of error and Stringer interface methods. Method invocation is enabled by default. * DisablePointerMethods Disables invocation of error and Stringer interface methods on types which only accept pointer receivers from non-pointer variables. This option relies on access to the unsafe package, so it will not have any effect when running in environments without access to the unsafe package such as Google App Engine or with the "safe" build tag specified. Pointer method invocation is enabled by default. * DisablePointerAddresses DisablePointerAddresses specifies whether to disable the printing of pointer addresses. This is useful when diffing data structures in tests. * DisableCapacities DisableCapacities specifies whether to disable the printing of capacities for arrays, slices, maps and channels. This is useful when diffing data structures in tests. * ContinueOnMethod Enables recursion into types after invoking error and Stringer interface methods. Recursion after method invocation is disabled by default. * SortKeys Specifies map keys should be sorted before being printed. Use this to have a more deterministic, diffable output. Note that only native types (bool, int, uint, floats, uintptr and string) and types which implement error or Stringer interfaces are supported, with other types sorted according to the reflect.Value.String() output which guarantees display stability. Natural map order is used by default. * SpewKeys SpewKeys specifies that, as a last resort attempt, map keys should be spewed to strings and sorted by those strings. This is only considered if SortKeys is true. ``` ## Unsafe Package Dependency This package relies on the unsafe package to perform some of the more advanced features, however it also supports a "limited" mode which allows it to work in environments where the unsafe package is not available. By default, it will operate in this mode on Google App Engine and when compiled with GopherJS. The "safe" build tag may also be specified to force the package to build without using the unsafe package. ## License Go-spew is licensed under the [copyfree](http://copyfree.org) ISC License. ================================================ FILE: vendor/github.com/davecgh/go-spew/cov_report.sh ================================================ #!/bin/sh # This script uses gocov to generate a test coverage report. # The gocov tool my be obtained with the following command: # go get github.com/axw/gocov/gocov # # It will be installed to $GOPATH/bin, so ensure that location is in your $PATH. # Check for gocov. if ! type gocov >/dev/null 2>&1; then echo >&2 "This script requires the gocov tool." echo >&2 "You may obtain it with the following command:" echo >&2 "go get github.com/axw/gocov/gocov" exit 1 fi # Only run the cgo tests if gcc is installed. if type gcc >/dev/null 2>&1; then (cd spew && gocov test -tags testcgo | gocov report) else (cd spew && gocov test | gocov report) fi ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/bypass.go ================================================ // Copyright (c) 2015-2016 Dave Collins // // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled // when the code is not running on Google App Engine, compiled by GopherJS, and // "-tags safe" is not added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. // +build !js,!appengine,!safe,!disableunsafe package spew import ( "reflect" "unsafe" ) const ( // UnsafeDisabled is a build-time constant which specifies whether or // not access to the unsafe package is available. UnsafeDisabled = false // ptrSize is the size of a pointer on the current arch. ptrSize = unsafe.Sizeof((*byte)(nil)) ) var ( // offsetPtr, offsetScalar, and offsetFlag are the offsets for the // internal reflect.Value fields. These values are valid before golang // commit ecccf07e7f9d which changed the format. The are also valid // after commit 82f48826c6c7 which changed the format again to mirror // the original format. Code in the init function updates these offsets // as necessary. offsetPtr = uintptr(ptrSize) offsetScalar = uintptr(0) offsetFlag = uintptr(ptrSize * 2) // flagKindWidth and flagKindShift indicate various bits that the // reflect package uses internally to track kind information. // // flagRO indicates whether or not the value field of a reflect.Value is // read-only. // // flagIndir indicates whether the value field of a reflect.Value is // the actual data or a pointer to the data. // // These values are valid before golang commit 90a7c3c86944 which // changed their positions. Code in the init function updates these // flags as necessary. flagKindWidth = uintptr(5) flagKindShift = uintptr(flagKindWidth - 1) flagRO = uintptr(1 << 0) flagIndir = uintptr(1 << 1) ) func init() { // Older versions of reflect.Value stored small integers directly in the // ptr field (which is named val in the older versions). Versions // between commits ecccf07e7f9d and 82f48826c6c7 added a new field named // scalar for this purpose which unfortunately came before the flag // field, so the offset of the flag field is different for those // versions. // // This code constructs a new reflect.Value from a known small integer // and checks if the size of the reflect.Value struct indicates it has // the scalar field. When it does, the offsets are updated accordingly. vv := reflect.ValueOf(0xf00) if unsafe.Sizeof(vv) == (ptrSize * 4) { offsetScalar = ptrSize * 2 offsetFlag = ptrSize * 3 } // Commit 90a7c3c86944 changed the flag positions such that the low // order bits are the kind. This code extracts the kind from the flags // field and ensures it's the correct type. When it's not, the flag // order has been changed to the newer format, so the flags are updated // accordingly. upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag) upfv := *(*uintptr)(upf) flagKindMask := uintptr((1<>flagKindShift != uintptr(reflect.Int) { flagKindShift = 0 flagRO = 1 << 5 flagIndir = 1 << 6 // Commit adf9b30e5594 modified the flags to separate the // flagRO flag into two bits which specifies whether or not the // field is embedded. This causes flagIndir to move over a bit // and means that flagRO is the combination of either of the // original flagRO bit and the new bit. // // This code detects the change by extracting what used to be // the indirect bit to ensure it's set. When it's not, the flag // order has been changed to the newer format, so the flags are // updated accordingly. if upfv&flagIndir == 0 { flagRO = 3 << 5 flagIndir = 1 << 7 } } } // unsafeReflectValue converts the passed reflect.Value into a one that bypasses // the typical safety restrictions preventing access to unaddressable and // unexported data. It works by digging the raw pointer to the underlying // value out of the protected value and generating a new unprotected (unsafe) // reflect.Value to it. // // This allows us to check for implementations of the Stringer and error // interfaces to be used for pretty printing ordinarily unaddressable and // inaccessible values such as unexported struct fields. func unsafeReflectValue(v reflect.Value) (rv reflect.Value) { indirects := 1 vt := v.Type() upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr) rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag)) if rvf&flagIndir != 0 { vt = reflect.PtrTo(v.Type()) indirects++ } else if offsetScalar != 0 { // The value is in the scalar field when it's not one of the // reference types. switch vt.Kind() { case reflect.Uintptr: case reflect.Chan: case reflect.Func: case reflect.Map: case reflect.Ptr: case reflect.UnsafePointer: default: upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetScalar) } } pv := reflect.NewAt(vt, upv) rv = pv for i := 0; i < indirects; i++ { rv = rv.Elem() } return rv } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/bypasssafe.go ================================================ // Copyright (c) 2015-2016 Dave Collins // // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled // when the code is running on Google App Engine, compiled by GopherJS, or // "-tags safe" is added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. // +build js appengine safe disableunsafe package spew import "reflect" const ( // UnsafeDisabled is a build-time constant which specifies whether or // not access to the unsafe package is available. UnsafeDisabled = true ) // unsafeReflectValue typically converts the passed reflect.Value into a one // that bypasses the typical safety restrictions preventing access to // unaddressable and unexported data. However, doing this relies on access to // the unsafe package. This is a stub version which simply returns the passed // reflect.Value when the unsafe package is not available. func unsafeReflectValue(v reflect.Value) reflect.Value { return v } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/common.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew import ( "bytes" "fmt" "io" "reflect" "sort" "strconv" ) // Some constants in the form of bytes to avoid string overhead. This mirrors // the technique used in the fmt package. var ( panicBytes = []byte("(PANIC=") plusBytes = []byte("+") iBytes = []byte("i") trueBytes = []byte("true") falseBytes = []byte("false") interfaceBytes = []byte("(interface {})") commaNewlineBytes = []byte(",\n") newlineBytes = []byte("\n") openBraceBytes = []byte("{") openBraceNewlineBytes = []byte("{\n") closeBraceBytes = []byte("}") asteriskBytes = []byte("*") colonBytes = []byte(":") colonSpaceBytes = []byte(": ") openParenBytes = []byte("(") closeParenBytes = []byte(")") spaceBytes = []byte(" ") pointerChainBytes = []byte("->") nilAngleBytes = []byte("") maxNewlineBytes = []byte("\n") maxShortBytes = []byte("") circularBytes = []byte("") circularShortBytes = []byte("") invalidAngleBytes = []byte("") openBracketBytes = []byte("[") closeBracketBytes = []byte("]") percentBytes = []byte("%") precisionBytes = []byte(".") openAngleBytes = []byte("<") closeAngleBytes = []byte(">") openMapBytes = []byte("map[") closeMapBytes = []byte("]") lenEqualsBytes = []byte("len=") capEqualsBytes = []byte("cap=") ) // hexDigits is used to map a decimal value to a hex digit. var hexDigits = "0123456789abcdef" // catchPanic handles any panics that might occur during the handleMethods // calls. func catchPanic(w io.Writer, v reflect.Value) { if err := recover(); err != nil { w.Write(panicBytes) fmt.Fprintf(w, "%v", err) w.Write(closeParenBytes) } } // handleMethods attempts to call the Error and String methods on the underlying // type the passed reflect.Value represents and outputes the result to Writer w. // // It handles panics in any called methods by catching and displaying the error // as the formatted value. func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) { // We need an interface to check if the type implements the error or // Stringer interface. However, the reflect package won't give us an // interface on certain things like unexported struct fields in order // to enforce visibility rules. We use unsafe, when it's available, // to bypass these restrictions since this package does not mutate the // values. if !v.CanInterface() { if UnsafeDisabled { return false } v = unsafeReflectValue(v) } // Choose whether or not to do error and Stringer interface lookups against // the base type or a pointer to the base type depending on settings. // Technically calling one of these methods with a pointer receiver can // mutate the value, however, types which choose to satisify an error or // Stringer interface with a pointer receiver should not be mutating their // state inside these interface methods. if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() { v = unsafeReflectValue(v) } if v.CanAddr() { v = v.Addr() } // Is it an error or Stringer? switch iface := v.Interface().(type) { case error: defer catchPanic(w, v) if cs.ContinueOnMethod { w.Write(openParenBytes) w.Write([]byte(iface.Error())) w.Write(closeParenBytes) w.Write(spaceBytes) return false } w.Write([]byte(iface.Error())) return true case fmt.Stringer: defer catchPanic(w, v) if cs.ContinueOnMethod { w.Write(openParenBytes) w.Write([]byte(iface.String())) w.Write(closeParenBytes) w.Write(spaceBytes) return false } w.Write([]byte(iface.String())) return true } return false } // printBool outputs a boolean value as true or false to Writer w. func printBool(w io.Writer, val bool) { if val { w.Write(trueBytes) } else { w.Write(falseBytes) } } // printInt outputs a signed integer value to Writer w. func printInt(w io.Writer, val int64, base int) { w.Write([]byte(strconv.FormatInt(val, base))) } // printUint outputs an unsigned integer value to Writer w. func printUint(w io.Writer, val uint64, base int) { w.Write([]byte(strconv.FormatUint(val, base))) } // printFloat outputs a floating point value using the specified precision, // which is expected to be 32 or 64bit, to Writer w. func printFloat(w io.Writer, val float64, precision int) { w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision))) } // printComplex outputs a complex value using the specified float precision // for the real and imaginary parts to Writer w. func printComplex(w io.Writer, c complex128, floatPrecision int) { r := real(c) w.Write(openParenBytes) w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision))) i := imag(c) if i >= 0 { w.Write(plusBytes) } w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision))) w.Write(iBytes) w.Write(closeParenBytes) } // printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x' // prefix to Writer w. func printHexPtr(w io.Writer, p uintptr) { // Null pointer. num := uint64(p) if num == 0 { w.Write(nilAngleBytes) return } // Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix buf := make([]byte, 18) // It's simpler to construct the hex string right to left. base := uint64(16) i := len(buf) - 1 for num >= base { buf[i] = hexDigits[num%base] num /= base i-- } buf[i] = hexDigits[num] // Add '0x' prefix. i-- buf[i] = 'x' i-- buf[i] = '0' // Strip unused leading bytes. buf = buf[i:] w.Write(buf) } // valuesSorter implements sort.Interface to allow a slice of reflect.Value // elements to be sorted. type valuesSorter struct { values []reflect.Value strings []string // either nil or same len and values cs *ConfigState } // newValuesSorter initializes a valuesSorter instance, which holds a set of // surrogate keys on which the data should be sorted. It uses flags in // ConfigState to decide if and how to populate those surrogate keys. func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface { vs := &valuesSorter{values: values, cs: cs} if canSortSimply(vs.values[0].Kind()) { return vs } if !cs.DisableMethods { vs.strings = make([]string, len(values)) for i := range vs.values { b := bytes.Buffer{} if !handleMethods(cs, &b, vs.values[i]) { vs.strings = nil break } vs.strings[i] = b.String() } } if vs.strings == nil && cs.SpewKeys { vs.strings = make([]string, len(values)) for i := range vs.values { vs.strings[i] = Sprintf("%#v", vs.values[i].Interface()) } } return vs } // canSortSimply tests whether a reflect.Kind is a primitive that can be sorted // directly, or whether it should be considered for sorting by surrogate keys // (if the ConfigState allows it). func canSortSimply(kind reflect.Kind) bool { // This switch parallels valueSortLess, except for the default case. switch kind { case reflect.Bool: return true case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: return true case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: return true case reflect.Float32, reflect.Float64: return true case reflect.String: return true case reflect.Uintptr: return true case reflect.Array: return true } return false } // Len returns the number of values in the slice. It is part of the // sort.Interface implementation. func (s *valuesSorter) Len() int { return len(s.values) } // Swap swaps the values at the passed indices. It is part of the // sort.Interface implementation. func (s *valuesSorter) Swap(i, j int) { s.values[i], s.values[j] = s.values[j], s.values[i] if s.strings != nil { s.strings[i], s.strings[j] = s.strings[j], s.strings[i] } } // valueSortLess returns whether the first value should sort before the second // value. It is used by valueSorter.Less as part of the sort.Interface // implementation. func valueSortLess(a, b reflect.Value) bool { switch a.Kind() { case reflect.Bool: return !a.Bool() && b.Bool() case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: return a.Int() < b.Int() case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: return a.Uint() < b.Uint() case reflect.Float32, reflect.Float64: return a.Float() < b.Float() case reflect.String: return a.String() < b.String() case reflect.Uintptr: return a.Uint() < b.Uint() case reflect.Array: // Compare the contents of both arrays. l := a.Len() for i := 0; i < l; i++ { av := a.Index(i) bv := b.Index(i) if av.Interface() == bv.Interface() { continue } return valueSortLess(av, bv) } } return a.String() < b.String() } // Less returns whether the value at index i should sort before the // value at index j. It is part of the sort.Interface implementation. func (s *valuesSorter) Less(i, j int) bool { if s.strings == nil { return valueSortLess(s.values[i], s.values[j]) } return s.strings[i] < s.strings[j] } // sortValues is a sort function that handles both native types and any type that // can be converted to error or Stringer. Other inputs are sorted according to // their Value.String() value to ensure display stability. func sortValues(values []reflect.Value, cs *ConfigState) { if len(values) == 0 { return } sort.Sort(newValuesSorter(values, cs)) } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/common_test.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew_test import ( "fmt" "reflect" "testing" "github.com/davecgh/go-spew/spew" ) // custom type to test Stinger interface on non-pointer receiver. type stringer string // String implements the Stringer interface for testing invocation of custom // stringers on types with non-pointer receivers. func (s stringer) String() string { return "stringer " + string(s) } // custom type to test Stinger interface on pointer receiver. type pstringer string // String implements the Stringer interface for testing invocation of custom // stringers on types with only pointer receivers. func (s *pstringer) String() string { return "stringer " + string(*s) } // xref1 and xref2 are cross referencing structs for testing circular reference // detection. type xref1 struct { ps2 *xref2 } type xref2 struct { ps1 *xref1 } // indirCir1, indirCir2, and indirCir3 are used to generate an indirect circular // reference for testing detection. type indirCir1 struct { ps2 *indirCir2 } type indirCir2 struct { ps3 *indirCir3 } type indirCir3 struct { ps1 *indirCir1 } // embed is used to test embedded structures. type embed struct { a string } // embedwrap is used to test embedded structures. type embedwrap struct { *embed e *embed } // panicer is used to intentionally cause a panic for testing spew properly // handles them type panicer int func (p panicer) String() string { panic("test panic") } // customError is used to test custom error interface invocation. type customError int func (e customError) Error() string { return fmt.Sprintf("error: %d", int(e)) } // stringizeWants converts a slice of wanted test output into a format suitable // for a test error message. func stringizeWants(wants []string) string { s := "" for i, want := range wants { if i > 0 { s += fmt.Sprintf("want%d: %s", i+1, want) } else { s += "want: " + want } } return s } // testFailed returns whether or not a test failed by checking if the result // of the test is in the slice of wanted strings. func testFailed(result string, wants []string) bool { for _, want := range wants { if result == want { return false } } return true } type sortableStruct struct { x int } func (ss sortableStruct) String() string { return fmt.Sprintf("ss.%d", ss.x) } type unsortableStruct struct { x int } type sortTestCase struct { input []reflect.Value expected []reflect.Value } func helpTestSortValues(tests []sortTestCase, cs *spew.ConfigState, t *testing.T) { getInterfaces := func(values []reflect.Value) []interface{} { interfaces := []interface{}{} for _, v := range values { interfaces = append(interfaces, v.Interface()) } return interfaces } for _, test := range tests { spew.SortValues(test.input, cs) // reflect.DeepEqual cannot really make sense of reflect.Value, // probably because of all the pointer tricks. For instance, // v(2.0) != v(2.0) on a 32-bits system. Turn them into interface{} // instead. input := getInterfaces(test.input) expected := getInterfaces(test.expected) if !reflect.DeepEqual(input, expected) { t.Errorf("Sort mismatch:\n %v != %v", input, expected) } } } // TestSortValues ensures the sort functionality for relect.Value based sorting // works as intended. func TestSortValues(t *testing.T) { v := reflect.ValueOf a := v("a") b := v("b") c := v("c") embedA := v(embed{"a"}) embedB := v(embed{"b"}) embedC := v(embed{"c"}) tests := []sortTestCase{ // No values. { []reflect.Value{}, []reflect.Value{}, }, // Bools. { []reflect.Value{v(false), v(true), v(false)}, []reflect.Value{v(false), v(false), v(true)}, }, // Ints. { []reflect.Value{v(2), v(1), v(3)}, []reflect.Value{v(1), v(2), v(3)}, }, // Uints. { []reflect.Value{v(uint8(2)), v(uint8(1)), v(uint8(3))}, []reflect.Value{v(uint8(1)), v(uint8(2)), v(uint8(3))}, }, // Floats. { []reflect.Value{v(2.0), v(1.0), v(3.0)}, []reflect.Value{v(1.0), v(2.0), v(3.0)}, }, // Strings. { []reflect.Value{b, a, c}, []reflect.Value{a, b, c}, }, // Array { []reflect.Value{v([3]int{3, 2, 1}), v([3]int{1, 3, 2}), v([3]int{1, 2, 3})}, []reflect.Value{v([3]int{1, 2, 3}), v([3]int{1, 3, 2}), v([3]int{3, 2, 1})}, }, // Uintptrs. { []reflect.Value{v(uintptr(2)), v(uintptr(1)), v(uintptr(3))}, []reflect.Value{v(uintptr(1)), v(uintptr(2)), v(uintptr(3))}, }, // SortableStructs. { // Note: not sorted - DisableMethods is set. []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, }, // UnsortableStructs. { // Note: not sorted - SpewKeys is false. []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, }, // Invalid. { []reflect.Value{embedB, embedA, embedC}, []reflect.Value{embedB, embedA, embedC}, }, } cs := spew.ConfigState{DisableMethods: true, SpewKeys: false} helpTestSortValues(tests, &cs, t) } // TestSortValuesWithMethods ensures the sort functionality for relect.Value // based sorting works as intended when using string methods. func TestSortValuesWithMethods(t *testing.T) { v := reflect.ValueOf a := v("a") b := v("b") c := v("c") tests := []sortTestCase{ // Ints. { []reflect.Value{v(2), v(1), v(3)}, []reflect.Value{v(1), v(2), v(3)}, }, // Strings. { []reflect.Value{b, a, c}, []reflect.Value{a, b, c}, }, // SortableStructs. { []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, []reflect.Value{v(sortableStruct{1}), v(sortableStruct{2}), v(sortableStruct{3})}, }, // UnsortableStructs. { // Note: not sorted - SpewKeys is false. []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, }, } cs := spew.ConfigState{DisableMethods: false, SpewKeys: false} helpTestSortValues(tests, &cs, t) } // TestSortValuesWithSpew ensures the sort functionality for relect.Value // based sorting works as intended when using spew to stringify keys. func TestSortValuesWithSpew(t *testing.T) { v := reflect.ValueOf a := v("a") b := v("b") c := v("c") tests := []sortTestCase{ // Ints. { []reflect.Value{v(2), v(1), v(3)}, []reflect.Value{v(1), v(2), v(3)}, }, // Strings. { []reflect.Value{b, a, c}, []reflect.Value{a, b, c}, }, // SortableStructs. { []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, []reflect.Value{v(sortableStruct{1}), v(sortableStruct{2}), v(sortableStruct{3})}, }, // UnsortableStructs. { []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, []reflect.Value{v(unsortableStruct{1}), v(unsortableStruct{2}), v(unsortableStruct{3})}, }, } cs := spew.ConfigState{DisableMethods: true, SpewKeys: true} helpTestSortValues(tests, &cs, t) } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/config.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew import ( "bytes" "fmt" "io" "os" ) // ConfigState houses the configuration options used by spew to format and // display values. There is a global instance, Config, that is used to control // all top-level Formatter and Dump functionality. Each ConfigState instance // provides methods equivalent to the top-level functions. // // The zero value for ConfigState provides no indentation. You would typically // want to set it to a space or a tab. // // Alternatively, you can use NewDefaultConfig to get a ConfigState instance // with default settings. See the documentation of NewDefaultConfig for default // values. type ConfigState struct { // Indent specifies the string to use for each indentation level. The // global config instance that all top-level functions use set this to a // single space by default. If you would like more indentation, you might // set this to a tab with "\t" or perhaps two spaces with " ". Indent string // MaxDepth controls the maximum number of levels to descend into nested // data structures. The default, 0, means there is no limit. // // NOTE: Circular data structures are properly detected, so it is not // necessary to set this value unless you specifically want to limit deeply // nested data structures. MaxDepth int // DisableMethods specifies whether or not error and Stringer interfaces are // invoked for types that implement them. DisableMethods bool // DisablePointerMethods specifies whether or not to check for and invoke // error and Stringer interfaces on types which only accept a pointer // receiver when the current type is not a pointer. // // NOTE: This might be an unsafe action since calling one of these methods // with a pointer receiver could technically mutate the value, however, // in practice, types which choose to satisify an error or Stringer // interface with a pointer receiver should not be mutating their state // inside these interface methods. As a result, this option relies on // access to the unsafe package, so it will not have any effect when // running in environments without access to the unsafe package such as // Google App Engine or with the "safe" build tag specified. DisablePointerMethods bool // DisablePointerAddresses specifies whether to disable the printing of // pointer addresses. This is useful when diffing data structures in tests. DisablePointerAddresses bool // DisableCapacities specifies whether to disable the printing of capacities // for arrays, slices, maps and channels. This is useful when diffing // data structures in tests. DisableCapacities bool // ContinueOnMethod specifies whether or not recursion should continue once // a custom error or Stringer interface is invoked. The default, false, // means it will print the results of invoking the custom error or Stringer // interface and return immediately instead of continuing to recurse into // the internals of the data type. // // NOTE: This flag does not have any effect if method invocation is disabled // via the DisableMethods or DisablePointerMethods options. ContinueOnMethod bool // SortKeys specifies map keys should be sorted before being printed. Use // this to have a more deterministic, diffable output. Note that only // native types (bool, int, uint, floats, uintptr and string) and types // that support the error or Stringer interfaces (if methods are // enabled) are supported, with other types sorted according to the // reflect.Value.String() output which guarantees display stability. SortKeys bool // SpewKeys specifies that, as a last resort attempt, map keys should // be spewed to strings and sorted by those strings. This is only // considered if SortKeys is true. SpewKeys bool } // Config is the active configuration of the top-level functions. // The configuration can be changed by modifying the contents of spew.Config. var Config = ConfigState{Indent: " "} // Errorf is a wrapper for fmt.Errorf that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the formatted string as a value that satisfies error. See NewFormatter // for formatting details. // // This function is shorthand for the following syntax: // // fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) { return fmt.Errorf(format, c.convertArgs(a)...) } // Fprint is a wrapper for fmt.Fprint that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) { return fmt.Fprint(w, c.convertArgs(a)...) } // Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { return fmt.Fprintf(w, format, c.convertArgs(a)...) } // Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it // passed with a Formatter interface returned by c.NewFormatter. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) { return fmt.Fprintln(w, c.convertArgs(a)...) } // Print is a wrapper for fmt.Print that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Print(c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Print(a ...interface{}) (n int, err error) { return fmt.Print(c.convertArgs(a)...) } // Printf is a wrapper for fmt.Printf that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) { return fmt.Printf(format, c.convertArgs(a)...) } // Println is a wrapper for fmt.Println that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Println(c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Println(a ...interface{}) (n int, err error) { return fmt.Println(c.convertArgs(a)...) } // Sprint is a wrapper for fmt.Sprint that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the resulting string. See NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Sprint(a ...interface{}) string { return fmt.Sprint(c.convertArgs(a)...) } // Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were // passed with a Formatter interface returned by c.NewFormatter. It returns // the resulting string. See NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Sprintf(format string, a ...interface{}) string { return fmt.Sprintf(format, c.convertArgs(a)...) } // Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it // were passed with a Formatter interface returned by c.NewFormatter. It // returns the resulting string. See NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b)) func (c *ConfigState) Sprintln(a ...interface{}) string { return fmt.Sprintln(c.convertArgs(a)...) } /* NewFormatter returns a custom formatter that satisfies the fmt.Formatter interface. As a result, it integrates cleanly with standard fmt package printing functions. The formatter is useful for inline printing of smaller data types similar to the standard %v format specifier. The custom formatter only responds to the %v (most compact), %+v (adds pointer addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb combinations. Any other verbs such as %x and %q will be sent to the the standard fmt package for formatting. In addition, the custom formatter ignores the width and precision arguments (however they will still work on the format specifiers not handled by the custom formatter). Typically this function shouldn't be called directly. It is much easier to make use of the custom formatter by calling one of the convenience functions such as c.Printf, c.Println, or c.Printf. */ func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter { return newFormatter(c, v) } // Fdump formats and displays the passed arguments to io.Writer w. It formats // exactly the same as Dump. func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) { fdump(c, w, a...) } /* Dump displays the passed parameters to standard out with newlines, customizable indentation, and additional debug information such as complete types and all pointer addresses used to indirect to the final value. It provides the following features over the built-in printing facilities provided by the fmt package: * Pointers are dereferenced and followed * Circular data structures are detected and handled properly * Custom Stringer/error interfaces are optionally invoked, including on unexported types * Custom types which only implement the Stringer/error interfaces via a pointer receiver are optionally invoked when passing non-pointer variables * Byte arrays and slices are dumped like the hexdump -C command which includes offsets, byte values in hex, and ASCII output The configuration options are controlled by modifying the public members of c. See ConfigState for options documentation. See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to get the formatted result as a string. */ func (c *ConfigState) Dump(a ...interface{}) { fdump(c, os.Stdout, a...) } // Sdump returns a string with the passed arguments formatted exactly the same // as Dump. func (c *ConfigState) Sdump(a ...interface{}) string { var buf bytes.Buffer fdump(c, &buf, a...) return buf.String() } // convertArgs accepts a slice of arguments and returns a slice of the same // length with each argument converted to a spew Formatter interface using // the ConfigState associated with s. func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) { formatters = make([]interface{}, len(args)) for index, arg := range args { formatters[index] = newFormatter(c, arg) } return formatters } // NewDefaultConfig returns a ConfigState with the following default settings. // // Indent: " " // MaxDepth: 0 // DisableMethods: false // DisablePointerMethods: false // ContinueOnMethod: false // SortKeys: false func NewDefaultConfig() *ConfigState { return &ConfigState{Indent: " "} } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/doc.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* Package spew implements a deep pretty printer for Go data structures to aid in debugging. A quick overview of the additional features spew provides over the built-in printing facilities for Go data types are as follows: * Pointers are dereferenced and followed * Circular data structures are detected and handled properly * Custom Stringer/error interfaces are optionally invoked, including on unexported types * Custom types which only implement the Stringer/error interfaces via a pointer receiver are optionally invoked when passing non-pointer variables * Byte arrays and slices are dumped like the hexdump -C command which includes offsets, byte values in hex, and ASCII output (only when using Dump style) There are two different approaches spew allows for dumping Go data structures: * Dump style which prints with newlines, customizable indentation, and additional debug information such as types and all pointer addresses used to indirect to the final value * A custom Formatter interface that integrates cleanly with the standard fmt package and replaces %v, %+v, %#v, and %#+v to provide inline printing similar to the default %v while providing the additional functionality outlined above and passing unsupported format verbs such as %x and %q along to fmt Quick Start This section demonstrates how to quickly get started with spew. See the sections below for further details on formatting and configuration options. To dump a variable with full newlines, indentation, type, and pointer information use Dump, Fdump, or Sdump: spew.Dump(myVar1, myVar2, ...) spew.Fdump(someWriter, myVar1, myVar2, ...) str := spew.Sdump(myVar1, myVar2, ...) Alternatively, if you would prefer to use format strings with a compacted inline printing style, use the convenience wrappers Printf, Fprintf, etc with %v (most compact), %+v (adds pointer addresses), %#v (adds types), or %#+v (adds types and pointer addresses): spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) Configuration Options Configuration of spew is handled by fields in the ConfigState type. For convenience, all of the top-level functions use a global state available via the spew.Config global. It is also possible to create a ConfigState instance that provides methods equivalent to the top-level functions. This allows concurrent configuration options. See the ConfigState documentation for more details. The following configuration options are available: * Indent String to use for each indentation level for Dump functions. It is a single space by default. A popular alternative is "\t". * MaxDepth Maximum number of levels to descend into nested data structures. There is no limit by default. * DisableMethods Disables invocation of error and Stringer interface methods. Method invocation is enabled by default. * DisablePointerMethods Disables invocation of error and Stringer interface methods on types which only accept pointer receivers from non-pointer variables. Pointer method invocation is enabled by default. * DisablePointerAddresses DisablePointerAddresses specifies whether to disable the printing of pointer addresses. This is useful when diffing data structures in tests. * DisableCapacities DisableCapacities specifies whether to disable the printing of capacities for arrays, slices, maps and channels. This is useful when diffing data structures in tests. * ContinueOnMethod Enables recursion into types after invoking error and Stringer interface methods. Recursion after method invocation is disabled by default. * SortKeys Specifies map keys should be sorted before being printed. Use this to have a more deterministic, diffable output. Note that only native types (bool, int, uint, floats, uintptr and string) and types which implement error or Stringer interfaces are supported with other types sorted according to the reflect.Value.String() output which guarantees display stability. Natural map order is used by default. * SpewKeys Specifies that, as a last resort attempt, map keys should be spewed to strings and sorted by those strings. This is only considered if SortKeys is true. Dump Usage Simply call spew.Dump with a list of variables you want to dump: spew.Dump(myVar1, myVar2, ...) You may also call spew.Fdump if you would prefer to output to an arbitrary io.Writer. For example, to dump to standard error: spew.Fdump(os.Stderr, myVar1, myVar2, ...) A third option is to call spew.Sdump to get the formatted output as a string: str := spew.Sdump(myVar1, myVar2, ...) Sample Dump Output See the Dump example for details on the setup of the types and variables being shown here. (main.Foo) { unexportedField: (*main.Bar)(0xf84002e210)({ flag: (main.Flag) flagTwo, data: (uintptr) }), ExportedField: (map[interface {}]interface {}) (len=1) { (string) (len=3) "one": (bool) true } } Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C command as shown. ([]uint8) (len=32 cap=32) { 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| 00000020 31 32 |12| } Custom Formatter Spew provides a custom formatter that implements the fmt.Formatter interface so that it integrates cleanly with standard fmt package printing functions. The formatter is useful for inline printing of smaller data types similar to the standard %v format specifier. The custom formatter only responds to the %v (most compact), %+v (adds pointer addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb combinations. Any other verbs such as %x and %q will be sent to the the standard fmt package for formatting. In addition, the custom formatter ignores the width and precision arguments (however they will still work on the format specifiers not handled by the custom formatter). Custom Formatter Usage The simplest way to make use of the spew custom formatter is to call one of the convenience functions such as spew.Printf, spew.Println, or spew.Printf. The functions have syntax you are most likely already familiar with: spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) spew.Println(myVar, myVar2) spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) See the Index for the full list convenience functions. Sample Formatter Output Double pointer to a uint8: %v: <**>5 %+v: <**>(0xf8400420d0->0xf8400420c8)5 %#v: (**uint8)5 %#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5 Pointer to circular struct with a uint8 field and a pointer to itself: %v: <*>{1 <*>} %+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)} %#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)} %#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)} See the Printf example for details on the setup of variables being shown here. Errors Since it is possible for custom Stringer/error interfaces to panic, spew detects them and handles them internally by printing the panic information inline with the output. Since spew is intended to provide deep pretty printing capabilities on structures, it intentionally does not return any errors. */ package spew ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/dump.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew import ( "bytes" "encoding/hex" "fmt" "io" "os" "reflect" "regexp" "strconv" "strings" ) var ( // uint8Type is a reflect.Type representing a uint8. It is used to // convert cgo types to uint8 slices for hexdumping. uint8Type = reflect.TypeOf(uint8(0)) // cCharRE is a regular expression that matches a cgo char. // It is used to detect character arrays to hexdump them. cCharRE = regexp.MustCompile("^.*\\._Ctype_char$") // cUnsignedCharRE is a regular expression that matches a cgo unsigned // char. It is used to detect unsigned character arrays to hexdump // them. cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$") // cUint8tCharRE is a regular expression that matches a cgo uint8_t. // It is used to detect uint8_t arrays to hexdump them. cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$") ) // dumpState contains information about the state of a dump operation. type dumpState struct { w io.Writer depth int pointers map[uintptr]int ignoreNextType bool ignoreNextIndent bool cs *ConfigState } // indent performs indentation according to the depth level and cs.Indent // option. func (d *dumpState) indent() { if d.ignoreNextIndent { d.ignoreNextIndent = false return } d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth)) } // unpackValue returns values inside of non-nil interfaces when possible. // This is useful for data types like structs, arrays, slices, and maps which // can contain varying types packed inside an interface. func (d *dumpState) unpackValue(v reflect.Value) reflect.Value { if v.Kind() == reflect.Interface && !v.IsNil() { v = v.Elem() } return v } // dumpPtr handles formatting of pointers by indirecting them as necessary. func (d *dumpState) dumpPtr(v reflect.Value) { // Remove pointers at or below the current depth from map used to detect // circular refs. for k, depth := range d.pointers { if depth >= d.depth { delete(d.pointers, k) } } // Keep list of all dereferenced pointers to show later. pointerChain := make([]uintptr, 0) // Figure out how many levels of indirection there are by dereferencing // pointers and unpacking interfaces down the chain while detecting circular // references. nilFound := false cycleFound := false indirects := 0 ve := v for ve.Kind() == reflect.Ptr { if ve.IsNil() { nilFound = true break } indirects++ addr := ve.Pointer() pointerChain = append(pointerChain, addr) if pd, ok := d.pointers[addr]; ok && pd < d.depth { cycleFound = true indirects-- break } d.pointers[addr] = d.depth ve = ve.Elem() if ve.Kind() == reflect.Interface { if ve.IsNil() { nilFound = true break } ve = ve.Elem() } } // Display type information. d.w.Write(openParenBytes) d.w.Write(bytes.Repeat(asteriskBytes, indirects)) d.w.Write([]byte(ve.Type().String())) d.w.Write(closeParenBytes) // Display pointer information. if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 { d.w.Write(openParenBytes) for i, addr := range pointerChain { if i > 0 { d.w.Write(pointerChainBytes) } printHexPtr(d.w, addr) } d.w.Write(closeParenBytes) } // Display dereferenced value. d.w.Write(openParenBytes) switch { case nilFound == true: d.w.Write(nilAngleBytes) case cycleFound == true: d.w.Write(circularBytes) default: d.ignoreNextType = true d.dump(ve) } d.w.Write(closeParenBytes) } // dumpSlice handles formatting of arrays and slices. Byte (uint8 under // reflection) arrays and slices are dumped in hexdump -C fashion. func (d *dumpState) dumpSlice(v reflect.Value) { // Determine whether this type should be hex dumped or not. Also, // for types which should be hexdumped, try to use the underlying data // first, then fall back to trying to convert them to a uint8 slice. var buf []uint8 doConvert := false doHexDump := false numEntries := v.Len() if numEntries > 0 { vt := v.Index(0).Type() vts := vt.String() switch { // C types that need to be converted. case cCharRE.MatchString(vts): fallthrough case cUnsignedCharRE.MatchString(vts): fallthrough case cUint8tCharRE.MatchString(vts): doConvert = true // Try to use existing uint8 slices and fall back to converting // and copying if that fails. case vt.Kind() == reflect.Uint8: // We need an addressable interface to convert the type // to a byte slice. However, the reflect package won't // give us an interface on certain things like // unexported struct fields in order to enforce // visibility rules. We use unsafe, when available, to // bypass these restrictions since this package does not // mutate the values. vs := v if !vs.CanInterface() || !vs.CanAddr() { vs = unsafeReflectValue(vs) } if !UnsafeDisabled { vs = vs.Slice(0, numEntries) // Use the existing uint8 slice if it can be // type asserted. iface := vs.Interface() if slice, ok := iface.([]uint8); ok { buf = slice doHexDump = true break } } // The underlying data needs to be converted if it can't // be type asserted to a uint8 slice. doConvert = true } // Copy and convert the underlying type if needed. if doConvert && vt.ConvertibleTo(uint8Type) { // Convert and copy each element into a uint8 byte // slice. buf = make([]uint8, numEntries) for i := 0; i < numEntries; i++ { vv := v.Index(i) buf[i] = uint8(vv.Convert(uint8Type).Uint()) } doHexDump = true } } // Hexdump the entire slice as needed. if doHexDump { indent := strings.Repeat(d.cs.Indent, d.depth) str := indent + hex.Dump(buf) str = strings.Replace(str, "\n", "\n"+indent, -1) str = strings.TrimRight(str, d.cs.Indent) d.w.Write([]byte(str)) return } // Recursively call dump for each item. for i := 0; i < numEntries; i++ { d.dump(d.unpackValue(v.Index(i))) if i < (numEntries - 1) { d.w.Write(commaNewlineBytes) } else { d.w.Write(newlineBytes) } } } // dump is the main workhorse for dumping a value. It uses the passed reflect // value to figure out what kind of object we are dealing with and formats it // appropriately. It is a recursive function, however circular data structures // are detected and handled properly. func (d *dumpState) dump(v reflect.Value) { // Handle invalid reflect values immediately. kind := v.Kind() if kind == reflect.Invalid { d.w.Write(invalidAngleBytes) return } // Handle pointers specially. if kind == reflect.Ptr { d.indent() d.dumpPtr(v) return } // Print type information unless already handled elsewhere. if !d.ignoreNextType { d.indent() d.w.Write(openParenBytes) d.w.Write([]byte(v.Type().String())) d.w.Write(closeParenBytes) d.w.Write(spaceBytes) } d.ignoreNextType = false // Display length and capacity if the built-in len and cap functions // work with the value's kind and the len/cap itself is non-zero. valueLen, valueCap := 0, 0 switch v.Kind() { case reflect.Array, reflect.Slice, reflect.Chan: valueLen, valueCap = v.Len(), v.Cap() case reflect.Map, reflect.String: valueLen = v.Len() } if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 { d.w.Write(openParenBytes) if valueLen != 0 { d.w.Write(lenEqualsBytes) printInt(d.w, int64(valueLen), 10) } if !d.cs.DisableCapacities && valueCap != 0 { if valueLen != 0 { d.w.Write(spaceBytes) } d.w.Write(capEqualsBytes) printInt(d.w, int64(valueCap), 10) } d.w.Write(closeParenBytes) d.w.Write(spaceBytes) } // Call Stringer/error interfaces if they exist and the handle methods flag // is enabled if !d.cs.DisableMethods { if (kind != reflect.Invalid) && (kind != reflect.Interface) { if handled := handleMethods(d.cs, d.w, v); handled { return } } } switch kind { case reflect.Invalid: // Do nothing. We should never get here since invalid has already // been handled above. case reflect.Bool: printBool(d.w, v.Bool()) case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: printInt(d.w, v.Int(), 10) case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: printUint(d.w, v.Uint(), 10) case reflect.Float32: printFloat(d.w, v.Float(), 32) case reflect.Float64: printFloat(d.w, v.Float(), 64) case reflect.Complex64: printComplex(d.w, v.Complex(), 32) case reflect.Complex128: printComplex(d.w, v.Complex(), 64) case reflect.Slice: if v.IsNil() { d.w.Write(nilAngleBytes) break } fallthrough case reflect.Array: d.w.Write(openBraceNewlineBytes) d.depth++ if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { d.indent() d.w.Write(maxNewlineBytes) } else { d.dumpSlice(v) } d.depth-- d.indent() d.w.Write(closeBraceBytes) case reflect.String: d.w.Write([]byte(strconv.Quote(v.String()))) case reflect.Interface: // The only time we should get here is for nil interfaces due to // unpackValue calls. if v.IsNil() { d.w.Write(nilAngleBytes) } case reflect.Ptr: // Do nothing. We should never get here since pointers have already // been handled above. case reflect.Map: // nil maps should be indicated as different than empty maps if v.IsNil() { d.w.Write(nilAngleBytes) break } d.w.Write(openBraceNewlineBytes) d.depth++ if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { d.indent() d.w.Write(maxNewlineBytes) } else { numEntries := v.Len() keys := v.MapKeys() if d.cs.SortKeys { sortValues(keys, d.cs) } for i, key := range keys { d.dump(d.unpackValue(key)) d.w.Write(colonSpaceBytes) d.ignoreNextIndent = true d.dump(d.unpackValue(v.MapIndex(key))) if i < (numEntries - 1) { d.w.Write(commaNewlineBytes) } else { d.w.Write(newlineBytes) } } } d.depth-- d.indent() d.w.Write(closeBraceBytes) case reflect.Struct: d.w.Write(openBraceNewlineBytes) d.depth++ if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { d.indent() d.w.Write(maxNewlineBytes) } else { vt := v.Type() numFields := v.NumField() for i := 0; i < numFields; i++ { d.indent() vtf := vt.Field(i) d.w.Write([]byte(vtf.Name)) d.w.Write(colonSpaceBytes) d.ignoreNextIndent = true d.dump(d.unpackValue(v.Field(i))) if i < (numFields - 1) { d.w.Write(commaNewlineBytes) } else { d.w.Write(newlineBytes) } } } d.depth-- d.indent() d.w.Write(closeBraceBytes) case reflect.Uintptr: printHexPtr(d.w, uintptr(v.Uint())) case reflect.UnsafePointer, reflect.Chan, reflect.Func: printHexPtr(d.w, v.Pointer()) // There were not any other types at the time this code was written, but // fall back to letting the default fmt package handle it in case any new // types are added. default: if v.CanInterface() { fmt.Fprintf(d.w, "%v", v.Interface()) } else { fmt.Fprintf(d.w, "%v", v.String()) } } } // fdump is a helper function to consolidate the logic from the various public // methods which take varying writers and config states. func fdump(cs *ConfigState, w io.Writer, a ...interface{}) { for _, arg := range a { if arg == nil { w.Write(interfaceBytes) w.Write(spaceBytes) w.Write(nilAngleBytes) w.Write(newlineBytes) continue } d := dumpState{w: w, cs: cs} d.pointers = make(map[uintptr]int) d.dump(reflect.ValueOf(arg)) d.w.Write(newlineBytes) } } // Fdump formats and displays the passed arguments to io.Writer w. It formats // exactly the same as Dump. func Fdump(w io.Writer, a ...interface{}) { fdump(&Config, w, a...) } // Sdump returns a string with the passed arguments formatted exactly the same // as Dump. func Sdump(a ...interface{}) string { var buf bytes.Buffer fdump(&Config, &buf, a...) return buf.String() } /* Dump displays the passed parameters to standard out with newlines, customizable indentation, and additional debug information such as complete types and all pointer addresses used to indirect to the final value. It provides the following features over the built-in printing facilities provided by the fmt package: * Pointers are dereferenced and followed * Circular data structures are detected and handled properly * Custom Stringer/error interfaces are optionally invoked, including on unexported types * Custom types which only implement the Stringer/error interfaces via a pointer receiver are optionally invoked when passing non-pointer variables * Byte arrays and slices are dumped like the hexdump -C command which includes offsets, byte values in hex, and ASCII output The configuration options are controlled by an exported package global, spew.Config. See ConfigState for options documentation. See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to get the formatted result as a string. */ func Dump(a ...interface{}) { fdump(&Config, os.Stdout, a...) } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/dump_test.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* Test Summary: NOTE: For each test, a nil pointer, a single pointer and double pointer to the base test element are also tested to ensure proper indirection across all types. - Max int8, int16, int32, int64, int - Max uint8, uint16, uint32, uint64, uint - Boolean true and false - Standard complex64 and complex128 - Array containing standard ints - Array containing type with custom formatter on pointer receiver only - Array containing interfaces - Array containing bytes - Slice containing standard float32 values - Slice containing type with custom formatter on pointer receiver only - Slice containing interfaces - Slice containing bytes - Nil slice - Standard string - Nil interface - Sub-interface - Map with string keys and int vals - Map with custom formatter type on pointer receiver only keys and vals - Map with interface keys and values - Map with nil interface value - Struct with primitives - Struct that contains another struct - Struct that contains custom type with Stringer pointer interface via both exported and unexported fields - Struct that contains embedded struct and field to same struct - Uintptr to 0 (null pointer) - Uintptr address of real variable - Unsafe.Pointer to 0 (null pointer) - Unsafe.Pointer to address of real variable - Nil channel - Standard int channel - Function with no params and no returns - Function with param and no returns - Function with multiple params and multiple returns - Struct that is circular through self referencing - Structs that are circular through cross referencing - Structs that are indirectly circular - Type that panics in its Stringer interface */ package spew_test import ( "bytes" "fmt" "testing" "unsafe" "github.com/davecgh/go-spew/spew" ) // dumpTest is used to describe a test to be performed against the Dump method. type dumpTest struct { in interface{} wants []string } // dumpTests houses all of the tests to be performed against the Dump method. var dumpTests = make([]dumpTest, 0) // addDumpTest is a helper method to append the passed input and desired result // to dumpTests func addDumpTest(in interface{}, wants ...string) { test := dumpTest{in, wants} dumpTests = append(dumpTests, test) } func addIntDumpTests() { // Max int8. v := int8(127) nv := (*int8)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "int8" vs := "127" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Max int16. v2 := int16(32767) nv2 := (*int16)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "int16" v2s := "32767" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") // Max int32. v3 := int32(2147483647) nv3 := (*int32)(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "int32" v3s := "2147483647" addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") addDumpTest(nv3, "(*"+v3t+")()\n") // Max int64. v4 := int64(9223372036854775807) nv4 := (*int64)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "int64" v4s := "9223372036854775807" addDumpTest(v4, "("+v4t+") "+v4s+"\n") addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") addDumpTest(nv4, "(*"+v4t+")()\n") // Max int. v5 := int(2147483647) nv5 := (*int)(nil) pv5 := &v5 v5Addr := fmt.Sprintf("%p", pv5) pv5Addr := fmt.Sprintf("%p", &pv5) v5t := "int" v5s := "2147483647" addDumpTest(v5, "("+v5t+") "+v5s+"\n") addDumpTest(pv5, "(*"+v5t+")("+v5Addr+")("+v5s+")\n") addDumpTest(&pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")("+v5s+")\n") addDumpTest(nv5, "(*"+v5t+")()\n") } func addUintDumpTests() { // Max uint8. v := uint8(255) nv := (*uint8)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "uint8" vs := "255" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Max uint16. v2 := uint16(65535) nv2 := (*uint16)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "uint16" v2s := "65535" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") // Max uint32. v3 := uint32(4294967295) nv3 := (*uint32)(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "uint32" v3s := "4294967295" addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") addDumpTest(nv3, "(*"+v3t+")()\n") // Max uint64. v4 := uint64(18446744073709551615) nv4 := (*uint64)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "uint64" v4s := "18446744073709551615" addDumpTest(v4, "("+v4t+") "+v4s+"\n") addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") addDumpTest(nv4, "(*"+v4t+")()\n") // Max uint. v5 := uint(4294967295) nv5 := (*uint)(nil) pv5 := &v5 v5Addr := fmt.Sprintf("%p", pv5) pv5Addr := fmt.Sprintf("%p", &pv5) v5t := "uint" v5s := "4294967295" addDumpTest(v5, "("+v5t+") "+v5s+"\n") addDumpTest(pv5, "(*"+v5t+")("+v5Addr+")("+v5s+")\n") addDumpTest(&pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")("+v5s+")\n") addDumpTest(nv5, "(*"+v5t+")()\n") } func addBoolDumpTests() { // Boolean true. v := bool(true) nv := (*bool)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "bool" vs := "true" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Boolean false. v2 := bool(false) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "bool" v2s := "false" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") } func addFloatDumpTests() { // Standard float32. v := float32(3.1415) nv := (*float32)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "float32" vs := "3.1415" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Standard float64. v2 := float64(3.1415926) nv2 := (*float64)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "float64" v2s := "3.1415926" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") } func addComplexDumpTests() { // Standard complex64. v := complex(float32(6), -2) nv := (*complex64)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "complex64" vs := "(6-2i)" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Standard complex128. v2 := complex(float64(-6), 2) nv2 := (*complex128)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "complex128" v2s := "(-6+2i)" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") } func addArrayDumpTests() { // Array containing standard ints. v := [3]int{1, 2, 3} vLen := fmt.Sprintf("%d", len(v)) vCap := fmt.Sprintf("%d", cap(v)) nv := (*[3]int)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "int" vs := "(len=" + vLen + " cap=" + vCap + ") {\n (" + vt + ") 1,\n (" + vt + ") 2,\n (" + vt + ") 3\n}" addDumpTest(v, "([3]"+vt+") "+vs+"\n") addDumpTest(pv, "(*[3]"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**[3]"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*[3]"+vt+")()\n") // Array containing type with custom formatter on pointer receiver only. v2i0 := pstringer("1") v2i1 := pstringer("2") v2i2 := pstringer("3") v2 := [3]pstringer{v2i0, v2i1, v2i2} v2i0Len := fmt.Sprintf("%d", len(v2i0)) v2i1Len := fmt.Sprintf("%d", len(v2i1)) v2i2Len := fmt.Sprintf("%d", len(v2i2)) v2Len := fmt.Sprintf("%d", len(v2)) v2Cap := fmt.Sprintf("%d", cap(v2)) nv2 := (*[3]pstringer)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "spew_test.pstringer" v2sp := "(len=" + v2Len + " cap=" + v2Cap + ") {\n (" + v2t + ") (len=" + v2i0Len + ") stringer 1,\n (" + v2t + ") (len=" + v2i1Len + ") stringer 2,\n (" + v2t + ") (len=" + v2i2Len + ") " + "stringer 3\n}" v2s := v2sp if spew.UnsafeDisabled { v2s = "(len=" + v2Len + " cap=" + v2Cap + ") {\n (" + v2t + ") (len=" + v2i0Len + ") \"1\",\n (" + v2t + ") (len=" + v2i1Len + ") \"2\",\n (" + v2t + ") (len=" + v2i2Len + ") " + "\"3\"\n}" } addDumpTest(v2, "([3]"+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*[3]"+v2t+")("+v2Addr+")("+v2sp+")\n") addDumpTest(&pv2, "(**[3]"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2sp+")\n") addDumpTest(nv2, "(*[3]"+v2t+")()\n") // Array containing interfaces. v3i0 := "one" v3 := [3]interface{}{v3i0, int(2), uint(3)} v3i0Len := fmt.Sprintf("%d", len(v3i0)) v3Len := fmt.Sprintf("%d", len(v3)) v3Cap := fmt.Sprintf("%d", cap(v3)) nv3 := (*[3]interface{})(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "[3]interface {}" v3t2 := "string" v3t3 := "int" v3t4 := "uint" v3s := "(len=" + v3Len + " cap=" + v3Cap + ") {\n (" + v3t2 + ") " + "(len=" + v3i0Len + ") \"one\",\n (" + v3t3 + ") 2,\n (" + v3t4 + ") 3\n}" addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") addDumpTest(nv3, "(*"+v3t+")()\n") // Array containing bytes. v4 := [34]byte{ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, } v4Len := fmt.Sprintf("%d", len(v4)) v4Cap := fmt.Sprintf("%d", cap(v4)) nv4 := (*[34]byte)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "[34]uint8" v4s := "(len=" + v4Len + " cap=" + v4Cap + ") " + "{\n 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20" + " |............... |\n" + " 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30" + " |!\"#$%&'()*+,-./0|\n" + " 00000020 31 32 " + " |12|\n}" addDumpTest(v4, "("+v4t+") "+v4s+"\n") addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") addDumpTest(nv4, "(*"+v4t+")()\n") } func addSliceDumpTests() { // Slice containing standard float32 values. v := []float32{3.14, 6.28, 12.56} vLen := fmt.Sprintf("%d", len(v)) vCap := fmt.Sprintf("%d", cap(v)) nv := (*[]float32)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "float32" vs := "(len=" + vLen + " cap=" + vCap + ") {\n (" + vt + ") 3.14,\n (" + vt + ") 6.28,\n (" + vt + ") 12.56\n}" addDumpTest(v, "([]"+vt+") "+vs+"\n") addDumpTest(pv, "(*[]"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**[]"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*[]"+vt+")()\n") // Slice containing type with custom formatter on pointer receiver only. v2i0 := pstringer("1") v2i1 := pstringer("2") v2i2 := pstringer("3") v2 := []pstringer{v2i0, v2i1, v2i2} v2i0Len := fmt.Sprintf("%d", len(v2i0)) v2i1Len := fmt.Sprintf("%d", len(v2i1)) v2i2Len := fmt.Sprintf("%d", len(v2i2)) v2Len := fmt.Sprintf("%d", len(v2)) v2Cap := fmt.Sprintf("%d", cap(v2)) nv2 := (*[]pstringer)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "spew_test.pstringer" v2s := "(len=" + v2Len + " cap=" + v2Cap + ") {\n (" + v2t + ") (len=" + v2i0Len + ") stringer 1,\n (" + v2t + ") (len=" + v2i1Len + ") stringer 2,\n (" + v2t + ") (len=" + v2i2Len + ") " + "stringer 3\n}" addDumpTest(v2, "([]"+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*[]"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**[]"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*[]"+v2t+")()\n") // Slice containing interfaces. v3i0 := "one" v3 := []interface{}{v3i0, int(2), uint(3), nil} v3i0Len := fmt.Sprintf("%d", len(v3i0)) v3Len := fmt.Sprintf("%d", len(v3)) v3Cap := fmt.Sprintf("%d", cap(v3)) nv3 := (*[]interface{})(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "[]interface {}" v3t2 := "string" v3t3 := "int" v3t4 := "uint" v3t5 := "interface {}" v3s := "(len=" + v3Len + " cap=" + v3Cap + ") {\n (" + v3t2 + ") " + "(len=" + v3i0Len + ") \"one\",\n (" + v3t3 + ") 2,\n (" + v3t4 + ") 3,\n (" + v3t5 + ") \n}" addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") addDumpTest(nv3, "(*"+v3t+")()\n") // Slice containing bytes. v4 := []byte{ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, } v4Len := fmt.Sprintf("%d", len(v4)) v4Cap := fmt.Sprintf("%d", cap(v4)) nv4 := (*[]byte)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "[]uint8" v4s := "(len=" + v4Len + " cap=" + v4Cap + ") " + "{\n 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20" + " |............... |\n" + " 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30" + " |!\"#$%&'()*+,-./0|\n" + " 00000020 31 32 " + " |12|\n}" addDumpTest(v4, "("+v4t+") "+v4s+"\n") addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") addDumpTest(nv4, "(*"+v4t+")()\n") // Nil slice. v5 := []int(nil) nv5 := (*[]int)(nil) pv5 := &v5 v5Addr := fmt.Sprintf("%p", pv5) pv5Addr := fmt.Sprintf("%p", &pv5) v5t := "[]int" v5s := "" addDumpTest(v5, "("+v5t+") "+v5s+"\n") addDumpTest(pv5, "(*"+v5t+")("+v5Addr+")("+v5s+")\n") addDumpTest(&pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")("+v5s+")\n") addDumpTest(nv5, "(*"+v5t+")()\n") } func addStringDumpTests() { // Standard string. v := "test" vLen := fmt.Sprintf("%d", len(v)) nv := (*string)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "string" vs := "(len=" + vLen + ") \"test\"" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") } func addInterfaceDumpTests() { // Nil interface. var v interface{} nv := (*interface{})(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "interface {}" vs := "" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Sub-interface. v2 := interface{}(uint16(65535)) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "uint16" v2s := "65535" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") } func addMapDumpTests() { // Map with string keys and int vals. k := "one" kk := "two" m := map[string]int{k: 1, kk: 2} klen := fmt.Sprintf("%d", len(k)) // not kLen to shut golint up kkLen := fmt.Sprintf("%d", len(kk)) mLen := fmt.Sprintf("%d", len(m)) nilMap := map[string]int(nil) nm := (*map[string]int)(nil) pm := &m mAddr := fmt.Sprintf("%p", pm) pmAddr := fmt.Sprintf("%p", &pm) mt := "map[string]int" mt1 := "string" mt2 := "int" ms := "(len=" + mLen + ") {\n (" + mt1 + ") (len=" + klen + ") " + "\"one\": (" + mt2 + ") 1,\n (" + mt1 + ") (len=" + kkLen + ") \"two\": (" + mt2 + ") 2\n}" ms2 := "(len=" + mLen + ") {\n (" + mt1 + ") (len=" + kkLen + ") " + "\"two\": (" + mt2 + ") 2,\n (" + mt1 + ") (len=" + klen + ") \"one\": (" + mt2 + ") 1\n}" addDumpTest(m, "("+mt+") "+ms+"\n", "("+mt+") "+ms2+"\n") addDumpTest(pm, "(*"+mt+")("+mAddr+")("+ms+")\n", "(*"+mt+")("+mAddr+")("+ms2+")\n") addDumpTest(&pm, "(**"+mt+")("+pmAddr+"->"+mAddr+")("+ms+")\n", "(**"+mt+")("+pmAddr+"->"+mAddr+")("+ms2+")\n") addDumpTest(nm, "(*"+mt+")()\n") addDumpTest(nilMap, "("+mt+") \n") // Map with custom formatter type on pointer receiver only keys and vals. k2 := pstringer("one") v2 := pstringer("1") m2 := map[pstringer]pstringer{k2: v2} k2Len := fmt.Sprintf("%d", len(k2)) v2Len := fmt.Sprintf("%d", len(v2)) m2Len := fmt.Sprintf("%d", len(m2)) nilMap2 := map[pstringer]pstringer(nil) nm2 := (*map[pstringer]pstringer)(nil) pm2 := &m2 m2Addr := fmt.Sprintf("%p", pm2) pm2Addr := fmt.Sprintf("%p", &pm2) m2t := "map[spew_test.pstringer]spew_test.pstringer" m2t1 := "spew_test.pstringer" m2t2 := "spew_test.pstringer" m2s := "(len=" + m2Len + ") {\n (" + m2t1 + ") (len=" + k2Len + ") " + "stringer one: (" + m2t2 + ") (len=" + v2Len + ") stringer 1\n}" if spew.UnsafeDisabled { m2s = "(len=" + m2Len + ") {\n (" + m2t1 + ") (len=" + k2Len + ") " + "\"one\": (" + m2t2 + ") (len=" + v2Len + ") \"1\"\n}" } addDumpTest(m2, "("+m2t+") "+m2s+"\n") addDumpTest(pm2, "(*"+m2t+")("+m2Addr+")("+m2s+")\n") addDumpTest(&pm2, "(**"+m2t+")("+pm2Addr+"->"+m2Addr+")("+m2s+")\n") addDumpTest(nm2, "(*"+m2t+")()\n") addDumpTest(nilMap2, "("+m2t+") \n") // Map with interface keys and values. k3 := "one" k3Len := fmt.Sprintf("%d", len(k3)) m3 := map[interface{}]interface{}{k3: 1} m3Len := fmt.Sprintf("%d", len(m3)) nilMap3 := map[interface{}]interface{}(nil) nm3 := (*map[interface{}]interface{})(nil) pm3 := &m3 m3Addr := fmt.Sprintf("%p", pm3) pm3Addr := fmt.Sprintf("%p", &pm3) m3t := "map[interface {}]interface {}" m3t1 := "string" m3t2 := "int" m3s := "(len=" + m3Len + ") {\n (" + m3t1 + ") (len=" + k3Len + ") " + "\"one\": (" + m3t2 + ") 1\n}" addDumpTest(m3, "("+m3t+") "+m3s+"\n") addDumpTest(pm3, "(*"+m3t+")("+m3Addr+")("+m3s+")\n") addDumpTest(&pm3, "(**"+m3t+")("+pm3Addr+"->"+m3Addr+")("+m3s+")\n") addDumpTest(nm3, "(*"+m3t+")()\n") addDumpTest(nilMap3, "("+m3t+") \n") // Map with nil interface value. k4 := "nil" k4Len := fmt.Sprintf("%d", len(k4)) m4 := map[string]interface{}{k4: nil} m4Len := fmt.Sprintf("%d", len(m4)) nilMap4 := map[string]interface{}(nil) nm4 := (*map[string]interface{})(nil) pm4 := &m4 m4Addr := fmt.Sprintf("%p", pm4) pm4Addr := fmt.Sprintf("%p", &pm4) m4t := "map[string]interface {}" m4t1 := "string" m4t2 := "interface {}" m4s := "(len=" + m4Len + ") {\n (" + m4t1 + ") (len=" + k4Len + ")" + " \"nil\": (" + m4t2 + ") \n}" addDumpTest(m4, "("+m4t+") "+m4s+"\n") addDumpTest(pm4, "(*"+m4t+")("+m4Addr+")("+m4s+")\n") addDumpTest(&pm4, "(**"+m4t+")("+pm4Addr+"->"+m4Addr+")("+m4s+")\n") addDumpTest(nm4, "(*"+m4t+")()\n") addDumpTest(nilMap4, "("+m4t+") \n") } func addStructDumpTests() { // Struct with primitives. type s1 struct { a int8 b uint8 } v := s1{127, 255} nv := (*s1)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.s1" vt2 := "int8" vt3 := "uint8" vs := "{\n a: (" + vt2 + ") 127,\n b: (" + vt3 + ") 255\n}" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Struct that contains another struct. type s2 struct { s1 s1 b bool } v2 := s2{s1{127, 255}, true} nv2 := (*s2)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "spew_test.s2" v2t2 := "spew_test.s1" v2t3 := "int8" v2t4 := "uint8" v2t5 := "bool" v2s := "{\n s1: (" + v2t2 + ") {\n a: (" + v2t3 + ") 127,\n b: (" + v2t4 + ") 255\n },\n b: (" + v2t5 + ") true\n}" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") // Struct that contains custom type with Stringer pointer interface via both // exported and unexported fields. type s3 struct { s pstringer S pstringer } v3 := s3{"test", "test2"} nv3 := (*s3)(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "spew_test.s3" v3t2 := "spew_test.pstringer" v3s := "{\n s: (" + v3t2 + ") (len=4) stringer test,\n S: (" + v3t2 + ") (len=5) stringer test2\n}" v3sp := v3s if spew.UnsafeDisabled { v3s = "{\n s: (" + v3t2 + ") (len=4) \"test\",\n S: (" + v3t2 + ") (len=5) \"test2\"\n}" v3sp = "{\n s: (" + v3t2 + ") (len=4) \"test\",\n S: (" + v3t2 + ") (len=5) stringer test2\n}" } addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3sp+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3sp+")\n") addDumpTest(nv3, "(*"+v3t+")()\n") // Struct that contains embedded struct and field to same struct. e := embed{"embedstr"} eLen := fmt.Sprintf("%d", len("embedstr")) v4 := embedwrap{embed: &e, e: &e} nv4 := (*embedwrap)(nil) pv4 := &v4 eAddr := fmt.Sprintf("%p", &e) v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "spew_test.embedwrap" v4t2 := "spew_test.embed" v4t3 := "string" v4s := "{\n embed: (*" + v4t2 + ")(" + eAddr + ")({\n a: (" + v4t3 + ") (len=" + eLen + ") \"embedstr\"\n }),\n e: (*" + v4t2 + ")(" + eAddr + ")({\n a: (" + v4t3 + ") (len=" + eLen + ")" + " \"embedstr\"\n })\n}" addDumpTest(v4, "("+v4t+") "+v4s+"\n") addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") addDumpTest(nv4, "(*"+v4t+")()\n") } func addUintptrDumpTests() { // Null pointer. v := uintptr(0) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "uintptr" vs := "" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") // Address of real variable. i := 1 v2 := uintptr(unsafe.Pointer(&i)) nv2 := (*uintptr)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "uintptr" v2s := fmt.Sprintf("%p", &i) addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") } func addUnsafePointerDumpTests() { // Null pointer. v := unsafe.Pointer(uintptr(0)) nv := (*unsafe.Pointer)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "unsafe.Pointer" vs := "" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Address of real variable. i := 1 v2 := unsafe.Pointer(&i) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "unsafe.Pointer" v2s := fmt.Sprintf("%p", &i) addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv, "(*"+vt+")()\n") } func addChanDumpTests() { // Nil channel. var v chan int pv := &v nv := (*chan int)(nil) vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "chan int" vs := "" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Real channel. v2 := make(chan int) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "chan int" v2s := fmt.Sprintf("%p", v2) addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") } func addFuncDumpTests() { // Function with no params and no returns. v := addIntDumpTests nv := (*func())(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "func()" vs := fmt.Sprintf("%p", v) addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") // Function with param and no returns. v2 := TestDump nv2 := (*func(*testing.T))(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "func(*testing.T)" v2s := fmt.Sprintf("%p", v2) addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") addDumpTest(nv2, "(*"+v2t+")()\n") // Function with multiple params and multiple returns. var v3 = func(i int, s string) (b bool, err error) { return true, nil } nv3 := (*func(int, string) (bool, error))(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "func(int, string) (bool, error)" v3s := fmt.Sprintf("%p", v3) addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") addDumpTest(nv3, "(*"+v3t+")()\n") } func addCircularDumpTests() { // Struct that is circular through self referencing. type circular struct { c *circular } v := circular{nil} v.c = &v pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.circular" vs := "{\n c: (*" + vt + ")(" + vAddr + ")({\n c: (*" + vt + ")(" + vAddr + ")()\n })\n}" vs2 := "{\n c: (*" + vt + ")(" + vAddr + ")()\n}" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs2+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs2+")\n") // Structs that are circular through cross referencing. v2 := xref1{nil} ts2 := xref2{&v2} v2.ps2 = &ts2 pv2 := &v2 ts2Addr := fmt.Sprintf("%p", &ts2) v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "spew_test.xref1" v2t2 := "spew_test.xref2" v2s := "{\n ps2: (*" + v2t2 + ")(" + ts2Addr + ")({\n ps1: (*" + v2t + ")(" + v2Addr + ")({\n ps2: (*" + v2t2 + ")(" + ts2Addr + ")()\n })\n })\n}" v2s2 := "{\n ps2: (*" + v2t2 + ")(" + ts2Addr + ")({\n ps1: (*" + v2t + ")(" + v2Addr + ")()\n })\n}" addDumpTest(v2, "("+v2t+") "+v2s+"\n") addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s2+")\n") addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s2+")\n") // Structs that are indirectly circular. v3 := indirCir1{nil} tic2 := indirCir2{nil} tic3 := indirCir3{&v3} tic2.ps3 = &tic3 v3.ps2 = &tic2 pv3 := &v3 tic2Addr := fmt.Sprintf("%p", &tic2) tic3Addr := fmt.Sprintf("%p", &tic3) v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "spew_test.indirCir1" v3t2 := "spew_test.indirCir2" v3t3 := "spew_test.indirCir3" v3s := "{\n ps2: (*" + v3t2 + ")(" + tic2Addr + ")({\n ps3: (*" + v3t3 + ")(" + tic3Addr + ")({\n ps1: (*" + v3t + ")(" + v3Addr + ")({\n ps2: (*" + v3t2 + ")(" + tic2Addr + ")()\n })\n })\n })\n}" v3s2 := "{\n ps2: (*" + v3t2 + ")(" + tic2Addr + ")({\n ps3: (*" + v3t3 + ")(" + tic3Addr + ")({\n ps1: (*" + v3t + ")(" + v3Addr + ")()\n })\n })\n}" addDumpTest(v3, "("+v3t+") "+v3s+"\n") addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s2+")\n") addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s2+")\n") } func addPanicDumpTests() { // Type that panics in its Stringer interface. v := panicer(127) nv := (*panicer)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.panicer" vs := "(PANIC=test panic)127" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") } func addErrorDumpTests() { // Type that has a custom Error interface. v := customError(127) nv := (*customError)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.customError" vs := "error: 127" addDumpTest(v, "("+vt+") "+vs+"\n") addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") addDumpTest(nv, "(*"+vt+")()\n") } // TestDump executes all of the tests described by dumpTests. func TestDump(t *testing.T) { // Setup tests. addIntDumpTests() addUintDumpTests() addBoolDumpTests() addFloatDumpTests() addComplexDumpTests() addArrayDumpTests() addSliceDumpTests() addStringDumpTests() addInterfaceDumpTests() addMapDumpTests() addStructDumpTests() addUintptrDumpTests() addUnsafePointerDumpTests() addChanDumpTests() addFuncDumpTests() addCircularDumpTests() addPanicDumpTests() addErrorDumpTests() addCgoDumpTests() t.Logf("Running %d tests", len(dumpTests)) for i, test := range dumpTests { buf := new(bytes.Buffer) spew.Fdump(buf, test.in) s := buf.String() if testFailed(s, test.wants) { t.Errorf("Dump #%d\n got: %s %s", i, s, stringizeWants(test.wants)) continue } } } func TestDumpSortedKeys(t *testing.T) { cfg := spew.ConfigState{SortKeys: true} s := cfg.Sdump(map[int]string{1: "1", 3: "3", 2: "2"}) expected := "(map[int]string) (len=3) {\n(int) 1: (string) (len=1) " + "\"1\",\n(int) 2: (string) (len=1) \"2\",\n(int) 3: (string) " + "(len=1) \"3\"\n" + "}\n" if s != expected { t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) } s = cfg.Sdump(map[stringer]int{"1": 1, "3": 3, "2": 2}) expected = "(map[spew_test.stringer]int) (len=3) {\n" + "(spew_test.stringer) (len=1) stringer 1: (int) 1,\n" + "(spew_test.stringer) (len=1) stringer 2: (int) 2,\n" + "(spew_test.stringer) (len=1) stringer 3: (int) 3\n" + "}\n" if s != expected { t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) } s = cfg.Sdump(map[pstringer]int{pstringer("1"): 1, pstringer("3"): 3, pstringer("2"): 2}) expected = "(map[spew_test.pstringer]int) (len=3) {\n" + "(spew_test.pstringer) (len=1) stringer 1: (int) 1,\n" + "(spew_test.pstringer) (len=1) stringer 2: (int) 2,\n" + "(spew_test.pstringer) (len=1) stringer 3: (int) 3\n" + "}\n" if spew.UnsafeDisabled { expected = "(map[spew_test.pstringer]int) (len=3) {\n" + "(spew_test.pstringer) (len=1) \"1\": (int) 1,\n" + "(spew_test.pstringer) (len=1) \"2\": (int) 2,\n" + "(spew_test.pstringer) (len=1) \"3\": (int) 3\n" + "}\n" } if s != expected { t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) } s = cfg.Sdump(map[customError]int{customError(1): 1, customError(3): 3, customError(2): 2}) expected = "(map[spew_test.customError]int) (len=3) {\n" + "(spew_test.customError) error: 1: (int) 1,\n" + "(spew_test.customError) error: 2: (int) 2,\n" + "(spew_test.customError) error: 3: (int) 3\n" + "}\n" if s != expected { t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) } } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go ================================================ // Copyright (c) 2013-2016 Dave Collins // // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled // when both cgo is supported and "-tags testcgo" is added to the go test // command line. This means the cgo tests are only added (and hence run) when // specifially requested. This configuration is used because spew itself // does not require cgo to run even though it does handle certain cgo types // specially. Rather than forcing all clients to require cgo and an external // C compiler just to run the tests, this scheme makes them optional. // +build cgo,testcgo package spew_test import ( "fmt" "github.com/davecgh/go-spew/spew/testdata" ) func addCgoDumpTests() { // C char pointer. v := testdata.GetCgoCharPointer() nv := testdata.GetCgoNullCharPointer() pv := &v vcAddr := fmt.Sprintf("%p", v) vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "*testdata._Ctype_char" vs := "116" addDumpTest(v, "("+vt+")("+vcAddr+")("+vs+")\n") addDumpTest(pv, "(*"+vt+")("+vAddr+"->"+vcAddr+")("+vs+")\n") addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+"->"+vcAddr+")("+vs+")\n") addDumpTest(nv, "("+vt+")()\n") // C char array. v2, v2l, v2c := testdata.GetCgoCharArray() v2Len := fmt.Sprintf("%d", v2l) v2Cap := fmt.Sprintf("%d", v2c) v2t := "[6]testdata._Ctype_char" v2s := "(len=" + v2Len + " cap=" + v2Cap + ") " + "{\n 00000000 74 65 73 74 32 00 " + " |test2.|\n}" addDumpTest(v2, "("+v2t+") "+v2s+"\n") // C unsigned char array. v3, v3l, v3c := testdata.GetCgoUnsignedCharArray() v3Len := fmt.Sprintf("%d", v3l) v3Cap := fmt.Sprintf("%d", v3c) v3t := "[6]testdata._Ctype_unsignedchar" v3t2 := "[6]testdata._Ctype_uchar" v3s := "(len=" + v3Len + " cap=" + v3Cap + ") " + "{\n 00000000 74 65 73 74 33 00 " + " |test3.|\n}" addDumpTest(v3, "("+v3t+") "+v3s+"\n", "("+v3t2+") "+v3s+"\n") // C signed char array. v4, v4l, v4c := testdata.GetCgoSignedCharArray() v4Len := fmt.Sprintf("%d", v4l) v4Cap := fmt.Sprintf("%d", v4c) v4t := "[6]testdata._Ctype_schar" v4t2 := "testdata._Ctype_schar" v4s := "(len=" + v4Len + " cap=" + v4Cap + ") " + "{\n (" + v4t2 + ") 116,\n (" + v4t2 + ") 101,\n (" + v4t2 + ") 115,\n (" + v4t2 + ") 116,\n (" + v4t2 + ") 52,\n (" + v4t2 + ") 0\n}" addDumpTest(v4, "("+v4t+") "+v4s+"\n") // C uint8_t array. v5, v5l, v5c := testdata.GetCgoUint8tArray() v5Len := fmt.Sprintf("%d", v5l) v5Cap := fmt.Sprintf("%d", v5c) v5t := "[6]testdata._Ctype_uint8_t" v5s := "(len=" + v5Len + " cap=" + v5Cap + ") " + "{\n 00000000 74 65 73 74 35 00 " + " |test5.|\n}" addDumpTest(v5, "("+v5t+") "+v5s+"\n") // C typedefed unsigned char array. v6, v6l, v6c := testdata.GetCgoTypdefedUnsignedCharArray() v6Len := fmt.Sprintf("%d", v6l) v6Cap := fmt.Sprintf("%d", v6c) v6t := "[6]testdata._Ctype_custom_uchar_t" v6s := "(len=" + v6Len + " cap=" + v6Cap + ") " + "{\n 00000000 74 65 73 74 36 00 " + " |test6.|\n}" addDumpTest(v6, "("+v6t+") "+v6s+"\n") } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/dumpnocgo_test.go ================================================ // Copyright (c) 2013 Dave Collins // // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled // when either cgo is not supported or "-tags testcgo" is not added to the go // test command line. This file intentionally does not setup any cgo tests in // this scenario. // +build !cgo !testcgo package spew_test func addCgoDumpTests() { // Don't add any tests for cgo since this file is only compiled when // there should not be any cgo tests. } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/example_test.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew_test import ( "fmt" "github.com/davecgh/go-spew/spew" ) type Flag int const ( flagOne Flag = iota flagTwo ) var flagStrings = map[Flag]string{ flagOne: "flagOne", flagTwo: "flagTwo", } func (f Flag) String() string { if s, ok := flagStrings[f]; ok { return s } return fmt.Sprintf("Unknown flag (%d)", int(f)) } type Bar struct { data uintptr } type Foo struct { unexportedField Bar ExportedField map[interface{}]interface{} } // This example demonstrates how to use Dump to dump variables to stdout. func ExampleDump() { // The following package level declarations are assumed for this example: /* type Flag int const ( flagOne Flag = iota flagTwo ) var flagStrings = map[Flag]string{ flagOne: "flagOne", flagTwo: "flagTwo", } func (f Flag) String() string { if s, ok := flagStrings[f]; ok { return s } return fmt.Sprintf("Unknown flag (%d)", int(f)) } type Bar struct { data uintptr } type Foo struct { unexportedField Bar ExportedField map[interface{}]interface{} } */ // Setup some sample data structures for the example. bar := Bar{uintptr(0)} s1 := Foo{bar, map[interface{}]interface{}{"one": true}} f := Flag(5) b := []byte{ 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, } // Dump! spew.Dump(s1, f, b) // Output: // (spew_test.Foo) { // unexportedField: (spew_test.Bar) { // data: (uintptr) // }, // ExportedField: (map[interface {}]interface {}) (len=1) { // (string) (len=3) "one": (bool) true // } // } // (spew_test.Flag) Unknown flag (5) // ([]uint8) (len=34 cap=34) { // 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | // 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| // 00000020 31 32 |12| // } // } // This example demonstrates how to use Printf to display a variable with a // format string and inline formatting. func ExamplePrintf() { // Create a double pointer to a uint 8. ui8 := uint8(5) pui8 := &ui8 ppui8 := &pui8 // Create a circular data type. type circular struct { ui8 uint8 c *circular } c := circular{ui8: 1} c.c = &c // Print! spew.Printf("ppui8: %v\n", ppui8) spew.Printf("circular: %v\n", c) // Output: // ppui8: <**>5 // circular: {1 <*>{1 <*>}} } // This example demonstrates how to use a ConfigState. func ExampleConfigState() { // Modify the indent level of the ConfigState only. The global // configuration is not modified. scs := spew.ConfigState{Indent: "\t"} // Output using the ConfigState instance. v := map[string]int{"one": 1} scs.Printf("v: %v\n", v) scs.Dump(v) // Output: // v: map[one:1] // (map[string]int) (len=1) { // (string) (len=3) "one": (int) 1 // } } // This example demonstrates how to use ConfigState.Dump to dump variables to // stdout func ExampleConfigState_Dump() { // See the top-level Dump example for details on the types used in this // example. // Create two ConfigState instances with different indentation. scs := spew.ConfigState{Indent: "\t"} scs2 := spew.ConfigState{Indent: " "} // Setup some sample data structures for the example. bar := Bar{uintptr(0)} s1 := Foo{bar, map[interface{}]interface{}{"one": true}} // Dump using the ConfigState instances. scs.Dump(s1) scs2.Dump(s1) // Output: // (spew_test.Foo) { // unexportedField: (spew_test.Bar) { // data: (uintptr) // }, // ExportedField: (map[interface {}]interface {}) (len=1) { // (string) (len=3) "one": (bool) true // } // } // (spew_test.Foo) { // unexportedField: (spew_test.Bar) { // data: (uintptr) // }, // ExportedField: (map[interface {}]interface {}) (len=1) { // (string) (len=3) "one": (bool) true // } // } // } // This example demonstrates how to use ConfigState.Printf to display a variable // with a format string and inline formatting. func ExampleConfigState_Printf() { // See the top-level Dump example for details on the types used in this // example. // Create two ConfigState instances and modify the method handling of the // first ConfigState only. scs := spew.NewDefaultConfig() scs2 := spew.NewDefaultConfig() scs.DisableMethods = true // Alternatively // scs := spew.ConfigState{Indent: " ", DisableMethods: true} // scs2 := spew.ConfigState{Indent: " "} // This is of type Flag which implements a Stringer and has raw value 1. f := flagTwo // Dump using the ConfigState instances. scs.Printf("f: %v\n", f) scs2.Printf("f: %v\n", f) // Output: // f: 1 // f: flagTwo } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/format.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew import ( "bytes" "fmt" "reflect" "strconv" "strings" ) // supportedFlags is a list of all the character flags supported by fmt package. const supportedFlags = "0-+# " // formatState implements the fmt.Formatter interface and contains information // about the state of a formatting operation. The NewFormatter function can // be used to get a new Formatter which can be used directly as arguments // in standard fmt package printing calls. type formatState struct { value interface{} fs fmt.State depth int pointers map[uintptr]int ignoreNextType bool cs *ConfigState } // buildDefaultFormat recreates the original format string without precision // and width information to pass in to fmt.Sprintf in the case of an // unrecognized type. Unless new types are added to the language, this // function won't ever be called. func (f *formatState) buildDefaultFormat() (format string) { buf := bytes.NewBuffer(percentBytes) for _, flag := range supportedFlags { if f.fs.Flag(int(flag)) { buf.WriteRune(flag) } } buf.WriteRune('v') format = buf.String() return format } // constructOrigFormat recreates the original format string including precision // and width information to pass along to the standard fmt package. This allows // automatic deferral of all format strings this package doesn't support. func (f *formatState) constructOrigFormat(verb rune) (format string) { buf := bytes.NewBuffer(percentBytes) for _, flag := range supportedFlags { if f.fs.Flag(int(flag)) { buf.WriteRune(flag) } } if width, ok := f.fs.Width(); ok { buf.WriteString(strconv.Itoa(width)) } if precision, ok := f.fs.Precision(); ok { buf.Write(precisionBytes) buf.WriteString(strconv.Itoa(precision)) } buf.WriteRune(verb) format = buf.String() return format } // unpackValue returns values inside of non-nil interfaces when possible and // ensures that types for values which have been unpacked from an interface // are displayed when the show types flag is also set. // This is useful for data types like structs, arrays, slices, and maps which // can contain varying types packed inside an interface. func (f *formatState) unpackValue(v reflect.Value) reflect.Value { if v.Kind() == reflect.Interface { f.ignoreNextType = false if !v.IsNil() { v = v.Elem() } } return v } // formatPtr handles formatting of pointers by indirecting them as necessary. func (f *formatState) formatPtr(v reflect.Value) { // Display nil if top level pointer is nil. showTypes := f.fs.Flag('#') if v.IsNil() && (!showTypes || f.ignoreNextType) { f.fs.Write(nilAngleBytes) return } // Remove pointers at or below the current depth from map used to detect // circular refs. for k, depth := range f.pointers { if depth >= f.depth { delete(f.pointers, k) } } // Keep list of all dereferenced pointers to possibly show later. pointerChain := make([]uintptr, 0) // Figure out how many levels of indirection there are by derferencing // pointers and unpacking interfaces down the chain while detecting circular // references. nilFound := false cycleFound := false indirects := 0 ve := v for ve.Kind() == reflect.Ptr { if ve.IsNil() { nilFound = true break } indirects++ addr := ve.Pointer() pointerChain = append(pointerChain, addr) if pd, ok := f.pointers[addr]; ok && pd < f.depth { cycleFound = true indirects-- break } f.pointers[addr] = f.depth ve = ve.Elem() if ve.Kind() == reflect.Interface { if ve.IsNil() { nilFound = true break } ve = ve.Elem() } } // Display type or indirection level depending on flags. if showTypes && !f.ignoreNextType { f.fs.Write(openParenBytes) f.fs.Write(bytes.Repeat(asteriskBytes, indirects)) f.fs.Write([]byte(ve.Type().String())) f.fs.Write(closeParenBytes) } else { if nilFound || cycleFound { indirects += strings.Count(ve.Type().String(), "*") } f.fs.Write(openAngleBytes) f.fs.Write([]byte(strings.Repeat("*", indirects))) f.fs.Write(closeAngleBytes) } // Display pointer information depending on flags. if f.fs.Flag('+') && (len(pointerChain) > 0) { f.fs.Write(openParenBytes) for i, addr := range pointerChain { if i > 0 { f.fs.Write(pointerChainBytes) } printHexPtr(f.fs, addr) } f.fs.Write(closeParenBytes) } // Display dereferenced value. switch { case nilFound == true: f.fs.Write(nilAngleBytes) case cycleFound == true: f.fs.Write(circularShortBytes) default: f.ignoreNextType = true f.format(ve) } } // format is the main workhorse for providing the Formatter interface. It // uses the passed reflect value to figure out what kind of object we are // dealing with and formats it appropriately. It is a recursive function, // however circular data structures are detected and handled properly. func (f *formatState) format(v reflect.Value) { // Handle invalid reflect values immediately. kind := v.Kind() if kind == reflect.Invalid { f.fs.Write(invalidAngleBytes) return } // Handle pointers specially. if kind == reflect.Ptr { f.formatPtr(v) return } // Print type information unless already handled elsewhere. if !f.ignoreNextType && f.fs.Flag('#') { f.fs.Write(openParenBytes) f.fs.Write([]byte(v.Type().String())) f.fs.Write(closeParenBytes) } f.ignoreNextType = false // Call Stringer/error interfaces if they exist and the handle methods // flag is enabled. if !f.cs.DisableMethods { if (kind != reflect.Invalid) && (kind != reflect.Interface) { if handled := handleMethods(f.cs, f.fs, v); handled { return } } } switch kind { case reflect.Invalid: // Do nothing. We should never get here since invalid has already // been handled above. case reflect.Bool: printBool(f.fs, v.Bool()) case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: printInt(f.fs, v.Int(), 10) case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: printUint(f.fs, v.Uint(), 10) case reflect.Float32: printFloat(f.fs, v.Float(), 32) case reflect.Float64: printFloat(f.fs, v.Float(), 64) case reflect.Complex64: printComplex(f.fs, v.Complex(), 32) case reflect.Complex128: printComplex(f.fs, v.Complex(), 64) case reflect.Slice: if v.IsNil() { f.fs.Write(nilAngleBytes) break } fallthrough case reflect.Array: f.fs.Write(openBracketBytes) f.depth++ if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { f.fs.Write(maxShortBytes) } else { numEntries := v.Len() for i := 0; i < numEntries; i++ { if i > 0 { f.fs.Write(spaceBytes) } f.ignoreNextType = true f.format(f.unpackValue(v.Index(i))) } } f.depth-- f.fs.Write(closeBracketBytes) case reflect.String: f.fs.Write([]byte(v.String())) case reflect.Interface: // The only time we should get here is for nil interfaces due to // unpackValue calls. if v.IsNil() { f.fs.Write(nilAngleBytes) } case reflect.Ptr: // Do nothing. We should never get here since pointers have already // been handled above. case reflect.Map: // nil maps should be indicated as different than empty maps if v.IsNil() { f.fs.Write(nilAngleBytes) break } f.fs.Write(openMapBytes) f.depth++ if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { f.fs.Write(maxShortBytes) } else { keys := v.MapKeys() if f.cs.SortKeys { sortValues(keys, f.cs) } for i, key := range keys { if i > 0 { f.fs.Write(spaceBytes) } f.ignoreNextType = true f.format(f.unpackValue(key)) f.fs.Write(colonBytes) f.ignoreNextType = true f.format(f.unpackValue(v.MapIndex(key))) } } f.depth-- f.fs.Write(closeMapBytes) case reflect.Struct: numFields := v.NumField() f.fs.Write(openBraceBytes) f.depth++ if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { f.fs.Write(maxShortBytes) } else { vt := v.Type() for i := 0; i < numFields; i++ { if i > 0 { f.fs.Write(spaceBytes) } vtf := vt.Field(i) if f.fs.Flag('+') || f.fs.Flag('#') { f.fs.Write([]byte(vtf.Name)) f.fs.Write(colonBytes) } f.format(f.unpackValue(v.Field(i))) } } f.depth-- f.fs.Write(closeBraceBytes) case reflect.Uintptr: printHexPtr(f.fs, uintptr(v.Uint())) case reflect.UnsafePointer, reflect.Chan, reflect.Func: printHexPtr(f.fs, v.Pointer()) // There were not any other types at the time this code was written, but // fall back to letting the default fmt package handle it if any get added. default: format := f.buildDefaultFormat() if v.CanInterface() { fmt.Fprintf(f.fs, format, v.Interface()) } else { fmt.Fprintf(f.fs, format, v.String()) } } } // Format satisfies the fmt.Formatter interface. See NewFormatter for usage // details. func (f *formatState) Format(fs fmt.State, verb rune) { f.fs = fs // Use standard formatting for verbs that are not v. if verb != 'v' { format := f.constructOrigFormat(verb) fmt.Fprintf(fs, format, f.value) return } if f.value == nil { if fs.Flag('#') { fs.Write(interfaceBytes) } fs.Write(nilAngleBytes) return } f.format(reflect.ValueOf(f.value)) } // newFormatter is a helper function to consolidate the logic from the various // public methods which take varying config states. func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter { fs := &formatState{value: v, cs: cs} fs.pointers = make(map[uintptr]int) return fs } /* NewFormatter returns a custom formatter that satisfies the fmt.Formatter interface. As a result, it integrates cleanly with standard fmt package printing functions. The formatter is useful for inline printing of smaller data types similar to the standard %v format specifier. The custom formatter only responds to the %v (most compact), %+v (adds pointer addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb combinations. Any other verbs such as %x and %q will be sent to the the standard fmt package for formatting. In addition, the custom formatter ignores the width and precision arguments (however they will still work on the format specifiers not handled by the custom formatter). Typically this function shouldn't be called directly. It is much easier to make use of the custom formatter by calling one of the convenience functions such as Printf, Println, or Fprintf. */ func NewFormatter(v interface{}) fmt.Formatter { return newFormatter(&Config, v) } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/format_test.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* Test Summary: NOTE: For each test, a nil pointer, a single pointer and double pointer to the base test element are also tested to ensure proper indirection across all types. - Max int8, int16, int32, int64, int - Max uint8, uint16, uint32, uint64, uint - Boolean true and false - Standard complex64 and complex128 - Array containing standard ints - Array containing type with custom formatter on pointer receiver only - Array containing interfaces - Slice containing standard float32 values - Slice containing type with custom formatter on pointer receiver only - Slice containing interfaces - Nil slice - Standard string - Nil interface - Sub-interface - Map with string keys and int vals - Map with custom formatter type on pointer receiver only keys and vals - Map with interface keys and values - Map with nil interface value - Struct with primitives - Struct that contains another struct - Struct that contains custom type with Stringer pointer interface via both exported and unexported fields - Struct that contains embedded struct and field to same struct - Uintptr to 0 (null pointer) - Uintptr address of real variable - Unsafe.Pointer to 0 (null pointer) - Unsafe.Pointer to address of real variable - Nil channel - Standard int channel - Function with no params and no returns - Function with param and no returns - Function with multiple params and multiple returns - Struct that is circular through self referencing - Structs that are circular through cross referencing - Structs that are indirectly circular - Type that panics in its Stringer interface - Type that has a custom Error interface - %x passthrough with uint - %#x passthrough with uint - %f passthrough with precision - %f passthrough with width and precision - %d passthrough with width - %q passthrough with string */ package spew_test import ( "bytes" "fmt" "testing" "unsafe" "github.com/davecgh/go-spew/spew" ) // formatterTest is used to describe a test to be performed against NewFormatter. type formatterTest struct { format string in interface{} wants []string } // formatterTests houses all of the tests to be performed against NewFormatter. var formatterTests = make([]formatterTest, 0) // addFormatterTest is a helper method to append the passed input and desired // result to formatterTests. func addFormatterTest(format string, in interface{}, wants ...string) { test := formatterTest{format, in, wants} formatterTests = append(formatterTests, test) } func addIntFormatterTests() { // Max int8. v := int8(127) nv := (*int8)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "int8" vs := "127" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Max int16. v2 := int16(32767) nv2 := (*int16)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "int16" v2s := "32767" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Max int32. v3 := int32(2147483647) nv3 := (*int32)(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "int32" v3s := "2147483647" addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s) addFormatterTest("%v", &pv3, "<**>"+v3s) addFormatterTest("%v", nv3, "") addFormatterTest("%+v", v3, v3s) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") // Max int64. v4 := int64(9223372036854775807) nv4 := (*int64)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "int64" v4s := "9223372036854775807" addFormatterTest("%v", v4, v4s) addFormatterTest("%v", pv4, "<*>"+v4s) addFormatterTest("%v", &pv4, "<**>"+v4s) addFormatterTest("%v", nv4, "") addFormatterTest("%+v", v4, v4s) addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%#v", v4, "("+v4t+")"+v4s) addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s) addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s) addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") addFormatterTest("%#+v", v4, "("+v4t+")"+v4s) addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s) addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") // Max int. v5 := int(2147483647) nv5 := (*int)(nil) pv5 := &v5 v5Addr := fmt.Sprintf("%p", pv5) pv5Addr := fmt.Sprintf("%p", &pv5) v5t := "int" v5s := "2147483647" addFormatterTest("%v", v5, v5s) addFormatterTest("%v", pv5, "<*>"+v5s) addFormatterTest("%v", &pv5, "<**>"+v5s) addFormatterTest("%v", nv5, "") addFormatterTest("%+v", v5, v5s) addFormatterTest("%+v", pv5, "<*>("+v5Addr+")"+v5s) addFormatterTest("%+v", &pv5, "<**>("+pv5Addr+"->"+v5Addr+")"+v5s) addFormatterTest("%+v", nv5, "") addFormatterTest("%#v", v5, "("+v5t+")"+v5s) addFormatterTest("%#v", pv5, "(*"+v5t+")"+v5s) addFormatterTest("%#v", &pv5, "(**"+v5t+")"+v5s) addFormatterTest("%#v", nv5, "(*"+v5t+")"+"") addFormatterTest("%#+v", v5, "("+v5t+")"+v5s) addFormatterTest("%#+v", pv5, "(*"+v5t+")("+v5Addr+")"+v5s) addFormatterTest("%#+v", &pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")"+v5s) addFormatterTest("%#+v", nv5, "(*"+v5t+")"+"") } func addUintFormatterTests() { // Max uint8. v := uint8(255) nv := (*uint8)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "uint8" vs := "255" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Max uint16. v2 := uint16(65535) nv2 := (*uint16)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "uint16" v2s := "65535" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Max uint32. v3 := uint32(4294967295) nv3 := (*uint32)(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "uint32" v3s := "4294967295" addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s) addFormatterTest("%v", &pv3, "<**>"+v3s) addFormatterTest("%v", nv3, "") addFormatterTest("%+v", v3, v3s) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") // Max uint64. v4 := uint64(18446744073709551615) nv4 := (*uint64)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "uint64" v4s := "18446744073709551615" addFormatterTest("%v", v4, v4s) addFormatterTest("%v", pv4, "<*>"+v4s) addFormatterTest("%v", &pv4, "<**>"+v4s) addFormatterTest("%v", nv4, "") addFormatterTest("%+v", v4, v4s) addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%#v", v4, "("+v4t+")"+v4s) addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s) addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s) addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") addFormatterTest("%#+v", v4, "("+v4t+")"+v4s) addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s) addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") // Max uint. v5 := uint(4294967295) nv5 := (*uint)(nil) pv5 := &v5 v5Addr := fmt.Sprintf("%p", pv5) pv5Addr := fmt.Sprintf("%p", &pv5) v5t := "uint" v5s := "4294967295" addFormatterTest("%v", v5, v5s) addFormatterTest("%v", pv5, "<*>"+v5s) addFormatterTest("%v", &pv5, "<**>"+v5s) addFormatterTest("%v", nv5, "") addFormatterTest("%+v", v5, v5s) addFormatterTest("%+v", pv5, "<*>("+v5Addr+")"+v5s) addFormatterTest("%+v", &pv5, "<**>("+pv5Addr+"->"+v5Addr+")"+v5s) addFormatterTest("%+v", nv5, "") addFormatterTest("%#v", v5, "("+v5t+")"+v5s) addFormatterTest("%#v", pv5, "(*"+v5t+")"+v5s) addFormatterTest("%#v", &pv5, "(**"+v5t+")"+v5s) addFormatterTest("%#v", nv5, "(*"+v5t+")"+"") addFormatterTest("%#+v", v5, "("+v5t+")"+v5s) addFormatterTest("%#+v", pv5, "(*"+v5t+")("+v5Addr+")"+v5s) addFormatterTest("%#+v", &pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")"+v5s) addFormatterTest("%#v", nv5, "(*"+v5t+")"+"") } func addBoolFormatterTests() { // Boolean true. v := bool(true) nv := (*bool)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "bool" vs := "true" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Boolean false. v2 := bool(false) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "bool" v2s := "false" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) } func addFloatFormatterTests() { // Standard float32. v := float32(3.1415) nv := (*float32)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "float32" vs := "3.1415" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Standard float64. v2 := float64(3.1415926) nv2 := (*float64)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "float64" v2s := "3.1415926" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") } func addComplexFormatterTests() { // Standard complex64. v := complex(float32(6), -2) nv := (*complex64)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "complex64" vs := "(6-2i)" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Standard complex128. v2 := complex(float64(-6), 2) nv2 := (*complex128)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "complex128" v2s := "(-6+2i)" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") } func addArrayFormatterTests() { // Array containing standard ints. v := [3]int{1, 2, 3} nv := (*[3]int)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "[3]int" vs := "[1 2 3]" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Array containing type with custom formatter on pointer receiver only. v2 := [3]pstringer{"1", "2", "3"} nv2 := (*[3]pstringer)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "[3]spew_test.pstringer" v2sp := "[stringer 1 stringer 2 stringer 3]" v2s := v2sp if spew.UnsafeDisabled { v2s = "[1 2 3]" } addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2sp) addFormatterTest("%v", &pv2, "<**>"+v2sp) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2sp) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2sp) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2sp) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2sp) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2sp) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2sp) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Array containing interfaces. v3 := [3]interface{}{"one", int(2), uint(3)} nv3 := (*[3]interface{})(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "[3]interface {}" v3t2 := "string" v3t3 := "int" v3t4 := "uint" v3s := "[one 2 3]" v3s2 := "[(" + v3t2 + ")one (" + v3t3 + ")2 (" + v3t4 + ")3]" addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s) addFormatterTest("%v", &pv3, "<**>"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%+v", v3, v3s) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s2) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s2) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s2) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s2) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s2) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s2) addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") } func addSliceFormatterTests() { // Slice containing standard float32 values. v := []float32{3.14, 6.28, 12.56} nv := (*[]float32)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "[]float32" vs := "[3.14 6.28 12.56]" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Slice containing type with custom formatter on pointer receiver only. v2 := []pstringer{"1", "2", "3"} nv2 := (*[]pstringer)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "[]spew_test.pstringer" v2s := "[stringer 1 stringer 2 stringer 3]" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Slice containing interfaces. v3 := []interface{}{"one", int(2), uint(3), nil} nv3 := (*[]interface{})(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "[]interface {}" v3t2 := "string" v3t3 := "int" v3t4 := "uint" v3t5 := "interface {}" v3s := "[one 2 3 ]" v3s2 := "[(" + v3t2 + ")one (" + v3t3 + ")2 (" + v3t4 + ")3 (" + v3t5 + ")]" addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s) addFormatterTest("%v", &pv3, "<**>"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%+v", v3, v3s) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s2) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s2) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s2) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s2) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s2) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s2) addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") // Nil slice. var v4 []int nv4 := (*[]int)(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "[]int" v4s := "" addFormatterTest("%v", v4, v4s) addFormatterTest("%v", pv4, "<*>"+v4s) addFormatterTest("%v", &pv4, "<**>"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%+v", v4, v4s) addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%#v", v4, "("+v4t+")"+v4s) addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s) addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s) addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") addFormatterTest("%#+v", v4, "("+v4t+")"+v4s) addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s) addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") } func addStringFormatterTests() { // Standard string. v := "test" nv := (*string)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "string" vs := "test" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") } func addInterfaceFormatterTests() { // Nil interface. var v interface{} nv := (*interface{})(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "interface {}" vs := "" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Sub-interface. v2 := interface{}(uint16(65535)) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "uint16" v2s := "65535" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) } func addMapFormatterTests() { // Map with string keys and int vals. v := map[string]int{"one": 1, "two": 2} nilMap := map[string]int(nil) nv := (*map[string]int)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "map[string]int" vs := "map[one:1 two:2]" vs2 := "map[two:2 one:1]" addFormatterTest("%v", v, vs, vs2) addFormatterTest("%v", pv, "<*>"+vs, "<*>"+vs2) addFormatterTest("%v", &pv, "<**>"+vs, "<**>"+vs2) addFormatterTest("%+v", nilMap, "") addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs, vs2) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs, "<*>("+vAddr+")"+vs2) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs, "<**>("+pvAddr+"->"+vAddr+")"+vs2) addFormatterTest("%+v", nilMap, "") addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs, "("+vt+")"+vs2) addFormatterTest("%#v", pv, "(*"+vt+")"+vs, "(*"+vt+")"+vs2) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs, "(**"+vt+")"+vs2) addFormatterTest("%#v", nilMap, "("+vt+")"+"") addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs, "("+vt+")"+vs2) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs, "(*"+vt+")("+vAddr+")"+vs2) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs2) addFormatterTest("%#+v", nilMap, "("+vt+")"+"") addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Map with custom formatter type on pointer receiver only keys and vals. v2 := map[pstringer]pstringer{"one": "1"} nv2 := (*map[pstringer]pstringer)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "map[spew_test.pstringer]spew_test.pstringer" v2s := "map[stringer one:stringer 1]" if spew.UnsafeDisabled { v2s = "map[one:1]" } addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Map with interface keys and values. v3 := map[interface{}]interface{}{"one": 1} nv3 := (*map[interface{}]interface{})(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "map[interface {}]interface {}" v3t1 := "string" v3t2 := "int" v3s := "map[one:1]" v3s2 := "map[(" + v3t1 + ")one:(" + v3t2 + ")1]" addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s) addFormatterTest("%v", &pv3, "<**>"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%+v", v3, v3s) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s2) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s2) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s2) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s2) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s2) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s2) addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") // Map with nil interface value v4 := map[string]interface{}{"nil": nil} nv4 := (*map[string]interface{})(nil) pv4 := &v4 v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "map[string]interface {}" v4t1 := "interface {}" v4s := "map[nil:]" v4s2 := "map[nil:(" + v4t1 + ")]" addFormatterTest("%v", v4, v4s) addFormatterTest("%v", pv4, "<*>"+v4s) addFormatterTest("%v", &pv4, "<**>"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%+v", v4, v4s) addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%#v", v4, "("+v4t+")"+v4s2) addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s2) addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s2) addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") addFormatterTest("%#+v", v4, "("+v4t+")"+v4s2) addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s2) addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s2) addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") } func addStructFormatterTests() { // Struct with primitives. type s1 struct { a int8 b uint8 } v := s1{127, 255} nv := (*s1)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.s1" vt2 := "int8" vt3 := "uint8" vs := "{127 255}" vs2 := "{a:127 b:255}" vs3 := "{a:(" + vt2 + ")127 b:(" + vt3 + ")255}" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs2) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs2) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs2) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs3) addFormatterTest("%#v", pv, "(*"+vt+")"+vs3) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs3) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs3) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs3) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs3) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Struct that contains another struct. type s2 struct { s1 s1 b bool } v2 := s2{s1{127, 255}, true} nv2 := (*s2)(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "spew_test.s2" v2t2 := "spew_test.s1" v2t3 := "int8" v2t4 := "uint8" v2t5 := "bool" v2s := "{{127 255} true}" v2s2 := "{s1:{a:127 b:255} b:true}" v2s3 := "{s1:(" + v2t2 + "){a:(" + v2t3 + ")127 b:(" + v2t4 + ")255} b:(" + v2t5 + ")true}" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s2) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s2) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s2) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s3) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s3) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s3) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s3) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s3) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s3) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Struct that contains custom type with Stringer pointer interface via both // exported and unexported fields. type s3 struct { s pstringer S pstringer } v3 := s3{"test", "test2"} nv3 := (*s3)(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "spew_test.s3" v3t2 := "spew_test.pstringer" v3s := "{stringer test stringer test2}" v3sp := v3s v3s2 := "{s:stringer test S:stringer test2}" v3s2p := v3s2 v3s3 := "{s:(" + v3t2 + ")stringer test S:(" + v3t2 + ")stringer test2}" v3s3p := v3s3 if spew.UnsafeDisabled { v3s = "{test test2}" v3sp = "{test stringer test2}" v3s2 = "{s:test S:test2}" v3s2p = "{s:test S:stringer test2}" v3s3 = "{s:(" + v3t2 + ")test S:(" + v3t2 + ")test2}" v3s3p = "{s:(" + v3t2 + ")test S:(" + v3t2 + ")stringer test2}" } addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3sp) addFormatterTest("%v", &pv3, "<**>"+v3sp) addFormatterTest("%+v", nv3, "") addFormatterTest("%+v", v3, v3s2) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s2p) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s2p) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s3) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s3p) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s3p) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s3) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s3p) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s3p) addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") // Struct that contains embedded struct and field to same struct. e := embed{"embedstr"} v4 := embedwrap{embed: &e, e: &e} nv4 := (*embedwrap)(nil) pv4 := &v4 eAddr := fmt.Sprintf("%p", &e) v4Addr := fmt.Sprintf("%p", pv4) pv4Addr := fmt.Sprintf("%p", &pv4) v4t := "spew_test.embedwrap" v4t2 := "spew_test.embed" v4t3 := "string" v4s := "{<*>{embedstr} <*>{embedstr}}" v4s2 := "{embed:<*>(" + eAddr + "){a:embedstr} e:<*>(" + eAddr + "){a:embedstr}}" v4s3 := "{embed:(*" + v4t2 + "){a:(" + v4t3 + ")embedstr} e:(*" + v4t2 + "){a:(" + v4t3 + ")embedstr}}" v4s4 := "{embed:(*" + v4t2 + ")(" + eAddr + "){a:(" + v4t3 + ")embedstr} e:(*" + v4t2 + ")(" + eAddr + "){a:(" + v4t3 + ")embedstr}}" addFormatterTest("%v", v4, v4s) addFormatterTest("%v", pv4, "<*>"+v4s) addFormatterTest("%v", &pv4, "<**>"+v4s) addFormatterTest("%+v", nv4, "") addFormatterTest("%+v", v4, v4s2) addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s2) addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s2) addFormatterTest("%+v", nv4, "") addFormatterTest("%#v", v4, "("+v4t+")"+v4s3) addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s3) addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s3) addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") addFormatterTest("%#+v", v4, "("+v4t+")"+v4s4) addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s4) addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s4) addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") } func addUintptrFormatterTests() { // Null pointer. v := uintptr(0) nv := (*uintptr)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "uintptr" vs := "" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Address of real variable. i := 1 v2 := uintptr(unsafe.Pointer(&i)) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "uintptr" v2s := fmt.Sprintf("%p", &i) addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) } func addUnsafePointerFormatterTests() { // Null pointer. v := unsafe.Pointer(uintptr(0)) nv := (*unsafe.Pointer)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "unsafe.Pointer" vs := "" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Address of real variable. i := 1 v2 := unsafe.Pointer(&i) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "unsafe.Pointer" v2s := fmt.Sprintf("%p", &i) addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) } func addChanFormatterTests() { // Nil channel. var v chan int pv := &v nv := (*chan int)(nil) vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "chan int" vs := "" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Real channel. v2 := make(chan int) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "chan int" v2s := fmt.Sprintf("%p", v2) addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) } func addFuncFormatterTests() { // Function with no params and no returns. v := addIntFormatterTests nv := (*func())(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "func()" vs := fmt.Sprintf("%p", v) addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") // Function with param and no returns. v2 := TestFormatter nv2 := (*func(*testing.T))(nil) pv2 := &v2 v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "func(*testing.T)" v2s := fmt.Sprintf("%p", v2) addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s) addFormatterTest("%v", &pv2, "<**>"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%+v", v2, v2s) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%+v", nv2, "") addFormatterTest("%#v", v2, "("+v2t+")"+v2s) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") // Function with multiple params and multiple returns. var v3 = func(i int, s string) (b bool, err error) { return true, nil } nv3 := (*func(int, string) (bool, error))(nil) pv3 := &v3 v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "func(int, string) (bool, error)" v3s := fmt.Sprintf("%p", v3) addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s) addFormatterTest("%v", &pv3, "<**>"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%+v", v3, v3s) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%+v", nv3, "") addFormatterTest("%#v", v3, "("+v3t+")"+v3s) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s) addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") addFormatterTest("%#+v", v3, "("+v3t+")"+v3s) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s) addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") } func addCircularFormatterTests() { // Struct that is circular through self referencing. type circular struct { c *circular } v := circular{nil} v.c = &v pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.circular" vs := "{<*>{<*>}}" vs2 := "{<*>}" vs3 := "{c:<*>(" + vAddr + "){c:<*>(" + vAddr + ")}}" vs4 := "{c:<*>(" + vAddr + ")}" vs5 := "{c:(*" + vt + "){c:(*" + vt + ")}}" vs6 := "{c:(*" + vt + ")}" vs7 := "{c:(*" + vt + ")(" + vAddr + "){c:(*" + vt + ")(" + vAddr + ")}}" vs8 := "{c:(*" + vt + ")(" + vAddr + ")}" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs2) addFormatterTest("%v", &pv, "<**>"+vs2) addFormatterTest("%+v", v, vs3) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs4) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs4) addFormatterTest("%#v", v, "("+vt+")"+vs5) addFormatterTest("%#v", pv, "(*"+vt+")"+vs6) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs6) addFormatterTest("%#+v", v, "("+vt+")"+vs7) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs8) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs8) // Structs that are circular through cross referencing. v2 := xref1{nil} ts2 := xref2{&v2} v2.ps2 = &ts2 pv2 := &v2 ts2Addr := fmt.Sprintf("%p", &ts2) v2Addr := fmt.Sprintf("%p", pv2) pv2Addr := fmt.Sprintf("%p", &pv2) v2t := "spew_test.xref1" v2t2 := "spew_test.xref2" v2s := "{<*>{<*>{<*>}}}" v2s2 := "{<*>{<*>}}" v2s3 := "{ps2:<*>(" + ts2Addr + "){ps1:<*>(" + v2Addr + "){ps2:<*>(" + ts2Addr + ")}}}" v2s4 := "{ps2:<*>(" + ts2Addr + "){ps1:<*>(" + v2Addr + ")}}" v2s5 := "{ps2:(*" + v2t2 + "){ps1:(*" + v2t + "){ps2:(*" + v2t2 + ")}}}" v2s6 := "{ps2:(*" + v2t2 + "){ps1:(*" + v2t + ")}}" v2s7 := "{ps2:(*" + v2t2 + ")(" + ts2Addr + "){ps1:(*" + v2t + ")(" + v2Addr + "){ps2:(*" + v2t2 + ")(" + ts2Addr + ")}}}" v2s8 := "{ps2:(*" + v2t2 + ")(" + ts2Addr + "){ps1:(*" + v2t + ")(" + v2Addr + ")}}" addFormatterTest("%v", v2, v2s) addFormatterTest("%v", pv2, "<*>"+v2s2) addFormatterTest("%v", &pv2, "<**>"+v2s2) addFormatterTest("%+v", v2, v2s3) addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s4) addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s4) addFormatterTest("%#v", v2, "("+v2t+")"+v2s5) addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s6) addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s6) addFormatterTest("%#+v", v2, "("+v2t+")"+v2s7) addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s8) addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s8) // Structs that are indirectly circular. v3 := indirCir1{nil} tic2 := indirCir2{nil} tic3 := indirCir3{&v3} tic2.ps3 = &tic3 v3.ps2 = &tic2 pv3 := &v3 tic2Addr := fmt.Sprintf("%p", &tic2) tic3Addr := fmt.Sprintf("%p", &tic3) v3Addr := fmt.Sprintf("%p", pv3) pv3Addr := fmt.Sprintf("%p", &pv3) v3t := "spew_test.indirCir1" v3t2 := "spew_test.indirCir2" v3t3 := "spew_test.indirCir3" v3s := "{<*>{<*>{<*>{<*>}}}}" v3s2 := "{<*>{<*>{<*>}}}" v3s3 := "{ps2:<*>(" + tic2Addr + "){ps3:<*>(" + tic3Addr + "){ps1:<*>(" + v3Addr + "){ps2:<*>(" + tic2Addr + ")}}}}" v3s4 := "{ps2:<*>(" + tic2Addr + "){ps3:<*>(" + tic3Addr + "){ps1:<*>(" + v3Addr + ")}}}" v3s5 := "{ps2:(*" + v3t2 + "){ps3:(*" + v3t3 + "){ps1:(*" + v3t + "){ps2:(*" + v3t2 + ")}}}}" v3s6 := "{ps2:(*" + v3t2 + "){ps3:(*" + v3t3 + "){ps1:(*" + v3t + ")}}}" v3s7 := "{ps2:(*" + v3t2 + ")(" + tic2Addr + "){ps3:(*" + v3t3 + ")(" + tic3Addr + "){ps1:(*" + v3t + ")(" + v3Addr + "){ps2:(*" + v3t2 + ")(" + tic2Addr + ")}}}}" v3s8 := "{ps2:(*" + v3t2 + ")(" + tic2Addr + "){ps3:(*" + v3t3 + ")(" + tic3Addr + "){ps1:(*" + v3t + ")(" + v3Addr + ")}}}" addFormatterTest("%v", v3, v3s) addFormatterTest("%v", pv3, "<*>"+v3s2) addFormatterTest("%v", &pv3, "<**>"+v3s2) addFormatterTest("%+v", v3, v3s3) addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s4) addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s4) addFormatterTest("%#v", v3, "("+v3t+")"+v3s5) addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s6) addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s6) addFormatterTest("%#+v", v3, "("+v3t+")"+v3s7) addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s8) addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s8) } func addPanicFormatterTests() { // Type that panics in its Stringer interface. v := panicer(127) nv := (*panicer)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.panicer" vs := "(PANIC=test panic)127" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") } func addErrorFormatterTests() { // Type that has a custom Error interface. v := customError(127) nv := (*customError)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) pvAddr := fmt.Sprintf("%p", &pv) vt := "spew_test.customError" vs := "error: 127" addFormatterTest("%v", v, vs) addFormatterTest("%v", pv, "<*>"+vs) addFormatterTest("%v", &pv, "<**>"+vs) addFormatterTest("%v", nv, "") addFormatterTest("%+v", v, vs) addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%+v", nv, "") addFormatterTest("%#v", v, "("+vt+")"+vs) addFormatterTest("%#v", pv, "(*"+vt+")"+vs) addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) addFormatterTest("%#v", nv, "(*"+vt+")"+"") addFormatterTest("%#+v", v, "("+vt+")"+vs) addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) addFormatterTest("%#+v", nv, "(*"+vt+")"+"") } func addPassthroughFormatterTests() { // %x passthrough with uint. v := uint(4294967295) pv := &v vAddr := fmt.Sprintf("%x", pv) pvAddr := fmt.Sprintf("%x", &pv) vs := "ffffffff" addFormatterTest("%x", v, vs) addFormatterTest("%x", pv, vAddr) addFormatterTest("%x", &pv, pvAddr) // %#x passthrough with uint. v2 := int(2147483647) pv2 := &v2 v2Addr := fmt.Sprintf("%#x", pv2) pv2Addr := fmt.Sprintf("%#x", &pv2) v2s := "0x7fffffff" addFormatterTest("%#x", v2, v2s) addFormatterTest("%#x", pv2, v2Addr) addFormatterTest("%#x", &pv2, pv2Addr) // %f passthrough with precision. addFormatterTest("%.2f", 3.1415, "3.14") addFormatterTest("%.3f", 3.1415, "3.142") addFormatterTest("%.4f", 3.1415, "3.1415") // %f passthrough with width and precision. addFormatterTest("%5.2f", 3.1415, " 3.14") addFormatterTest("%6.3f", 3.1415, " 3.142") addFormatterTest("%7.4f", 3.1415, " 3.1415") // %d passthrough with width. addFormatterTest("%3d", 127, "127") addFormatterTest("%4d", 127, " 127") addFormatterTest("%5d", 127, " 127") // %q passthrough with string. addFormatterTest("%q", "test", "\"test\"") } // TestFormatter executes all of the tests described by formatterTests. func TestFormatter(t *testing.T) { // Setup tests. addIntFormatterTests() addUintFormatterTests() addBoolFormatterTests() addFloatFormatterTests() addComplexFormatterTests() addArrayFormatterTests() addSliceFormatterTests() addStringFormatterTests() addInterfaceFormatterTests() addMapFormatterTests() addStructFormatterTests() addUintptrFormatterTests() addUnsafePointerFormatterTests() addChanFormatterTests() addFuncFormatterTests() addCircularFormatterTests() addPanicFormatterTests() addErrorFormatterTests() addPassthroughFormatterTests() t.Logf("Running %d tests", len(formatterTests)) for i, test := range formatterTests { buf := new(bytes.Buffer) spew.Fprintf(buf, test.format, test.in) s := buf.String() if testFailed(s, test.wants) { t.Errorf("Formatter #%d format: %s got: %s %s", i, test.format, s, stringizeWants(test.wants)) continue } } } type testStruct struct { x int } func (ts testStruct) String() string { return fmt.Sprintf("ts.%d", ts.x) } type testStructP struct { x int } func (ts *testStructP) String() string { return fmt.Sprintf("ts.%d", ts.x) } func TestPrintSortedKeys(t *testing.T) { cfg := spew.ConfigState{SortKeys: true} s := cfg.Sprint(map[int]string{1: "1", 3: "3", 2: "2"}) expected := "map[1:1 2:2 3:3]" if s != expected { t.Errorf("Sorted keys mismatch 1:\n %v %v", s, expected) } s = cfg.Sprint(map[stringer]int{"1": 1, "3": 3, "2": 2}) expected = "map[stringer 1:1 stringer 2:2 stringer 3:3]" if s != expected { t.Errorf("Sorted keys mismatch 2:\n %v %v", s, expected) } s = cfg.Sprint(map[pstringer]int{pstringer("1"): 1, pstringer("3"): 3, pstringer("2"): 2}) expected = "map[stringer 1:1 stringer 2:2 stringer 3:3]" if spew.UnsafeDisabled { expected = "map[1:1 2:2 3:3]" } if s != expected { t.Errorf("Sorted keys mismatch 3:\n %v %v", s, expected) } s = cfg.Sprint(map[testStruct]int{testStruct{1}: 1, testStruct{3}: 3, testStruct{2}: 2}) expected = "map[ts.1:1 ts.2:2 ts.3:3]" if s != expected { t.Errorf("Sorted keys mismatch 4:\n %v %v", s, expected) } if !spew.UnsafeDisabled { s = cfg.Sprint(map[testStructP]int{testStructP{1}: 1, testStructP{3}: 3, testStructP{2}: 2}) expected = "map[ts.1:1 ts.2:2 ts.3:3]" if s != expected { t.Errorf("Sorted keys mismatch 5:\n %v %v", s, expected) } } s = cfg.Sprint(map[customError]int{customError(1): 1, customError(3): 3, customError(2): 2}) expected = "map[error: 1:1 error: 2:2 error: 3:3]" if s != expected { t.Errorf("Sorted keys mismatch 6:\n %v %v", s, expected) } } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/internal_test.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* This test file is part of the spew package rather than than the spew_test package because it needs access to internals to properly test certain cases which are not possible via the public interface since they should never happen. */ package spew import ( "bytes" "reflect" "testing" ) // dummyFmtState implements a fake fmt.State to use for testing invalid // reflect.Value handling. This is necessary because the fmt package catches // invalid values before invoking the formatter on them. type dummyFmtState struct { bytes.Buffer } func (dfs *dummyFmtState) Flag(f int) bool { if f == int('+') { return true } return false } func (dfs *dummyFmtState) Precision() (int, bool) { return 0, false } func (dfs *dummyFmtState) Width() (int, bool) { return 0, false } // TestInvalidReflectValue ensures the dump and formatter code handles an // invalid reflect value properly. This needs access to internal state since it // should never happen in real code and therefore can't be tested via the public // API. func TestInvalidReflectValue(t *testing.T) { i := 1 // Dump invalid reflect value. v := new(reflect.Value) buf := new(bytes.Buffer) d := dumpState{w: buf, cs: &Config} d.dump(*v) s := buf.String() want := "" if s != want { t.Errorf("InvalidReflectValue #%d\n got: %s want: %s", i, s, want) } i++ // Formatter invalid reflect value. buf2 := new(dummyFmtState) f := formatState{value: *v, cs: &Config, fs: buf2} f.format(*v) s = buf2.String() want = "" if s != want { t.Errorf("InvalidReflectValue #%d got: %s want: %s", i, s, want) } } // SortValues makes the internal sortValues function available to the test // package. func SortValues(values []reflect.Value, cs *ConfigState) { sortValues(values, cs) } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go ================================================ // Copyright (c) 2013-2016 Dave Collins // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled // when the code is not running on Google App Engine, compiled by GopherJS, and // "-tags safe" is not added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. // +build !js,!appengine,!safe,!disableunsafe /* This test file is part of the spew package rather than than the spew_test package because it needs access to internals to properly test certain cases which are not possible via the public interface since they should never happen. */ package spew import ( "bytes" "reflect" "testing" "unsafe" ) // changeKind uses unsafe to intentionally change the kind of a reflect.Value to // the maximum kind value which does not exist. This is needed to test the // fallback code which punts to the standard fmt library for new types that // might get added to the language. func changeKind(v *reflect.Value, readOnly bool) { rvf := (*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + offsetFlag)) *rvf = *rvf | ((1< * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew import ( "fmt" "io" ) // Errorf is a wrapper for fmt.Errorf that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the formatted string as a value that satisfies error. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b)) func Errorf(format string, a ...interface{}) (err error) { return fmt.Errorf(format, convertArgs(a)...) } // Fprint is a wrapper for fmt.Fprint that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b)) func Fprint(w io.Writer, a ...interface{}) (n int, err error) { return fmt.Fprint(w, convertArgs(a)...) } // Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b)) func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { return fmt.Fprintf(w, format, convertArgs(a)...) } // Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it // passed with a default Formatter interface returned by NewFormatter. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b)) func Fprintln(w io.Writer, a ...interface{}) (n int, err error) { return fmt.Fprintln(w, convertArgs(a)...) } // Print is a wrapper for fmt.Print that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b)) func Print(a ...interface{}) (n int, err error) { return fmt.Print(convertArgs(a)...) } // Printf is a wrapper for fmt.Printf that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b)) func Printf(format string, a ...interface{}) (n int, err error) { return fmt.Printf(format, convertArgs(a)...) } // Println is a wrapper for fmt.Println that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the number of bytes written and any write error encountered. See // NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b)) func Println(a ...interface{}) (n int, err error) { return fmt.Println(convertArgs(a)...) } // Sprint is a wrapper for fmt.Sprint that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the resulting string. See NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b)) func Sprint(a ...interface{}) string { return fmt.Sprint(convertArgs(a)...) } // Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were // passed with a default Formatter interface returned by NewFormatter. It // returns the resulting string. See NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b)) func Sprintf(format string, a ...interface{}) string { return fmt.Sprintf(format, convertArgs(a)...) } // Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it // were passed with a default Formatter interface returned by NewFormatter. It // returns the resulting string. See NewFormatter for formatting details. // // This function is shorthand for the following syntax: // // fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b)) func Sprintln(a ...interface{}) string { return fmt.Sprintln(convertArgs(a)...) } // convertArgs accepts a slice of arguments and returns a slice of the same // length with each argument converted to a default spew Formatter interface. func convertArgs(args []interface{}) (formatters []interface{}) { formatters = make([]interface{}, len(args)) for index, arg := range args { formatters[index] = NewFormatter(arg) } return formatters } ================================================ FILE: vendor/github.com/davecgh/go-spew/spew/spew_test.go ================================================ /* * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ package spew_test import ( "bytes" "fmt" "io/ioutil" "os" "testing" "github.com/davecgh/go-spew/spew" ) // spewFunc is used to identify which public function of the spew package or // ConfigState a test applies to. type spewFunc int const ( fCSFdump spewFunc = iota fCSFprint fCSFprintf fCSFprintln fCSPrint fCSPrintln fCSSdump fCSSprint fCSSprintf fCSSprintln fCSErrorf fCSNewFormatter fErrorf fFprint fFprintln fPrint fPrintln fSdump fSprint fSprintf fSprintln ) // Map of spewFunc values to names for pretty printing. var spewFuncStrings = map[spewFunc]string{ fCSFdump: "ConfigState.Fdump", fCSFprint: "ConfigState.Fprint", fCSFprintf: "ConfigState.Fprintf", fCSFprintln: "ConfigState.Fprintln", fCSSdump: "ConfigState.Sdump", fCSPrint: "ConfigState.Print", fCSPrintln: "ConfigState.Println", fCSSprint: "ConfigState.Sprint", fCSSprintf: "ConfigState.Sprintf", fCSSprintln: "ConfigState.Sprintln", fCSErrorf: "ConfigState.Errorf", fCSNewFormatter: "ConfigState.NewFormatter", fErrorf: "spew.Errorf", fFprint: "spew.Fprint", fFprintln: "spew.Fprintln", fPrint: "spew.Print", fPrintln: "spew.Println", fSdump: "spew.Sdump", fSprint: "spew.Sprint", fSprintf: "spew.Sprintf", fSprintln: "spew.Sprintln", } func (f spewFunc) String() string { if s, ok := spewFuncStrings[f]; ok { return s } return fmt.Sprintf("Unknown spewFunc (%d)", int(f)) } // spewTest is used to describe a test to be performed against the public // functions of the spew package or ConfigState. type spewTest struct { cs *spew.ConfigState f spewFunc format string in interface{} want string } // spewTests houses the tests to be performed against the public functions of // the spew package and ConfigState. // // These tests are only intended to ensure the public functions are exercised // and are intentionally not exhaustive of types. The exhaustive type // tests are handled in the dump and format tests. var spewTests []spewTest // redirStdout is a helper function to return the standard output from f as a // byte slice. func redirStdout(f func()) ([]byte, error) { tempFile, err := ioutil.TempFile("", "ss-test") if err != nil { return nil, err } fileName := tempFile.Name() defer os.Remove(fileName) // Ignore error origStdout := os.Stdout os.Stdout = tempFile f() os.Stdout = origStdout tempFile.Close() return ioutil.ReadFile(fileName) } func initSpewTests() { // Config states with various settings. scsDefault := spew.NewDefaultConfig() scsNoMethods := &spew.ConfigState{Indent: " ", DisableMethods: true} scsNoPmethods := &spew.ConfigState{Indent: " ", DisablePointerMethods: true} scsMaxDepth := &spew.ConfigState{Indent: " ", MaxDepth: 1} scsContinue := &spew.ConfigState{Indent: " ", ContinueOnMethod: true} scsNoPtrAddr := &spew.ConfigState{DisablePointerAddresses: true} scsNoCap := &spew.ConfigState{DisableCapacities: true} // Variables for tests on types which implement Stringer interface with and // without a pointer receiver. ts := stringer("test") tps := pstringer("test") type ptrTester struct { s *struct{} } tptr := &ptrTester{s: &struct{}{}} // depthTester is used to test max depth handling for structs, array, slices // and maps. type depthTester struct { ic indirCir1 arr [1]string slice []string m map[string]int } dt := depthTester{indirCir1{nil}, [1]string{"arr"}, []string{"slice"}, map[string]int{"one": 1}} // Variable for tests on types which implement error interface. te := customError(10) spewTests = []spewTest{ {scsDefault, fCSFdump, "", int8(127), "(int8) 127\n"}, {scsDefault, fCSFprint, "", int16(32767), "32767"}, {scsDefault, fCSFprintf, "%v", int32(2147483647), "2147483647"}, {scsDefault, fCSFprintln, "", int(2147483647), "2147483647\n"}, {scsDefault, fCSPrint, "", int64(9223372036854775807), "9223372036854775807"}, {scsDefault, fCSPrintln, "", uint8(255), "255\n"}, {scsDefault, fCSSdump, "", uint8(64), "(uint8) 64\n"}, {scsDefault, fCSSprint, "", complex(1, 2), "(1+2i)"}, {scsDefault, fCSSprintf, "%v", complex(float32(3), 4), "(3+4i)"}, {scsDefault, fCSSprintln, "", complex(float64(5), 6), "(5+6i)\n"}, {scsDefault, fCSErrorf, "%#v", uint16(65535), "(uint16)65535"}, {scsDefault, fCSNewFormatter, "%v", uint32(4294967295), "4294967295"}, {scsDefault, fErrorf, "%v", uint64(18446744073709551615), "18446744073709551615"}, {scsDefault, fFprint, "", float32(3.14), "3.14"}, {scsDefault, fFprintln, "", float64(6.28), "6.28\n"}, {scsDefault, fPrint, "", true, "true"}, {scsDefault, fPrintln, "", false, "false\n"}, {scsDefault, fSdump, "", complex(-10, -20), "(complex128) (-10-20i)\n"}, {scsDefault, fSprint, "", complex(-1, -2), "(-1-2i)"}, {scsDefault, fSprintf, "%v", complex(float32(-3), -4), "(-3-4i)"}, {scsDefault, fSprintln, "", complex(float64(-5), -6), "(-5-6i)\n"}, {scsNoMethods, fCSFprint, "", ts, "test"}, {scsNoMethods, fCSFprint, "", &ts, "<*>test"}, {scsNoMethods, fCSFprint, "", tps, "test"}, {scsNoMethods, fCSFprint, "", &tps, "<*>test"}, {scsNoPmethods, fCSFprint, "", ts, "stringer test"}, {scsNoPmethods, fCSFprint, "", &ts, "<*>stringer test"}, {scsNoPmethods, fCSFprint, "", tps, "test"}, {scsNoPmethods, fCSFprint, "", &tps, "<*>stringer test"}, {scsMaxDepth, fCSFprint, "", dt, "{{} [] [] map[]}"}, {scsMaxDepth, fCSFdump, "", dt, "(spew_test.depthTester) {\n" + " ic: (spew_test.indirCir1) {\n \n },\n" + " arr: ([1]string) (len=1 cap=1) {\n \n },\n" + " slice: ([]string) (len=1 cap=1) {\n \n },\n" + " m: (map[string]int) (len=1) {\n \n }\n}\n"}, {scsContinue, fCSFprint, "", ts, "(stringer test) test"}, {scsContinue, fCSFdump, "", ts, "(spew_test.stringer) " + "(len=4) (stringer test) \"test\"\n"}, {scsContinue, fCSFprint, "", te, "(error: 10) 10"}, {scsContinue, fCSFdump, "", te, "(spew_test.customError) " + "(error: 10) 10\n"}, {scsNoPtrAddr, fCSFprint, "", tptr, "<*>{<*>{}}"}, {scsNoPtrAddr, fCSSdump, "", tptr, "(*spew_test.ptrTester)({\ns: (*struct {})({\n})\n})\n"}, {scsNoCap, fCSSdump, "", make([]string, 0, 10), "([]string) {\n}\n"}, {scsNoCap, fCSSdump, "", make([]string, 1, 10), "([]string) (len=1) {\n(string) \"\"\n}\n"}, } } // TestSpew executes all of the tests described by spewTests. func TestSpew(t *testing.T) { initSpewTests() t.Logf("Running %d tests", len(spewTests)) for i, test := range spewTests { buf := new(bytes.Buffer) switch test.f { case fCSFdump: test.cs.Fdump(buf, test.in) case fCSFprint: test.cs.Fprint(buf, test.in) case fCSFprintf: test.cs.Fprintf(buf, test.format, test.in) case fCSFprintln: test.cs.Fprintln(buf, test.in) case fCSPrint: b, err := redirStdout(func() { test.cs.Print(test.in) }) if err != nil { t.Errorf("%v #%d %v", test.f, i, err) continue } buf.Write(b) case fCSPrintln: b, err := redirStdout(func() { test.cs.Println(test.in) }) if err != nil { t.Errorf("%v #%d %v", test.f, i, err) continue } buf.Write(b) case fCSSdump: str := test.cs.Sdump(test.in) buf.WriteString(str) case fCSSprint: str := test.cs.Sprint(test.in) buf.WriteString(str) case fCSSprintf: str := test.cs.Sprintf(test.format, test.in) buf.WriteString(str) case fCSSprintln: str := test.cs.Sprintln(test.in) buf.WriteString(str) case fCSErrorf: err := test.cs.Errorf(test.format, test.in) buf.WriteString(err.Error()) case fCSNewFormatter: fmt.Fprintf(buf, test.format, test.cs.NewFormatter(test.in)) case fErrorf: err := spew.Errorf(test.format, test.in) buf.WriteString(err.Error()) case fFprint: spew.Fprint(buf, test.in) case fFprintln: spew.Fprintln(buf, test.in) case fPrint: b, err := redirStdout(func() { spew.Print(test.in) }) if err != nil { t.Errorf("%v #%d %v", test.f, i, err) continue } buf.Write(b) case fPrintln: b, err := redirStdout(func() { spew.Println(test.in) }) if err != nil { t.Errorf("%v #%d %v", test.f, i, err) continue } buf.Write(b) case fSdump: str := spew.Sdump(test.in) buf.WriteString(str) case fSprint: str := spew.Sprint(test.in) buf.WriteString(str) case fSprintf: str := spew.Sprintf(test.format, test.in) buf.WriteString(str) case fSprintln: str := spew.Sprintln(test.in) buf.WriteString(str) default: t.Errorf("%v #%d unrecognized function", test.f, i) continue } s := buf.String() if test.want != s { t.Errorf("ConfigState #%d\n got: %s want: %s", i, s, test.want) continue } } } ================================================ FILE: vendor/github.com/davecgh/go-spew/test_coverage.txt ================================================ github.com/davecgh/go-spew/spew/dump.go dumpState.dump 100.00% (88/88) github.com/davecgh/go-spew/spew/format.go formatState.format 100.00% (82/82) github.com/davecgh/go-spew/spew/format.go formatState.formatPtr 100.00% (52/52) github.com/davecgh/go-spew/spew/dump.go dumpState.dumpPtr 100.00% (44/44) github.com/davecgh/go-spew/spew/dump.go dumpState.dumpSlice 100.00% (39/39) github.com/davecgh/go-spew/spew/common.go handleMethods 100.00% (30/30) github.com/davecgh/go-spew/spew/common.go printHexPtr 100.00% (18/18) github.com/davecgh/go-spew/spew/common.go unsafeReflectValue 100.00% (13/13) github.com/davecgh/go-spew/spew/format.go formatState.constructOrigFormat 100.00% (12/12) github.com/davecgh/go-spew/spew/dump.go fdump 100.00% (11/11) github.com/davecgh/go-spew/spew/format.go formatState.Format 100.00% (11/11) github.com/davecgh/go-spew/spew/common.go init 100.00% (10/10) github.com/davecgh/go-spew/spew/common.go printComplex 100.00% (9/9) github.com/davecgh/go-spew/spew/common.go valuesSorter.Less 100.00% (8/8) github.com/davecgh/go-spew/spew/format.go formatState.buildDefaultFormat 100.00% (7/7) github.com/davecgh/go-spew/spew/format.go formatState.unpackValue 100.00% (5/5) github.com/davecgh/go-spew/spew/dump.go dumpState.indent 100.00% (4/4) github.com/davecgh/go-spew/spew/common.go catchPanic 100.00% (4/4) github.com/davecgh/go-spew/spew/config.go ConfigState.convertArgs 100.00% (4/4) github.com/davecgh/go-spew/spew/spew.go convertArgs 100.00% (4/4) github.com/davecgh/go-spew/spew/format.go newFormatter 100.00% (3/3) github.com/davecgh/go-spew/spew/dump.go Sdump 100.00% (3/3) github.com/davecgh/go-spew/spew/common.go printBool 100.00% (3/3) github.com/davecgh/go-spew/spew/common.go sortValues 100.00% (3/3) github.com/davecgh/go-spew/spew/config.go ConfigState.Sdump 100.00% (3/3) github.com/davecgh/go-spew/spew/dump.go dumpState.unpackValue 100.00% (3/3) github.com/davecgh/go-spew/spew/spew.go Printf 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Println 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Sprint 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Sprintf 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Sprintln 100.00% (1/1) github.com/davecgh/go-spew/spew/common.go printFloat 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go NewDefaultConfig 100.00% (1/1) github.com/davecgh/go-spew/spew/common.go printInt 100.00% (1/1) github.com/davecgh/go-spew/spew/common.go printUint 100.00% (1/1) github.com/davecgh/go-spew/spew/common.go valuesSorter.Len 100.00% (1/1) github.com/davecgh/go-spew/spew/common.go valuesSorter.Swap 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Errorf 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Fprint 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Fprintf 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Fprintln 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Print 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Printf 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Println 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Sprint 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Sprintf 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Sprintln 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.NewFormatter 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Fdump 100.00% (1/1) github.com/davecgh/go-spew/spew/config.go ConfigState.Dump 100.00% (1/1) github.com/davecgh/go-spew/spew/dump.go Fdump 100.00% (1/1) github.com/davecgh/go-spew/spew/dump.go Dump 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Fprintln 100.00% (1/1) github.com/davecgh/go-spew/spew/format.go NewFormatter 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Errorf 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Fprint 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Fprintf 100.00% (1/1) github.com/davecgh/go-spew/spew/spew.go Print 100.00% (1/1) github.com/davecgh/go-spew/spew ------------------------------- 100.00% (505/505) ================================================ FILE: vendor/github.com/golang/protobuf/.gitignore ================================================ .DS_Store *.[568ao] *.ao *.so *.pyc ._* .nfs.* [568a].out *~ *.orig core _obj _test _testmain.go protoc-gen-go/testdata/multi/*.pb.go _conformance/_conformance ================================================ FILE: vendor/github.com/golang/protobuf/.travis.yml ================================================ sudo: false language: go go: - 1.6.x - 1.7.x - 1.8.x - 1.9.x install: - go get -v -d -t github.com/golang/protobuf/... - curl -L https://github.com/google/protobuf/releases/download/v3.3.0/protoc-3.3.0-linux-x86_64.zip -o /tmp/protoc.zip - unzip /tmp/protoc.zip -d $HOME/protoc env: - PATH=$HOME/protoc/bin:$PATH script: - make all test ================================================ FILE: vendor/github.com/golang/protobuf/AUTHORS ================================================ # This source code refers to The Go Authors for copyright purposes. # The master list of authors is in the main Go distribution, # visible at http://tip.golang.org/AUTHORS. ================================================ FILE: vendor/github.com/golang/protobuf/CONTRIBUTORS ================================================ # This source code was written by the Go contributors. # The master list of contributors is in the main Go distribution, # visible at http://tip.golang.org/CONTRIBUTORS. ================================================ FILE: vendor/github.com/golang/protobuf/LICENSE ================================================ Go support for Protocol Buffers - Google's data interchange format Copyright 2010 The Go Authors. All rights reserved. https://github.com/golang/protobuf Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: vendor/github.com/golang/protobuf/Make.protobuf ================================================ # Go support for Protocol Buffers - Google's data interchange format # # Copyright 2010 The Go Authors. All rights reserved. # https://github.com/golang/protobuf # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Includable Makefile to add a rule for generating .pb.go files from .proto files # (Google protocol buffer descriptions). # Typical use if myproto.proto is a file in package mypackage in this directory: # # include $(GOROOT)/src/pkg/github.com/golang/protobuf/Make.protobuf %.pb.go: %.proto protoc --go_out=. $< ================================================ FILE: vendor/github.com/golang/protobuf/Makefile ================================================ # Go support for Protocol Buffers - Google's data interchange format # # Copyright 2010 The Go Authors. All rights reserved. # https://github.com/golang/protobuf # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. all: install install: go install ./proto ./jsonpb ./ptypes go install ./protoc-gen-go test: go test ./proto ./jsonpb ./ptypes make -C protoc-gen-go/testdata test clean: go clean ./... nuke: go clean -i ./... regenerate: make -C protoc-gen-go/descriptor regenerate make -C protoc-gen-go/plugin regenerate make -C protoc-gen-go/testdata regenerate make -C proto/testdata regenerate make -C jsonpb/jsonpb_test_proto regenerate make -C _conformance regenerate ================================================ FILE: vendor/github.com/golang/protobuf/README.md ================================================ # Go support for Protocol Buffers [![Build Status](https://travis-ci.org/golang/protobuf.svg?branch=master)](https://travis-ci.org/golang/protobuf) Google's data interchange format. Copyright 2010 The Go Authors. https://github.com/golang/protobuf This package and the code it generates requires at least Go 1.4. This software implements Go bindings for protocol buffers. For information about protocol buffers themselves, see https://developers.google.com/protocol-buffers/ ## Installation ## To use this software, you must: - Install the standard C++ implementation of protocol buffers from https://developers.google.com/protocol-buffers/ - Of course, install the Go compiler and tools from https://golang.org/ See https://golang.org/doc/install for details or, if you are using gccgo, follow the instructions at https://golang.org/doc/install/gccgo - Grab the code from the repository and install the proto package. The simplest way is to run `go get -u github.com/golang/protobuf/protoc-gen-go`. The compiler plugin, protoc-gen-go, will be installed in $GOBIN, defaulting to $GOPATH/bin. It must be in your $PATH for the protocol compiler, protoc, to find it. This software has two parts: a 'protocol compiler plugin' that generates Go source files that, once compiled, can access and manage protocol buffers; and a library that implements run-time support for encoding (marshaling), decoding (unmarshaling), and accessing protocol buffers. There is support for gRPC in Go using protocol buffers. See the note at the bottom of this file for details. There are no insertion points in the plugin. ## Using protocol buffers with Go ## Once the software is installed, there are two steps to using it. First you must compile the protocol buffer definitions and then import them, with the support library, into your program. To compile the protocol buffer definition, run protoc with the --go_out parameter set to the directory you want to output the Go code to. protoc --go_out=. *.proto The generated files will be suffixed .pb.go. See the Test code below for an example using such a file. The package comment for the proto library contains text describing the interface provided in Go for protocol buffers. Here is an edited version. ========== The proto package converts data structures to and from the wire format of protocol buffers. It works in concert with the Go source code generated for .proto files by the protocol compiler. A summary of the properties of the protocol buffer interface for a protocol buffer variable v: - Names are turned from camel_case to CamelCase for export. - There are no methods on v to set fields; just treat them as structure fields. - There are getters that return a field's value if set, and return the field's default value if unset. The getters work even if the receiver is a nil message. - The zero value for a struct is its correct initialization state. All desired fields must be set before marshaling. - A Reset() method will restore a protobuf struct to its zero state. - Non-repeated fields are pointers to the values; nil means unset. That is, optional or required field int32 f becomes F *int32. - Repeated fields are slices. - Helper functions are available to aid the setting of fields. Helpers for getting values are superseded by the GetFoo methods and their use is deprecated. msg.Foo = proto.String("hello") // set field - Constants are defined to hold the default values of all fields that have them. They have the form Default_StructName_FieldName. Because the getter methods handle defaulted values, direct use of these constants should be rare. - Enums are given type names and maps from names to values. Enum values are prefixed with the enum's type name. Enum types have a String method, and a Enum method to assist in message construction. - Nested groups and enums have type names prefixed with the name of the surrounding message type. - Extensions are given descriptor names that start with E_, followed by an underscore-delimited list of the nested messages that contain it (if any) followed by the CamelCased name of the extension field itself. HasExtension, ClearExtension, GetExtension and SetExtension are functions for manipulating extensions. - Oneof field sets are given a single field in their message, with distinguished wrapper types for each possible field value. - Marshal and Unmarshal are functions to encode and decode the wire format. When the .proto file specifies `syntax="proto3"`, there are some differences: - Non-repeated fields of non-message type are values instead of pointers. - Enum types do not get an Enum method. Consider file test.proto, containing ```proto package example; enum FOO { X = 17; }; message Test { required string label = 1; optional int32 type = 2 [default=77]; repeated int64 reps = 3; optional group OptionalGroup = 4 { required string RequiredField = 5; } } ``` To create and play with a Test object from the example package, ```go package main import ( "log" "github.com/golang/protobuf/proto" "path/to/example" ) func main() { test := &example.Test { Label: proto.String("hello"), Type: proto.Int32(17), Reps: []int64{1, 2, 3}, Optionalgroup: &example.Test_OptionalGroup { RequiredField: proto.String("good bye"), }, } data, err := proto.Marshal(test) if err != nil { log.Fatal("marshaling error: ", err) } newTest := &example.Test{} err = proto.Unmarshal(data, newTest) if err != nil { log.Fatal("unmarshaling error: ", err) } // Now test and newTest contain the same data. if test.GetLabel() != newTest.GetLabel() { log.Fatalf("data mismatch %q != %q", test.GetLabel(), newTest.GetLabel()) } // etc. } ``` ## Parameters ## To pass extra parameters to the plugin, use a comma-separated parameter list separated from the output directory by a colon: protoc --go_out=plugins=grpc,import_path=mypackage:. *.proto - `import_prefix=xxx` - a prefix that is added onto the beginning of all imports. Useful for things like generating protos in a subdirectory, or regenerating vendored protobufs in-place. - `import_path=foo/bar` - used as the package if no input files declare `go_package`. If it contains slashes, everything up to the rightmost slash is ignored. - `plugins=plugin1+plugin2` - specifies the list of sub-plugins to load. The only plugin in this repo is `grpc`. - `Mfoo/bar.proto=quux/shme` - declares that foo/bar.proto is associated with Go package quux/shme. This is subject to the import_prefix parameter. ## gRPC Support ## If a proto file specifies RPC services, protoc-gen-go can be instructed to generate code compatible with gRPC (http://www.grpc.io/). To do this, pass the `plugins` parameter to protoc-gen-go; the usual way is to insert it into the --go_out argument to protoc: protoc --go_out=plugins=grpc:. *.proto ## Compatibility ## The library and the generated code are expected to be stable over time. However, we reserve the right to make breaking changes without notice for the following reasons: - Security. A security issue in the specification or implementation may come to light whose resolution requires breaking compatibility. We reserve the right to address such security issues. - Unspecified behavior. There are some aspects of the Protocol Buffers specification that are undefined. Programs that depend on such unspecified behavior may break in future releases. - Specification errors or changes. If it becomes necessary to address an inconsistency, incompleteness, or change in the Protocol Buffers specification, resolving the issue could affect the meaning or legality of existing programs. We reserve the right to address such issues, including updating the implementations. - Bugs. If the library has a bug that violates the specification, a program that depends on the buggy behavior may break if the bug is fixed. We reserve the right to fix such bugs. - Adding methods or fields to generated structs. These may conflict with field names that already exist in a schema, causing applications to break. When the code generator encounters a field in the schema that would collide with a generated field or method name, the code generator will append an underscore to the generated field or method name. - Adding, removing, or changing methods or fields in generated structs that start with `XXX`. These parts of the generated code are exported out of necessity, but should not be considered part of the public API. - Adding, removing, or changing unexported symbols in generated code. Any breaking changes outside of these will be announced 6 months in advance to protobuf@googlegroups.com. You should, whenever possible, use generated code created by the `protoc-gen-go` tool built at the same commit as the `proto` package. The `proto` package declares package-level constants in the form `ProtoPackageIsVersionX`. Application code and generated code may depend on one of these constants to ensure that compilation will fail if the available version of the proto library is too old. Whenever we make a change to the generated code that requires newer library support, in the same commit we will increment the version number of the generated code and declare a new package-level constant whose name incorporates the latest version number. Removing a compatibility constant is considered a breaking change and would be subject to the announcement policy stated above. The `protoc-gen-go/generator` package exposes a plugin interface, which is used by the gRPC code generation. This interface is not supported and is subject to incompatible changes without notice. ================================================ FILE: vendor/github.com/golang/protobuf/proto/Makefile ================================================ # Go support for Protocol Buffers - Google's data interchange format # # Copyright 2010 The Go Authors. All rights reserved. # https://github.com/golang/protobuf # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. install: go install test: install generate-test-pbs go test generate-test-pbs: make install make -C testdata protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto make ================================================ FILE: vendor/github.com/golang/protobuf/proto/all_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "bytes" "encoding/json" "errors" "fmt" "math" "math/rand" "reflect" "runtime/debug" "strings" "testing" "time" . "github.com/golang/protobuf/proto" . "github.com/golang/protobuf/proto/testdata" ) var globalO *Buffer func old() *Buffer { if globalO == nil { globalO = NewBuffer(nil) } globalO.Reset() return globalO } func equalbytes(b1, b2 []byte, t *testing.T) { if len(b1) != len(b2) { t.Errorf("wrong lengths: 2*%d != %d", len(b1), len(b2)) return } for i := 0; i < len(b1); i++ { if b1[i] != b2[i] { t.Errorf("bad byte[%d]:%x %x: %s %s", i, b1[i], b2[i], b1, b2) } } } func initGoTestField() *GoTestField { f := new(GoTestField) f.Label = String("label") f.Type = String("type") return f } // These are all structurally equivalent but the tag numbers differ. // (It's remarkable that required, optional, and repeated all have // 8 letters.) func initGoTest_RequiredGroup() *GoTest_RequiredGroup { return &GoTest_RequiredGroup{ RequiredField: String("required"), } } func initGoTest_OptionalGroup() *GoTest_OptionalGroup { return &GoTest_OptionalGroup{ RequiredField: String("optional"), } } func initGoTest_RepeatedGroup() *GoTest_RepeatedGroup { return &GoTest_RepeatedGroup{ RequiredField: String("repeated"), } } func initGoTest(setdefaults bool) *GoTest { pb := new(GoTest) if setdefaults { pb.F_BoolDefaulted = Bool(Default_GoTest_F_BoolDefaulted) pb.F_Int32Defaulted = Int32(Default_GoTest_F_Int32Defaulted) pb.F_Int64Defaulted = Int64(Default_GoTest_F_Int64Defaulted) pb.F_Fixed32Defaulted = Uint32(Default_GoTest_F_Fixed32Defaulted) pb.F_Fixed64Defaulted = Uint64(Default_GoTest_F_Fixed64Defaulted) pb.F_Uint32Defaulted = Uint32(Default_GoTest_F_Uint32Defaulted) pb.F_Uint64Defaulted = Uint64(Default_GoTest_F_Uint64Defaulted) pb.F_FloatDefaulted = Float32(Default_GoTest_F_FloatDefaulted) pb.F_DoubleDefaulted = Float64(Default_GoTest_F_DoubleDefaulted) pb.F_StringDefaulted = String(Default_GoTest_F_StringDefaulted) pb.F_BytesDefaulted = Default_GoTest_F_BytesDefaulted pb.F_Sint32Defaulted = Int32(Default_GoTest_F_Sint32Defaulted) pb.F_Sint64Defaulted = Int64(Default_GoTest_F_Sint64Defaulted) } pb.Kind = GoTest_TIME.Enum() pb.RequiredField = initGoTestField() pb.F_BoolRequired = Bool(true) pb.F_Int32Required = Int32(3) pb.F_Int64Required = Int64(6) pb.F_Fixed32Required = Uint32(32) pb.F_Fixed64Required = Uint64(64) pb.F_Uint32Required = Uint32(3232) pb.F_Uint64Required = Uint64(6464) pb.F_FloatRequired = Float32(3232) pb.F_DoubleRequired = Float64(6464) pb.F_StringRequired = String("string") pb.F_BytesRequired = []byte("bytes") pb.F_Sint32Required = Int32(-32) pb.F_Sint64Required = Int64(-64) pb.Requiredgroup = initGoTest_RequiredGroup() return pb } func fail(msg string, b *bytes.Buffer, s string, t *testing.T) { data := b.Bytes() ld := len(data) ls := len(s) / 2 fmt.Printf("fail %s ld=%d ls=%d\n", msg, ld, ls) // find the interesting spot - n n := ls if ld < ls { n = ld } j := 0 for i := 0; i < n; i++ { bs := hex(s[j])*16 + hex(s[j+1]) j += 2 if data[i] == bs { continue } n = i break } l := n - 10 if l < 0 { l = 0 } h := n + 10 // find the interesting spot - n fmt.Printf("is[%d]:", l) for i := l; i < h; i++ { if i >= ld { fmt.Printf(" --") continue } fmt.Printf(" %.2x", data[i]) } fmt.Printf("\n") fmt.Printf("sb[%d]:", l) for i := l; i < h; i++ { if i >= ls { fmt.Printf(" --") continue } bs := hex(s[j])*16 + hex(s[j+1]) j += 2 fmt.Printf(" %.2x", bs) } fmt.Printf("\n") t.Fail() // t.Errorf("%s: \ngood: %s\nbad: %x", msg, s, b.Bytes()) // Print the output in a partially-decoded format; can // be helpful when updating the test. It produces the output // that is pasted, with minor edits, into the argument to verify(). // data := b.Bytes() // nesting := 0 // for b.Len() > 0 { // start := len(data) - b.Len() // var u uint64 // u, err := DecodeVarint(b) // if err != nil { // fmt.Printf("decode error on varint:", err) // return // } // wire := u & 0x7 // tag := u >> 3 // switch wire { // case WireVarint: // v, err := DecodeVarint(b) // if err != nil { // fmt.Printf("decode error on varint:", err) // return // } // fmt.Printf("\t\t\"%x\" // field %d, encoding %d, value %d\n", // data[start:len(data)-b.Len()], tag, wire, v) // case WireFixed32: // v, err := DecodeFixed32(b) // if err != nil { // fmt.Printf("decode error on fixed32:", err) // return // } // fmt.Printf("\t\t\"%x\" // field %d, encoding %d, value %d\n", // data[start:len(data)-b.Len()], tag, wire, v) // case WireFixed64: // v, err := DecodeFixed64(b) // if err != nil { // fmt.Printf("decode error on fixed64:", err) // return // } // fmt.Printf("\t\t\"%x\" // field %d, encoding %d, value %d\n", // data[start:len(data)-b.Len()], tag, wire, v) // case WireBytes: // nb, err := DecodeVarint(b) // if err != nil { // fmt.Printf("decode error on bytes:", err) // return // } // after_tag := len(data) - b.Len() // str := make([]byte, nb) // _, err = b.Read(str) // if err != nil { // fmt.Printf("decode error on bytes:", err) // return // } // fmt.Printf("\t\t\"%x\" \"%x\" // field %d, encoding %d (FIELD)\n", // data[start:after_tag], str, tag, wire) // case WireStartGroup: // nesting++ // fmt.Printf("\t\t\"%x\"\t\t// start group field %d level %d\n", // data[start:len(data)-b.Len()], tag, nesting) // case WireEndGroup: // fmt.Printf("\t\t\"%x\"\t\t// end group field %d level %d\n", // data[start:len(data)-b.Len()], tag, nesting) // nesting-- // default: // fmt.Printf("unrecognized wire type %d\n", wire) // return // } // } } func hex(c uint8) uint8 { if '0' <= c && c <= '9' { return c - '0' } if 'a' <= c && c <= 'f' { return 10 + c - 'a' } if 'A' <= c && c <= 'F' { return 10 + c - 'A' } return 0 } func equal(b []byte, s string, t *testing.T) bool { if 2*len(b) != len(s) { // fail(fmt.Sprintf("wrong lengths: 2*%d != %d", len(b), len(s)), b, s, t) fmt.Printf("wrong lengths: 2*%d != %d\n", len(b), len(s)) return false } for i, j := 0, 0; i < len(b); i, j = i+1, j+2 { x := hex(s[j])*16 + hex(s[j+1]) if b[i] != x { // fail(fmt.Sprintf("bad byte[%d]:%x %x", i, b[i], x), b, s, t) fmt.Printf("bad byte[%d]:%x %x", i, b[i], x) return false } } return true } func overify(t *testing.T, pb *GoTest, expected string) { o := old() err := o.Marshal(pb) if err != nil { fmt.Printf("overify marshal-1 err = %v", err) o.DebugPrint("", o.Bytes()) t.Fatalf("expected = %s", expected) } if !equal(o.Bytes(), expected, t) { o.DebugPrint("overify neq 1", o.Bytes()) t.Fatalf("expected = %s", expected) } // Now test Unmarshal by recreating the original buffer. pbd := new(GoTest) err = o.Unmarshal(pbd) if err != nil { t.Fatalf("overify unmarshal err = %v", err) o.DebugPrint("", o.Bytes()) t.Fatalf("string = %s", expected) } o.Reset() err = o.Marshal(pbd) if err != nil { t.Errorf("overify marshal-2 err = %v", err) o.DebugPrint("", o.Bytes()) t.Fatalf("string = %s", expected) } if !equal(o.Bytes(), expected, t) { o.DebugPrint("overify neq 2", o.Bytes()) t.Fatalf("string = %s", expected) } } // Simple tests for numeric encode/decode primitives (varint, etc.) func TestNumericPrimitives(t *testing.T) { for i := uint64(0); i < 1e6; i += 111 { o := old() if o.EncodeVarint(i) != nil { t.Error("EncodeVarint") break } x, e := o.DecodeVarint() if e != nil { t.Fatal("DecodeVarint") } if x != i { t.Fatal("varint decode fail:", i, x) } o = old() if o.EncodeFixed32(i) != nil { t.Fatal("encFixed32") } x, e = o.DecodeFixed32() if e != nil { t.Fatal("decFixed32") } if x != i { t.Fatal("fixed32 decode fail:", i, x) } o = old() if o.EncodeFixed64(i*1234567) != nil { t.Error("encFixed64") break } x, e = o.DecodeFixed64() if e != nil { t.Error("decFixed64") break } if x != i*1234567 { t.Error("fixed64 decode fail:", i*1234567, x) break } o = old() i32 := int32(i - 12345) if o.EncodeZigzag32(uint64(i32)) != nil { t.Fatal("EncodeZigzag32") } x, e = o.DecodeZigzag32() if e != nil { t.Fatal("DecodeZigzag32") } if x != uint64(uint32(i32)) { t.Fatal("zigzag32 decode fail:", i32, x) } o = old() i64 := int64(i - 12345) if o.EncodeZigzag64(uint64(i64)) != nil { t.Fatal("EncodeZigzag64") } x, e = o.DecodeZigzag64() if e != nil { t.Fatal("DecodeZigzag64") } if x != uint64(i64) { t.Fatal("zigzag64 decode fail:", i64, x) } } } // fakeMarshaler is a simple struct implementing Marshaler and Message interfaces. type fakeMarshaler struct { b []byte err error } func (f *fakeMarshaler) Marshal() ([]byte, error) { return f.b, f.err } func (f *fakeMarshaler) String() string { return fmt.Sprintf("Bytes: %v Error: %v", f.b, f.err) } func (f *fakeMarshaler) ProtoMessage() {} func (f *fakeMarshaler) Reset() {} type msgWithFakeMarshaler struct { M *fakeMarshaler `protobuf:"bytes,1,opt,name=fake"` } func (m *msgWithFakeMarshaler) String() string { return CompactTextString(m) } func (m *msgWithFakeMarshaler) ProtoMessage() {} func (m *msgWithFakeMarshaler) Reset() {} // Simple tests for proto messages that implement the Marshaler interface. func TestMarshalerEncoding(t *testing.T) { tests := []struct { name string m Message want []byte errType reflect.Type }{ { name: "Marshaler that fails", m: &fakeMarshaler{ err: errors.New("some marshal err"), b: []byte{5, 6, 7}, }, // Since the Marshal method returned bytes, they should be written to the // buffer. (For efficiency, we assume that Marshal implementations are // always correct w.r.t. RequiredNotSetError and output.) want: []byte{5, 6, 7}, errType: reflect.TypeOf(errors.New("some marshal err")), }, { name: "Marshaler that fails with RequiredNotSetError", m: &msgWithFakeMarshaler{ M: &fakeMarshaler{ err: &RequiredNotSetError{}, b: []byte{5, 6, 7}, }, }, // Since there's an error that can be continued after, // the buffer should be written. want: []byte{ 10, 3, // for &msgWithFakeMarshaler 5, 6, 7, // for &fakeMarshaler }, errType: reflect.TypeOf(&RequiredNotSetError{}), }, { name: "Marshaler that succeeds", m: &fakeMarshaler{ b: []byte{0, 1, 2, 3, 4, 127, 255}, }, want: []byte{0, 1, 2, 3, 4, 127, 255}, }, } for _, test := range tests { b := NewBuffer(nil) err := b.Marshal(test.m) if reflect.TypeOf(err) != test.errType { t.Errorf("%s: got err %T(%v) wanted %T", test.name, err, err, test.errType) } if !reflect.DeepEqual(test.want, b.Bytes()) { t.Errorf("%s: got bytes %v wanted %v", test.name, b.Bytes(), test.want) } if size := Size(test.m); size != len(b.Bytes()) { t.Errorf("%s: Size(_) = %v, but marshaled to %v bytes", test.name, size, len(b.Bytes())) } m, mErr := Marshal(test.m) if !bytes.Equal(b.Bytes(), m) { t.Errorf("%s: Marshal returned %v, but (*Buffer).Marshal wrote %v", test.name, m, b.Bytes()) } if !reflect.DeepEqual(err, mErr) { t.Errorf("%s: Marshal err = %q, but (*Buffer).Marshal returned %q", test.name, fmt.Sprint(mErr), fmt.Sprint(err)) } } } // Simple tests for bytes func TestBytesPrimitives(t *testing.T) { o := old() bytes := []byte{'n', 'o', 'w', ' ', 'i', 's', ' ', 't', 'h', 'e', ' ', 't', 'i', 'm', 'e'} if o.EncodeRawBytes(bytes) != nil { t.Error("EncodeRawBytes") } decb, e := o.DecodeRawBytes(false) if e != nil { t.Error("DecodeRawBytes") } equalbytes(bytes, decb, t) } // Simple tests for strings func TestStringPrimitives(t *testing.T) { o := old() s := "now is the time" if o.EncodeStringBytes(s) != nil { t.Error("enc_string") } decs, e := o.DecodeStringBytes() if e != nil { t.Error("dec_string") } if s != decs { t.Error("string encode/decode fail:", s, decs) } } // Do we catch the "required bit not set" case? func TestRequiredBit(t *testing.T) { o := old() pb := new(GoTest) err := o.Marshal(pb) if err == nil { t.Error("did not catch missing required fields") } else if strings.Index(err.Error(), "Kind") < 0 { t.Error("wrong error type:", err) } } // Check that all fields are nil. // Clearly silly, and a residue from a more interesting test with an earlier, // different initialization property, but it once caught a compiler bug so // it lives. func checkInitialized(pb *GoTest, t *testing.T) { if pb.F_BoolDefaulted != nil { t.Error("New or Reset did not set boolean:", *pb.F_BoolDefaulted) } if pb.F_Int32Defaulted != nil { t.Error("New or Reset did not set int32:", *pb.F_Int32Defaulted) } if pb.F_Int64Defaulted != nil { t.Error("New or Reset did not set int64:", *pb.F_Int64Defaulted) } if pb.F_Fixed32Defaulted != nil { t.Error("New or Reset did not set fixed32:", *pb.F_Fixed32Defaulted) } if pb.F_Fixed64Defaulted != nil { t.Error("New or Reset did not set fixed64:", *pb.F_Fixed64Defaulted) } if pb.F_Uint32Defaulted != nil { t.Error("New or Reset did not set uint32:", *pb.F_Uint32Defaulted) } if pb.F_Uint64Defaulted != nil { t.Error("New or Reset did not set uint64:", *pb.F_Uint64Defaulted) } if pb.F_FloatDefaulted != nil { t.Error("New or Reset did not set float:", *pb.F_FloatDefaulted) } if pb.F_DoubleDefaulted != nil { t.Error("New or Reset did not set double:", *pb.F_DoubleDefaulted) } if pb.F_StringDefaulted != nil { t.Error("New or Reset did not set string:", *pb.F_StringDefaulted) } if pb.F_BytesDefaulted != nil { t.Error("New or Reset did not set bytes:", string(pb.F_BytesDefaulted)) } if pb.F_Sint32Defaulted != nil { t.Error("New or Reset did not set int32:", *pb.F_Sint32Defaulted) } if pb.F_Sint64Defaulted != nil { t.Error("New or Reset did not set int64:", *pb.F_Sint64Defaulted) } } // Does Reset() reset? func TestReset(t *testing.T) { pb := initGoTest(true) // muck with some values pb.F_BoolDefaulted = Bool(false) pb.F_Int32Defaulted = Int32(237) pb.F_Int64Defaulted = Int64(12346) pb.F_Fixed32Defaulted = Uint32(32000) pb.F_Fixed64Defaulted = Uint64(666) pb.F_Uint32Defaulted = Uint32(323232) pb.F_Uint64Defaulted = nil pb.F_FloatDefaulted = nil pb.F_DoubleDefaulted = Float64(0) pb.F_StringDefaulted = String("gotcha") pb.F_BytesDefaulted = []byte("asdfasdf") pb.F_Sint32Defaulted = Int32(123) pb.F_Sint64Defaulted = Int64(789) pb.Reset() checkInitialized(pb, t) } // All required fields set, no defaults provided. func TestEncodeDecode1(t *testing.T) { pb := initGoTest(false) overify(t, pb, "0807"+ // field 1, encoding 0, value 7 "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) "5001"+ // field 10, encoding 0, value 1 "5803"+ // field 11, encoding 0, value 3 "6006"+ // field 12, encoding 0, value 6 "6d20000000"+ // field 13, encoding 5, value 0x20 "714000000000000000"+ // field 14, encoding 1, value 0x40 "78a019"+ // field 15, encoding 0, value 0xca0 = 3232 "8001c032"+ // field 16, encoding 0, value 0x1940 = 6464 "8d0100004a45"+ // field 17, encoding 5, value 3232.0 "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 "9a0106"+"737472696e67"+ // field 19, encoding 2, string "string" "b304"+ // field 70, encoding 3, start group "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" "b404"+ // field 70, encoding 4, end group "aa0605"+"6279746573"+ // field 101, encoding 2, string "bytes" "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 "b8067f") // field 103, encoding 0, 0x7f zigzag64 } // All required fields set, defaults provided. func TestEncodeDecode2(t *testing.T) { pb := initGoTest(true) overify(t, pb, "0807"+ // field 1, encoding 0, value 7 "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) "5001"+ // field 10, encoding 0, value 1 "5803"+ // field 11, encoding 0, value 3 "6006"+ // field 12, encoding 0, value 6 "6d20000000"+ // field 13, encoding 5, value 32 "714000000000000000"+ // field 14, encoding 1, value 64 "78a019"+ // field 15, encoding 0, value 3232 "8001c032"+ // field 16, encoding 0, value 6464 "8d0100004a45"+ // field 17, encoding 5, value 3232.0 "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" "c00201"+ // field 40, encoding 0, value 1 "c80220"+ // field 41, encoding 0, value 32 "d00240"+ // field 42, encoding 0, value 64 "dd0240010000"+ // field 43, encoding 5, value 320 "e1028002000000000000"+ // field 44, encoding 1, value 640 "e8028019"+ // field 45, encoding 0, value 3200 "f0028032"+ // field 46, encoding 0, value 6400 "fd02e0659948"+ // field 47, encoding 5, value 314159.0 "81030000000050971041"+ // field 48, encoding 1, value 271828.0 "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" "b304"+ // start group field 70 level 1 "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" "b404"+ // end group field 70 level 1 "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" "90193f"+ // field 402, encoding 0, value 63 "98197f") // field 403, encoding 0, value 127 } // All default fields set to their default value by hand func TestEncodeDecode3(t *testing.T) { pb := initGoTest(false) pb.F_BoolDefaulted = Bool(true) pb.F_Int32Defaulted = Int32(32) pb.F_Int64Defaulted = Int64(64) pb.F_Fixed32Defaulted = Uint32(320) pb.F_Fixed64Defaulted = Uint64(640) pb.F_Uint32Defaulted = Uint32(3200) pb.F_Uint64Defaulted = Uint64(6400) pb.F_FloatDefaulted = Float32(314159) pb.F_DoubleDefaulted = Float64(271828) pb.F_StringDefaulted = String("hello, \"world!\"\n") pb.F_BytesDefaulted = []byte("Bignose") pb.F_Sint32Defaulted = Int32(-32) pb.F_Sint64Defaulted = Int64(-64) overify(t, pb, "0807"+ // field 1, encoding 0, value 7 "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) "5001"+ // field 10, encoding 0, value 1 "5803"+ // field 11, encoding 0, value 3 "6006"+ // field 12, encoding 0, value 6 "6d20000000"+ // field 13, encoding 5, value 32 "714000000000000000"+ // field 14, encoding 1, value 64 "78a019"+ // field 15, encoding 0, value 3232 "8001c032"+ // field 16, encoding 0, value 6464 "8d0100004a45"+ // field 17, encoding 5, value 3232.0 "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" "c00201"+ // field 40, encoding 0, value 1 "c80220"+ // field 41, encoding 0, value 32 "d00240"+ // field 42, encoding 0, value 64 "dd0240010000"+ // field 43, encoding 5, value 320 "e1028002000000000000"+ // field 44, encoding 1, value 640 "e8028019"+ // field 45, encoding 0, value 3200 "f0028032"+ // field 46, encoding 0, value 6400 "fd02e0659948"+ // field 47, encoding 5, value 314159.0 "81030000000050971041"+ // field 48, encoding 1, value 271828.0 "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" "b304"+ // start group field 70 level 1 "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" "b404"+ // end group field 70 level 1 "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" "90193f"+ // field 402, encoding 0, value 63 "98197f") // field 403, encoding 0, value 127 } // All required fields set, defaults provided, all non-defaulted optional fields have values. func TestEncodeDecode4(t *testing.T) { pb := initGoTest(true) pb.Table = String("hello") pb.Param = Int32(7) pb.OptionalField = initGoTestField() pb.F_BoolOptional = Bool(true) pb.F_Int32Optional = Int32(32) pb.F_Int64Optional = Int64(64) pb.F_Fixed32Optional = Uint32(3232) pb.F_Fixed64Optional = Uint64(6464) pb.F_Uint32Optional = Uint32(323232) pb.F_Uint64Optional = Uint64(646464) pb.F_FloatOptional = Float32(32.) pb.F_DoubleOptional = Float64(64.) pb.F_StringOptional = String("hello") pb.F_BytesOptional = []byte("Bignose") pb.F_Sint32Optional = Int32(-32) pb.F_Sint64Optional = Int64(-64) pb.Optionalgroup = initGoTest_OptionalGroup() overify(t, pb, "0807"+ // field 1, encoding 0, value 7 "1205"+"68656c6c6f"+ // field 2, encoding 2, string "hello" "1807"+ // field 3, encoding 0, value 7 "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) "320d"+"0a056c6162656c120474797065"+ // field 6, encoding 2 (GoTestField) "5001"+ // field 10, encoding 0, value 1 "5803"+ // field 11, encoding 0, value 3 "6006"+ // field 12, encoding 0, value 6 "6d20000000"+ // field 13, encoding 5, value 32 "714000000000000000"+ // field 14, encoding 1, value 64 "78a019"+ // field 15, encoding 0, value 3232 "8001c032"+ // field 16, encoding 0, value 6464 "8d0100004a45"+ // field 17, encoding 5, value 3232.0 "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" "f00101"+ // field 30, encoding 0, value 1 "f80120"+ // field 31, encoding 0, value 32 "800240"+ // field 32, encoding 0, value 64 "8d02a00c0000"+ // field 33, encoding 5, value 3232 "91024019000000000000"+ // field 34, encoding 1, value 6464 "9802a0dd13"+ // field 35, encoding 0, value 323232 "a002c0ba27"+ // field 36, encoding 0, value 646464 "ad0200000042"+ // field 37, encoding 5, value 32.0 "b1020000000000005040"+ // field 38, encoding 1, value 64.0 "ba0205"+"68656c6c6f"+ // field 39, encoding 2, string "hello" "c00201"+ // field 40, encoding 0, value 1 "c80220"+ // field 41, encoding 0, value 32 "d00240"+ // field 42, encoding 0, value 64 "dd0240010000"+ // field 43, encoding 5, value 320 "e1028002000000000000"+ // field 44, encoding 1, value 640 "e8028019"+ // field 45, encoding 0, value 3200 "f0028032"+ // field 46, encoding 0, value 6400 "fd02e0659948"+ // field 47, encoding 5, value 314159.0 "81030000000050971041"+ // field 48, encoding 1, value 271828.0 "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" "b304"+ // start group field 70 level 1 "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" "b404"+ // end group field 70 level 1 "d305"+ // start group field 90 level 1 "da0508"+"6f7074696f6e616c"+ // field 91, encoding 2, string "optional" "d405"+ // end group field 90 level 1 "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 "ea1207"+"4269676e6f7365"+ // field 301, encoding 2, string "Bignose" "f0123f"+ // field 302, encoding 0, value 63 "f8127f"+ // field 303, encoding 0, value 127 "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" "90193f"+ // field 402, encoding 0, value 63 "98197f") // field 403, encoding 0, value 127 } // All required fields set, defaults provided, all repeated fields given two values. func TestEncodeDecode5(t *testing.T) { pb := initGoTest(true) pb.RepeatedField = []*GoTestField{initGoTestField(), initGoTestField()} pb.F_BoolRepeated = []bool{false, true} pb.F_Int32Repeated = []int32{32, 33} pb.F_Int64Repeated = []int64{64, 65} pb.F_Fixed32Repeated = []uint32{3232, 3333} pb.F_Fixed64Repeated = []uint64{6464, 6565} pb.F_Uint32Repeated = []uint32{323232, 333333} pb.F_Uint64Repeated = []uint64{646464, 656565} pb.F_FloatRepeated = []float32{32., 33.} pb.F_DoubleRepeated = []float64{64., 65.} pb.F_StringRepeated = []string{"hello", "sailor"} pb.F_BytesRepeated = [][]byte{[]byte("big"), []byte("nose")} pb.F_Sint32Repeated = []int32{32, -32} pb.F_Sint64Repeated = []int64{64, -64} pb.Repeatedgroup = []*GoTest_RepeatedGroup{initGoTest_RepeatedGroup(), initGoTest_RepeatedGroup()} overify(t, pb, "0807"+ // field 1, encoding 0, value 7 "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) "2a0d"+"0a056c6162656c120474797065"+ // field 5, encoding 2 (GoTestField) "2a0d"+"0a056c6162656c120474797065"+ // field 5, encoding 2 (GoTestField) "5001"+ // field 10, encoding 0, value 1 "5803"+ // field 11, encoding 0, value 3 "6006"+ // field 12, encoding 0, value 6 "6d20000000"+ // field 13, encoding 5, value 32 "714000000000000000"+ // field 14, encoding 1, value 64 "78a019"+ // field 15, encoding 0, value 3232 "8001c032"+ // field 16, encoding 0, value 6464 "8d0100004a45"+ // field 17, encoding 5, value 3232.0 "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" "a00100"+ // field 20, encoding 0, value 0 "a00101"+ // field 20, encoding 0, value 1 "a80120"+ // field 21, encoding 0, value 32 "a80121"+ // field 21, encoding 0, value 33 "b00140"+ // field 22, encoding 0, value 64 "b00141"+ // field 22, encoding 0, value 65 "bd01a00c0000"+ // field 23, encoding 5, value 3232 "bd01050d0000"+ // field 23, encoding 5, value 3333 "c1014019000000000000"+ // field 24, encoding 1, value 6464 "c101a519000000000000"+ // field 24, encoding 1, value 6565 "c801a0dd13"+ // field 25, encoding 0, value 323232 "c80195ac14"+ // field 25, encoding 0, value 333333 "d001c0ba27"+ // field 26, encoding 0, value 646464 "d001b58928"+ // field 26, encoding 0, value 656565 "dd0100000042"+ // field 27, encoding 5, value 32.0 "dd0100000442"+ // field 27, encoding 5, value 33.0 "e1010000000000005040"+ // field 28, encoding 1, value 64.0 "e1010000000000405040"+ // field 28, encoding 1, value 65.0 "ea0105"+"68656c6c6f"+ // field 29, encoding 2, string "hello" "ea0106"+"7361696c6f72"+ // field 29, encoding 2, string "sailor" "c00201"+ // field 40, encoding 0, value 1 "c80220"+ // field 41, encoding 0, value 32 "d00240"+ // field 42, encoding 0, value 64 "dd0240010000"+ // field 43, encoding 5, value 320 "e1028002000000000000"+ // field 44, encoding 1, value 640 "e8028019"+ // field 45, encoding 0, value 3200 "f0028032"+ // field 46, encoding 0, value 6400 "fd02e0659948"+ // field 47, encoding 5, value 314159.0 "81030000000050971041"+ // field 48, encoding 1, value 271828.0 "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" "b304"+ // start group field 70 level 1 "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" "b404"+ // end group field 70 level 1 "8305"+ // start group field 80 level 1 "8a0508"+"7265706561746564"+ // field 81, encoding 2, string "repeated" "8405"+ // end group field 80 level 1 "8305"+ // start group field 80 level 1 "8a0508"+"7265706561746564"+ // field 81, encoding 2, string "repeated" "8405"+ // end group field 80 level 1 "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 "ca0c03"+"626967"+ // field 201, encoding 2, string "big" "ca0c04"+"6e6f7365"+ // field 201, encoding 2, string "nose" "d00c40"+ // field 202, encoding 0, value 32 "d00c3f"+ // field 202, encoding 0, value -32 "d80c8001"+ // field 203, encoding 0, value 64 "d80c7f"+ // field 203, encoding 0, value -64 "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" "90193f"+ // field 402, encoding 0, value 63 "98197f") // field 403, encoding 0, value 127 } // All required fields set, all packed repeated fields given two values. func TestEncodeDecode6(t *testing.T) { pb := initGoTest(false) pb.F_BoolRepeatedPacked = []bool{false, true} pb.F_Int32RepeatedPacked = []int32{32, 33} pb.F_Int64RepeatedPacked = []int64{64, 65} pb.F_Fixed32RepeatedPacked = []uint32{3232, 3333} pb.F_Fixed64RepeatedPacked = []uint64{6464, 6565} pb.F_Uint32RepeatedPacked = []uint32{323232, 333333} pb.F_Uint64RepeatedPacked = []uint64{646464, 656565} pb.F_FloatRepeatedPacked = []float32{32., 33.} pb.F_DoubleRepeatedPacked = []float64{64., 65.} pb.F_Sint32RepeatedPacked = []int32{32, -32} pb.F_Sint64RepeatedPacked = []int64{64, -64} overify(t, pb, "0807"+ // field 1, encoding 0, value 7 "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) "5001"+ // field 10, encoding 0, value 1 "5803"+ // field 11, encoding 0, value 3 "6006"+ // field 12, encoding 0, value 6 "6d20000000"+ // field 13, encoding 5, value 32 "714000000000000000"+ // field 14, encoding 1, value 64 "78a019"+ // field 15, encoding 0, value 3232 "8001c032"+ // field 16, encoding 0, value 6464 "8d0100004a45"+ // field 17, encoding 5, value 3232.0 "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" "9203020001"+ // field 50, encoding 2, 2 bytes, value 0, value 1 "9a03022021"+ // field 51, encoding 2, 2 bytes, value 32, value 33 "a203024041"+ // field 52, encoding 2, 2 bytes, value 64, value 65 "aa0308"+ // field 53, encoding 2, 8 bytes "a00c0000050d0000"+ // value 3232, value 3333 "b20310"+ // field 54, encoding 2, 16 bytes "4019000000000000a519000000000000"+ // value 6464, value 6565 "ba0306"+ // field 55, encoding 2, 6 bytes "a0dd1395ac14"+ // value 323232, value 333333 "c20306"+ // field 56, encoding 2, 6 bytes "c0ba27b58928"+ // value 646464, value 656565 "ca0308"+ // field 57, encoding 2, 8 bytes "0000004200000442"+ // value 32.0, value 33.0 "d20310"+ // field 58, encoding 2, 16 bytes "00000000000050400000000000405040"+ // value 64.0, value 65.0 "b304"+ // start group field 70 level 1 "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" "b404"+ // end group field 70 level 1 "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 "b21f02"+ // field 502, encoding 2, 2 bytes "403f"+ // value 32, value -32 "ba1f03"+ // field 503, encoding 2, 3 bytes "80017f") // value 64, value -64 } // Test that we can encode empty bytes fields. func TestEncodeDecodeBytes1(t *testing.T) { pb := initGoTest(false) // Create our bytes pb.F_BytesRequired = []byte{} pb.F_BytesRepeated = [][]byte{{}} pb.F_BytesOptional = []byte{} d, err := Marshal(pb) if err != nil { t.Error(err) } pbd := new(GoTest) if err := Unmarshal(d, pbd); err != nil { t.Error(err) } if pbd.F_BytesRequired == nil || len(pbd.F_BytesRequired) != 0 { t.Error("required empty bytes field is incorrect") } if pbd.F_BytesRepeated == nil || len(pbd.F_BytesRepeated) == 1 && pbd.F_BytesRepeated[0] == nil { t.Error("repeated empty bytes field is incorrect") } if pbd.F_BytesOptional == nil || len(pbd.F_BytesOptional) != 0 { t.Error("optional empty bytes field is incorrect") } } // Test that we encode nil-valued fields of a repeated bytes field correctly. // Since entries in a repeated field cannot be nil, nil must mean empty value. func TestEncodeDecodeBytes2(t *testing.T) { pb := initGoTest(false) // Create our bytes pb.F_BytesRepeated = [][]byte{nil} d, err := Marshal(pb) if err != nil { t.Error(err) } pbd := new(GoTest) if err := Unmarshal(d, pbd); err != nil { t.Error(err) } if len(pbd.F_BytesRepeated) != 1 || pbd.F_BytesRepeated[0] == nil { t.Error("Unexpected value for repeated bytes field") } } // All required fields set, defaults provided, all repeated fields given two values. func TestSkippingUnrecognizedFields(t *testing.T) { o := old() pb := initGoTestField() // Marshal it normally. o.Marshal(pb) // Now new a GoSkipTest record. skip := &GoSkipTest{ SkipInt32: Int32(32), SkipFixed32: Uint32(3232), SkipFixed64: Uint64(6464), SkipString: String("skipper"), Skipgroup: &GoSkipTest_SkipGroup{ GroupInt32: Int32(75), GroupString: String("wxyz"), }, } // Marshal it into same buffer. o.Marshal(skip) pbd := new(GoTestField) o.Unmarshal(pbd) // The __unrecognized field should be a marshaling of GoSkipTest skipd := new(GoSkipTest) o.SetBuf(pbd.XXX_unrecognized) o.Unmarshal(skipd) if *skipd.SkipInt32 != *skip.SkipInt32 { t.Error("skip int32", skipd.SkipInt32) } if *skipd.SkipFixed32 != *skip.SkipFixed32 { t.Error("skip fixed32", skipd.SkipFixed32) } if *skipd.SkipFixed64 != *skip.SkipFixed64 { t.Error("skip fixed64", skipd.SkipFixed64) } if *skipd.SkipString != *skip.SkipString { t.Error("skip string", *skipd.SkipString) } if *skipd.Skipgroup.GroupInt32 != *skip.Skipgroup.GroupInt32 { t.Error("skip group int32", skipd.Skipgroup.GroupInt32) } if *skipd.Skipgroup.GroupString != *skip.Skipgroup.GroupString { t.Error("skip group string", *skipd.Skipgroup.GroupString) } } // Check that unrecognized fields of a submessage are preserved. func TestSubmessageUnrecognizedFields(t *testing.T) { nm := &NewMessage{ Nested: &NewMessage_Nested{ Name: String("Nigel"), FoodGroup: String("carbs"), }, } b, err := Marshal(nm) if err != nil { t.Fatalf("Marshal of NewMessage: %v", err) } // Unmarshal into an OldMessage. om := new(OldMessage) if err := Unmarshal(b, om); err != nil { t.Fatalf("Unmarshal to OldMessage: %v", err) } exp := &OldMessage{ Nested: &OldMessage_Nested{ Name: String("Nigel"), // normal protocol buffer users should not do this XXX_unrecognized: []byte("\x12\x05carbs"), }, } if !Equal(om, exp) { t.Errorf("om = %v, want %v", om, exp) } // Clone the OldMessage. om = Clone(om).(*OldMessage) if !Equal(om, exp) { t.Errorf("Clone(om) = %v, want %v", om, exp) } // Marshal the OldMessage, then unmarshal it into an empty NewMessage. if b, err = Marshal(om); err != nil { t.Fatalf("Marshal of OldMessage: %v", err) } t.Logf("Marshal(%v) -> %q", om, b) nm2 := new(NewMessage) if err := Unmarshal(b, nm2); err != nil { t.Fatalf("Unmarshal to NewMessage: %v", err) } if !Equal(nm, nm2) { t.Errorf("NewMessage round-trip: %v => %v", nm, nm2) } } // Check that an int32 field can be upgraded to an int64 field. func TestNegativeInt32(t *testing.T) { om := &OldMessage{ Num: Int32(-1), } b, err := Marshal(om) if err != nil { t.Fatalf("Marshal of OldMessage: %v", err) } // Check the size. It should be 11 bytes; // 1 for the field/wire type, and 10 for the negative number. if len(b) != 11 { t.Errorf("%v marshaled as %q, wanted 11 bytes", om, b) } // Unmarshal into a NewMessage. nm := new(NewMessage) if err := Unmarshal(b, nm); err != nil { t.Fatalf("Unmarshal to NewMessage: %v", err) } want := &NewMessage{ Num: Int64(-1), } if !Equal(nm, want) { t.Errorf("nm = %v, want %v", nm, want) } } // Check that we can grow an array (repeated field) to have many elements. // This test doesn't depend only on our encoding; for variety, it makes sure // we create, encode, and decode the correct contents explicitly. It's therefore // a bit messier. // This test also uses (and hence tests) the Marshal/Unmarshal functions // instead of the methods. func TestBigRepeated(t *testing.T) { pb := initGoTest(true) // Create the arrays const N = 50 // Internally the library starts much smaller. pb.Repeatedgroup = make([]*GoTest_RepeatedGroup, N) pb.F_Sint64Repeated = make([]int64, N) pb.F_Sint32Repeated = make([]int32, N) pb.F_BytesRepeated = make([][]byte, N) pb.F_StringRepeated = make([]string, N) pb.F_DoubleRepeated = make([]float64, N) pb.F_FloatRepeated = make([]float32, N) pb.F_Uint64Repeated = make([]uint64, N) pb.F_Uint32Repeated = make([]uint32, N) pb.F_Fixed64Repeated = make([]uint64, N) pb.F_Fixed32Repeated = make([]uint32, N) pb.F_Int64Repeated = make([]int64, N) pb.F_Int32Repeated = make([]int32, N) pb.F_BoolRepeated = make([]bool, N) pb.RepeatedField = make([]*GoTestField, N) // Fill in the arrays with checkable values. igtf := initGoTestField() igtrg := initGoTest_RepeatedGroup() for i := 0; i < N; i++ { pb.Repeatedgroup[i] = igtrg pb.F_Sint64Repeated[i] = int64(i) pb.F_Sint32Repeated[i] = int32(i) s := fmt.Sprint(i) pb.F_BytesRepeated[i] = []byte(s) pb.F_StringRepeated[i] = s pb.F_DoubleRepeated[i] = float64(i) pb.F_FloatRepeated[i] = float32(i) pb.F_Uint64Repeated[i] = uint64(i) pb.F_Uint32Repeated[i] = uint32(i) pb.F_Fixed64Repeated[i] = uint64(i) pb.F_Fixed32Repeated[i] = uint32(i) pb.F_Int64Repeated[i] = int64(i) pb.F_Int32Repeated[i] = int32(i) pb.F_BoolRepeated[i] = i%2 == 0 pb.RepeatedField[i] = igtf } // Marshal. buf, _ := Marshal(pb) // Now test Unmarshal by recreating the original buffer. pbd := new(GoTest) Unmarshal(buf, pbd) // Check the checkable values for i := uint64(0); i < N; i++ { if pbd.Repeatedgroup[i] == nil { // TODO: more checking? t.Error("pbd.Repeatedgroup bad") } var x uint64 x = uint64(pbd.F_Sint64Repeated[i]) if x != i { t.Error("pbd.F_Sint64Repeated bad", x, i) } x = uint64(pbd.F_Sint32Repeated[i]) if x != i { t.Error("pbd.F_Sint32Repeated bad", x, i) } s := fmt.Sprint(i) equalbytes(pbd.F_BytesRepeated[i], []byte(s), t) if pbd.F_StringRepeated[i] != s { t.Error("pbd.F_Sint32Repeated bad", pbd.F_StringRepeated[i], i) } x = uint64(pbd.F_DoubleRepeated[i]) if x != i { t.Error("pbd.F_DoubleRepeated bad", x, i) } x = uint64(pbd.F_FloatRepeated[i]) if x != i { t.Error("pbd.F_FloatRepeated bad", x, i) } x = pbd.F_Uint64Repeated[i] if x != i { t.Error("pbd.F_Uint64Repeated bad", x, i) } x = uint64(pbd.F_Uint32Repeated[i]) if x != i { t.Error("pbd.F_Uint32Repeated bad", x, i) } x = pbd.F_Fixed64Repeated[i] if x != i { t.Error("pbd.F_Fixed64Repeated bad", x, i) } x = uint64(pbd.F_Fixed32Repeated[i]) if x != i { t.Error("pbd.F_Fixed32Repeated bad", x, i) } x = uint64(pbd.F_Int64Repeated[i]) if x != i { t.Error("pbd.F_Int64Repeated bad", x, i) } x = uint64(pbd.F_Int32Repeated[i]) if x != i { t.Error("pbd.F_Int32Repeated bad", x, i) } if pbd.F_BoolRepeated[i] != (i%2 == 0) { t.Error("pbd.F_BoolRepeated bad", x, i) } if pbd.RepeatedField[i] == nil { // TODO: more checking? t.Error("pbd.RepeatedField bad") } } } // Verify we give a useful message when decoding to the wrong structure type. func TestTypeMismatch(t *testing.T) { pb1 := initGoTest(true) // Marshal o := old() o.Marshal(pb1) // Now Unmarshal it to the wrong type. pb2 := initGoTestField() err := o.Unmarshal(pb2) if err == nil { t.Error("expected error, got no error") } else if !strings.Contains(err.Error(), "bad wiretype") { t.Error("expected bad wiretype error, got", err) } } func encodeDecode(t *testing.T, in, out Message, msg string) { buf, err := Marshal(in) if err != nil { t.Fatalf("failed marshaling %v: %v", msg, err) } if err := Unmarshal(buf, out); err != nil { t.Fatalf("failed unmarshaling %v: %v", msg, err) } } func TestPackedNonPackedDecoderSwitching(t *testing.T) { np, p := new(NonPackedTest), new(PackedTest) // non-packed -> packed np.A = []int32{0, 1, 1, 2, 3, 5} encodeDecode(t, np, p, "non-packed -> packed") if !reflect.DeepEqual(np.A, p.B) { t.Errorf("failed non-packed -> packed; np.A=%+v, p.B=%+v", np.A, p.B) } // packed -> non-packed np.Reset() p.B = []int32{3, 1, 4, 1, 5, 9} encodeDecode(t, p, np, "packed -> non-packed") if !reflect.DeepEqual(p.B, np.A) { t.Errorf("failed packed -> non-packed; p.B=%+v, np.A=%+v", p.B, np.A) } } func TestProto1RepeatedGroup(t *testing.T) { pb := &MessageList{ Message: []*MessageList_Message{ { Name: String("blah"), Count: Int32(7), }, // NOTE: pb.Message[1] is a nil nil, }, } o := old() err := o.Marshal(pb) if err == nil || !strings.Contains(err.Error(), "repeated field Message has nil") { t.Fatalf("unexpected or no error when marshaling: %v", err) } } // Test that enums work. Checks for a bug introduced by making enums // named types instead of int32: newInt32FromUint64 would crash with // a type mismatch in reflect.PointTo. func TestEnum(t *testing.T) { pb := new(GoEnum) pb.Foo = FOO_FOO1.Enum() o := old() if err := o.Marshal(pb); err != nil { t.Fatal("error encoding enum:", err) } pb1 := new(GoEnum) if err := o.Unmarshal(pb1); err != nil { t.Fatal("error decoding enum:", err) } if *pb1.Foo != FOO_FOO1 { t.Error("expected 7 but got ", *pb1.Foo) } } // Enum types have String methods. Check that enum fields can be printed. // We don't care what the value actually is, just as long as it doesn't crash. func TestPrintingNilEnumFields(t *testing.T) { pb := new(GoEnum) _ = fmt.Sprintf("%+v", pb) } // Verify that absent required fields cause Marshal/Unmarshal to return errors. func TestRequiredFieldEnforcement(t *testing.T) { pb := new(GoTestField) _, err := Marshal(pb) if err == nil { t.Error("marshal: expected error, got nil") } else if _, ok := err.(*RequiredNotSetError); !ok || !strings.Contains(err.Error(), "Label") { t.Errorf("marshal: bad error type: %v", err) } // A slightly sneaky, yet valid, proto. It encodes the same required field twice, // so simply counting the required fields is insufficient. // field 1, encoding 2, value "hi" buf := []byte("\x0A\x02hi\x0A\x02hi") err = Unmarshal(buf, pb) if err == nil { t.Error("unmarshal: expected error, got nil") } else if _, ok := err.(*RequiredNotSetError); !ok || !strings.Contains(err.Error(), "{Unknown}") { t.Errorf("unmarshal: bad error type: %v", err) } } // Verify that absent required fields in groups cause Marshal/Unmarshal to return errors. func TestRequiredFieldEnforcementGroups(t *testing.T) { pb := &GoTestRequiredGroupField{Group: &GoTestRequiredGroupField_Group{}} if _, err := Marshal(pb); err == nil { t.Error("marshal: expected error, got nil") } else if _, ok := err.(*RequiredNotSetError); !ok || !strings.Contains(err.Error(), "Group.Field") { t.Errorf("marshal: bad error type: %v", err) } buf := []byte{11, 12} if err := Unmarshal(buf, pb); err == nil { t.Error("unmarshal: expected error, got nil") } else if _, ok := err.(*RequiredNotSetError); !ok || !strings.Contains(err.Error(), "Group.{Unknown}") { t.Errorf("unmarshal: bad error type: %v", err) } } func TestTypedNilMarshal(t *testing.T) { // A typed nil should return ErrNil and not crash. { var m *GoEnum if _, err := Marshal(m); err != ErrNil { t.Errorf("Marshal(%#v): got %v, want ErrNil", m, err) } } { m := &Communique{Union: &Communique_Msg{nil}} if _, err := Marshal(m); err == nil || err == ErrNil { t.Errorf("Marshal(%#v): got %v, want errOneofHasNil", m, err) } } } // A type that implements the Marshaler interface, but is not nillable. type nonNillableInt uint64 func (nni nonNillableInt) Marshal() ([]byte, error) { return EncodeVarint(uint64(nni)), nil } type NNIMessage struct { nni nonNillableInt } func (*NNIMessage) Reset() {} func (*NNIMessage) String() string { return "" } func (*NNIMessage) ProtoMessage() {} // A type that implements the Marshaler interface and is nillable. type nillableMessage struct { x uint64 } func (nm *nillableMessage) Marshal() ([]byte, error) { return EncodeVarint(nm.x), nil } type NMMessage struct { nm *nillableMessage } func (*NMMessage) Reset() {} func (*NMMessage) String() string { return "" } func (*NMMessage) ProtoMessage() {} // Verify a type that uses the Marshaler interface, but has a nil pointer. func TestNilMarshaler(t *testing.T) { // Try a struct with a Marshaler field that is nil. // It should be directly marshable. nmm := new(NMMessage) if _, err := Marshal(nmm); err != nil { t.Error("unexpected error marshaling nmm: ", err) } // Try a struct with a Marshaler field that is not nillable. nnim := new(NNIMessage) nnim.nni = 7 var _ Marshaler = nnim.nni // verify it is truly a Marshaler if _, err := Marshal(nnim); err != nil { t.Error("unexpected error marshaling nnim: ", err) } } func TestAllSetDefaults(t *testing.T) { // Exercise SetDefaults with all scalar field types. m := &Defaults{ // NaN != NaN, so override that here. F_Nan: Float32(1.7), } expected := &Defaults{ F_Bool: Bool(true), F_Int32: Int32(32), F_Int64: Int64(64), F_Fixed32: Uint32(320), F_Fixed64: Uint64(640), F_Uint32: Uint32(3200), F_Uint64: Uint64(6400), F_Float: Float32(314159), F_Double: Float64(271828), F_String: String(`hello, "world!"` + "\n"), F_Bytes: []byte("Bignose"), F_Sint32: Int32(-32), F_Sint64: Int64(-64), F_Enum: Defaults_GREEN.Enum(), F_Pinf: Float32(float32(math.Inf(1))), F_Ninf: Float32(float32(math.Inf(-1))), F_Nan: Float32(1.7), StrZero: String(""), } SetDefaults(m) if !Equal(m, expected) { t.Errorf("SetDefaults failed\n got %v\nwant %v", m, expected) } } func TestSetDefaultsWithSetField(t *testing.T) { // Check that a set value is not overridden. m := &Defaults{ F_Int32: Int32(12), } SetDefaults(m) if v := m.GetF_Int32(); v != 12 { t.Errorf("m.FInt32 = %v, want 12", v) } } func TestSetDefaultsWithSubMessage(t *testing.T) { m := &OtherMessage{ Key: Int64(123), Inner: &InnerMessage{ Host: String("gopher"), }, } expected := &OtherMessage{ Key: Int64(123), Inner: &InnerMessage{ Host: String("gopher"), Port: Int32(4000), }, } SetDefaults(m) if !Equal(m, expected) { t.Errorf("\n got %v\nwant %v", m, expected) } } func TestSetDefaultsWithRepeatedSubMessage(t *testing.T) { m := &MyMessage{ RepInner: []*InnerMessage{{}}, } expected := &MyMessage{ RepInner: []*InnerMessage{{ Port: Int32(4000), }}, } SetDefaults(m) if !Equal(m, expected) { t.Errorf("\n got %v\nwant %v", m, expected) } } func TestSetDefaultWithRepeatedNonMessage(t *testing.T) { m := &MyMessage{ Pet: []string{"turtle", "wombat"}, } expected := Clone(m) SetDefaults(m) if !Equal(m, expected) { t.Errorf("\n got %v\nwant %v", m, expected) } } func TestMaximumTagNumber(t *testing.T) { m := &MaxTag{ LastField: String("natural goat essence"), } buf, err := Marshal(m) if err != nil { t.Fatalf("proto.Marshal failed: %v", err) } m2 := new(MaxTag) if err := Unmarshal(buf, m2); err != nil { t.Fatalf("proto.Unmarshal failed: %v", err) } if got, want := m2.GetLastField(), *m.LastField; got != want { t.Errorf("got %q, want %q", got, want) } } func TestJSON(t *testing.T) { m := &MyMessage{ Count: Int32(4), Pet: []string{"bunny", "kitty"}, Inner: &InnerMessage{ Host: String("cauchy"), }, Bikeshed: MyMessage_GREEN.Enum(), } const expected = `{"count":4,"pet":["bunny","kitty"],"inner":{"host":"cauchy"},"bikeshed":1}` b, err := json.Marshal(m) if err != nil { t.Fatalf("json.Marshal failed: %v", err) } s := string(b) if s != expected { t.Errorf("got %s\nwant %s", s, expected) } received := new(MyMessage) if err := json.Unmarshal(b, received); err != nil { t.Fatalf("json.Unmarshal failed: %v", err) } if !Equal(received, m) { t.Fatalf("got %s, want %s", received, m) } // Test unmarshalling of JSON with symbolic enum name. const old = `{"count":4,"pet":["bunny","kitty"],"inner":{"host":"cauchy"},"bikeshed":"GREEN"}` received.Reset() if err := json.Unmarshal([]byte(old), received); err != nil { t.Fatalf("json.Unmarshal failed: %v", err) } if !Equal(received, m) { t.Fatalf("got %s, want %s", received, m) } } func TestBadWireType(t *testing.T) { b := []byte{7<<3 | 6} // field 7, wire type 6 pb := new(OtherMessage) if err := Unmarshal(b, pb); err == nil { t.Errorf("Unmarshal did not fail") } else if !strings.Contains(err.Error(), "unknown wire type") { t.Errorf("wrong error: %v", err) } } func TestBytesWithInvalidLength(t *testing.T) { // If a byte sequence has an invalid (negative) length, Unmarshal should not panic. b := []byte{2<<3 | WireBytes, 0xff, 0xff, 0xff, 0xff, 0xff, 0} Unmarshal(b, new(MyMessage)) } func TestLengthOverflow(t *testing.T) { // Overflowing a length should not panic. b := []byte{2<<3 | WireBytes, 1, 1, 3<<3 | WireBytes, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x01} Unmarshal(b, new(MyMessage)) } func TestVarintOverflow(t *testing.T) { // Overflowing a 64-bit length should not be allowed. b := []byte{1<<3 | WireVarint, 0x01, 3<<3 | WireBytes, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01} if err := Unmarshal(b, new(MyMessage)); err == nil { t.Fatalf("Overflowed uint64 length without error") } } func TestUnmarshalFuzz(t *testing.T) { const N = 1000 seed := time.Now().UnixNano() t.Logf("RNG seed is %d", seed) rng := rand.New(rand.NewSource(seed)) buf := make([]byte, 20) for i := 0; i < N; i++ { for j := range buf { buf[j] = byte(rng.Intn(256)) } fuzzUnmarshal(t, buf) } } func TestMergeMessages(t *testing.T) { pb := &MessageList{Message: []*MessageList_Message{{Name: String("x"), Count: Int32(1)}}} data, err := Marshal(pb) if err != nil { t.Fatalf("Marshal: %v", err) } pb1 := new(MessageList) if err := Unmarshal(data, pb1); err != nil { t.Fatalf("first Unmarshal: %v", err) } if err := Unmarshal(data, pb1); err != nil { t.Fatalf("second Unmarshal: %v", err) } if len(pb1.Message) != 1 { t.Errorf("two Unmarshals produced %d Messages, want 1", len(pb1.Message)) } pb2 := new(MessageList) if err := UnmarshalMerge(data, pb2); err != nil { t.Fatalf("first UnmarshalMerge: %v", err) } if err := UnmarshalMerge(data, pb2); err != nil { t.Fatalf("second UnmarshalMerge: %v", err) } if len(pb2.Message) != 2 { t.Errorf("two UnmarshalMerges produced %d Messages, want 2", len(pb2.Message)) } } func TestExtensionMarshalOrder(t *testing.T) { m := &MyMessage{Count: Int(123)} if err := SetExtension(m, E_Ext_More, &Ext{Data: String("alpha")}); err != nil { t.Fatalf("SetExtension: %v", err) } if err := SetExtension(m, E_Ext_Text, String("aleph")); err != nil { t.Fatalf("SetExtension: %v", err) } if err := SetExtension(m, E_Ext_Number, Int32(1)); err != nil { t.Fatalf("SetExtension: %v", err) } // Serialize m several times, and check we get the same bytes each time. var orig []byte for i := 0; i < 100; i++ { b, err := Marshal(m) if err != nil { t.Fatalf("Marshal: %v", err) } if i == 0 { orig = b continue } if !bytes.Equal(b, orig) { t.Errorf("Bytes differ on attempt #%d", i) } } } // Many extensions, because small maps might not iterate differently on each iteration. var exts = []*ExtensionDesc{ E_X201, E_X202, E_X203, E_X204, E_X205, E_X206, E_X207, E_X208, E_X209, E_X210, E_X211, E_X212, E_X213, E_X214, E_X215, E_X216, E_X217, E_X218, E_X219, E_X220, E_X221, E_X222, E_X223, E_X224, E_X225, E_X226, E_X227, E_X228, E_X229, E_X230, E_X231, E_X232, E_X233, E_X234, E_X235, E_X236, E_X237, E_X238, E_X239, E_X240, E_X241, E_X242, E_X243, E_X244, E_X245, E_X246, E_X247, E_X248, E_X249, E_X250, } func TestMessageSetMarshalOrder(t *testing.T) { m := &MyMessageSet{} for _, x := range exts { if err := SetExtension(m, x, &Empty{}); err != nil { t.Fatalf("SetExtension: %v", err) } } buf, err := Marshal(m) if err != nil { t.Fatalf("Marshal: %v", err) } // Serialize m several times, and check we get the same bytes each time. for i := 0; i < 10; i++ { b1, err := Marshal(m) if err != nil { t.Fatalf("Marshal: %v", err) } if !bytes.Equal(b1, buf) { t.Errorf("Bytes differ on re-Marshal #%d", i) } m2 := &MyMessageSet{} if err := Unmarshal(buf, m2); err != nil { t.Errorf("Unmarshal: %v", err) } b2, err := Marshal(m2) if err != nil { t.Errorf("re-Marshal: %v", err) } if !bytes.Equal(b2, buf) { t.Errorf("Bytes differ on round-trip #%d", i) } } } func TestUnmarshalMergesMessages(t *testing.T) { // If a nested message occurs twice in the input, // the fields should be merged when decoding. a := &OtherMessage{ Key: Int64(123), Inner: &InnerMessage{ Host: String("polhode"), Port: Int32(1234), }, } aData, err := Marshal(a) if err != nil { t.Fatalf("Marshal(a): %v", err) } b := &OtherMessage{ Weight: Float32(1.2), Inner: &InnerMessage{ Host: String("herpolhode"), Connected: Bool(true), }, } bData, err := Marshal(b) if err != nil { t.Fatalf("Marshal(b): %v", err) } want := &OtherMessage{ Key: Int64(123), Weight: Float32(1.2), Inner: &InnerMessage{ Host: String("herpolhode"), Port: Int32(1234), Connected: Bool(true), }, } got := new(OtherMessage) if err := Unmarshal(append(aData, bData...), got); err != nil { t.Fatalf("Unmarshal: %v", err) } if !Equal(got, want) { t.Errorf("\n got %v\nwant %v", got, want) } } func TestEncodingSizes(t *testing.T) { tests := []struct { m Message n int }{ {&Defaults{F_Int32: Int32(math.MaxInt32)}, 6}, {&Defaults{F_Int32: Int32(math.MinInt32)}, 11}, {&Defaults{F_Uint32: Uint32(uint32(math.MaxInt32) + 1)}, 6}, {&Defaults{F_Uint32: Uint32(math.MaxUint32)}, 6}, } for _, test := range tests { b, err := Marshal(test.m) if err != nil { t.Errorf("Marshal(%v): %v", test.m, err) continue } if len(b) != test.n { t.Errorf("Marshal(%v) yielded %d bytes, want %d bytes", test.m, len(b), test.n) } } } func TestRequiredNotSetError(t *testing.T) { pb := initGoTest(false) pb.RequiredField.Label = nil pb.F_Int32Required = nil pb.F_Int64Required = nil expected := "0807" + // field 1, encoding 0, value 7 "2206" + "120474797065" + // field 4, encoding 2 (GoTestField) "5001" + // field 10, encoding 0, value 1 "6d20000000" + // field 13, encoding 5, value 0x20 "714000000000000000" + // field 14, encoding 1, value 0x40 "78a019" + // field 15, encoding 0, value 0xca0 = 3232 "8001c032" + // field 16, encoding 0, value 0x1940 = 6464 "8d0100004a45" + // field 17, encoding 5, value 3232.0 "9101000000000040b940" + // field 18, encoding 1, value 6464.0 "9a0106" + "737472696e67" + // field 19, encoding 2, string "string" "b304" + // field 70, encoding 3, start group "ba0408" + "7265717569726564" + // field 71, encoding 2, string "required" "b404" + // field 70, encoding 4, end group "aa0605" + "6279746573" + // field 101, encoding 2, string "bytes" "b0063f" + // field 102, encoding 0, 0x3f zigzag32 "b8067f" // field 103, encoding 0, 0x7f zigzag64 o := old() bytes, err := Marshal(pb) if _, ok := err.(*RequiredNotSetError); !ok { fmt.Printf("marshal-1 err = %v, want *RequiredNotSetError", err) o.DebugPrint("", bytes) t.Fatalf("expected = %s", expected) } if strings.Index(err.Error(), "RequiredField.Label") < 0 { t.Errorf("marshal-1 wrong err msg: %v", err) } if !equal(bytes, expected, t) { o.DebugPrint("neq 1", bytes) t.Fatalf("expected = %s", expected) } // Now test Unmarshal by recreating the original buffer. pbd := new(GoTest) err = Unmarshal(bytes, pbd) if _, ok := err.(*RequiredNotSetError); !ok { t.Fatalf("unmarshal err = %v, want *RequiredNotSetError", err) o.DebugPrint("", bytes) t.Fatalf("string = %s", expected) } if strings.Index(err.Error(), "RequiredField.{Unknown}") < 0 { t.Errorf("unmarshal wrong err msg: %v", err) } bytes, err = Marshal(pbd) if _, ok := err.(*RequiredNotSetError); !ok { t.Errorf("marshal-2 err = %v, want *RequiredNotSetError", err) o.DebugPrint("", bytes) t.Fatalf("string = %s", expected) } if strings.Index(err.Error(), "RequiredField.Label") < 0 { t.Errorf("marshal-2 wrong err msg: %v", err) } if !equal(bytes, expected, t) { o.DebugPrint("neq 2", bytes) t.Fatalf("string = %s", expected) } } func fuzzUnmarshal(t *testing.T, data []byte) { defer func() { if e := recover(); e != nil { t.Errorf("These bytes caused a panic: %+v", data) t.Logf("Stack:\n%s", debug.Stack()) t.FailNow() } }() pb := new(MyMessage) Unmarshal(data, pb) } func TestMapFieldMarshal(t *testing.T) { m := &MessageWithMap{ NameMapping: map[int32]string{ 1: "Rob", 4: "Ian", 8: "Dave", }, } b, err := Marshal(m) if err != nil { t.Fatalf("Marshal: %v", err) } // b should be the concatenation of these three byte sequences in some order. parts := []string{ "\n\a\b\x01\x12\x03Rob", "\n\a\b\x04\x12\x03Ian", "\n\b\b\x08\x12\x04Dave", } ok := false for i := range parts { for j := range parts { if j == i { continue } for k := range parts { if k == i || k == j { continue } try := parts[i] + parts[j] + parts[k] if bytes.Equal(b, []byte(try)) { ok = true break } } } } if !ok { t.Fatalf("Incorrect Marshal output.\n got %q\nwant %q (or a permutation of that)", b, parts[0]+parts[1]+parts[2]) } t.Logf("FYI b: %q", b) (new(Buffer)).DebugPrint("Dump of b", b) } func TestMapFieldRoundTrips(t *testing.T) { m := &MessageWithMap{ NameMapping: map[int32]string{ 1: "Rob", 4: "Ian", 8: "Dave", }, MsgMapping: map[int64]*FloatingPoint{ 0x7001: &FloatingPoint{F: Float64(2.0)}, }, ByteMapping: map[bool][]byte{ false: []byte("that's not right!"), true: []byte("aye, 'tis true!"), }, } b, err := Marshal(m) if err != nil { t.Fatalf("Marshal: %v", err) } t.Logf("FYI b: %q", b) m2 := new(MessageWithMap) if err := Unmarshal(b, m2); err != nil { t.Fatalf("Unmarshal: %v", err) } for _, pair := range [][2]interface{}{ {m.NameMapping, m2.NameMapping}, {m.MsgMapping, m2.MsgMapping}, {m.ByteMapping, m2.ByteMapping}, } { if !reflect.DeepEqual(pair[0], pair[1]) { t.Errorf("Map did not survive a round trip.\ninitial: %v\n final: %v", pair[0], pair[1]) } } } func TestMapFieldWithNil(t *testing.T) { m1 := &MessageWithMap{ MsgMapping: map[int64]*FloatingPoint{ 1: nil, }, } b, err := Marshal(m1) if err != nil { t.Fatalf("Marshal: %v", err) } m2 := new(MessageWithMap) if err := Unmarshal(b, m2); err != nil { t.Fatalf("Unmarshal: %v, got these bytes: %v", err, b) } if v, ok := m2.MsgMapping[1]; !ok { t.Error("msg_mapping[1] not present") } else if v != nil { t.Errorf("msg_mapping[1] not nil: %v", v) } } func TestMapFieldWithNilBytes(t *testing.T) { m1 := &MessageWithMap{ ByteMapping: map[bool][]byte{ false: []byte{}, true: nil, }, } n := Size(m1) b, err := Marshal(m1) if err != nil { t.Fatalf("Marshal: %v", err) } if n != len(b) { t.Errorf("Size(m1) = %d; want len(Marshal(m1)) = %d", n, len(b)) } m2 := new(MessageWithMap) if err := Unmarshal(b, m2); err != nil { t.Fatalf("Unmarshal: %v, got these bytes: %v", err, b) } if v, ok := m2.ByteMapping[false]; !ok { t.Error("byte_mapping[false] not present") } else if len(v) != 0 { t.Errorf("byte_mapping[false] not empty: %#v", v) } if v, ok := m2.ByteMapping[true]; !ok { t.Error("byte_mapping[true] not present") } else if len(v) != 0 { t.Errorf("byte_mapping[true] not empty: %#v", v) } } func TestDecodeMapFieldMissingKey(t *testing.T) { b := []byte{ 0x0A, 0x03, // message, tag 1 (name_mapping), of length 3 bytes // no key 0x12, 0x01, 0x6D, // string value of length 1 byte, value "m" } got := &MessageWithMap{} err := Unmarshal(b, got) if err != nil { t.Fatalf("failed to marshal map with missing key: %v", err) } want := &MessageWithMap{NameMapping: map[int32]string{0: "m"}} if !Equal(got, want) { t.Errorf("Unmarshaled map with no key was not as expected. got: %v, want %v", got, want) } } func TestDecodeMapFieldMissingValue(t *testing.T) { b := []byte{ 0x0A, 0x02, // message, tag 1 (name_mapping), of length 2 bytes 0x08, 0x01, // varint key, value 1 // no value } got := &MessageWithMap{} err := Unmarshal(b, got) if err != nil { t.Fatalf("failed to marshal map with missing value: %v", err) } want := &MessageWithMap{NameMapping: map[int32]string{1: ""}} if !Equal(got, want) { t.Errorf("Unmarshaled map with no value was not as expected. got: %v, want %v", got, want) } } func TestOneof(t *testing.T) { m := &Communique{} b, err := Marshal(m) if err != nil { t.Fatalf("Marshal of empty message with oneof: %v", err) } if len(b) != 0 { t.Errorf("Marshal of empty message yielded too many bytes: %v", b) } m = &Communique{ Union: &Communique_Name{"Barry"}, } // Round-trip. b, err = Marshal(m) if err != nil { t.Fatalf("Marshal of message with oneof: %v", err) } if len(b) != 7 { // name tag/wire (1) + name len (1) + name (5) t.Errorf("Incorrect marshal of message with oneof: %v", b) } m.Reset() if err := Unmarshal(b, m); err != nil { t.Fatalf("Unmarshal of message with oneof: %v", err) } if x, ok := m.Union.(*Communique_Name); !ok || x.Name != "Barry" { t.Errorf("After round trip, Union = %+v", m.Union) } if name := m.GetName(); name != "Barry" { t.Errorf("After round trip, GetName = %q, want %q", name, "Barry") } // Let's try with a message in the oneof. m.Union = &Communique_Msg{&Strings{StringField: String("deep deep string")}} b, err = Marshal(m) if err != nil { t.Fatalf("Marshal of message with oneof set to message: %v", err) } if len(b) != 20 { // msg tag/wire (1) + msg len (1) + msg (1 + 1 + 16) t.Errorf("Incorrect marshal of message with oneof set to message: %v", b) } m.Reset() if err := Unmarshal(b, m); err != nil { t.Fatalf("Unmarshal of message with oneof set to message: %v", err) } ss, ok := m.Union.(*Communique_Msg) if !ok || ss.Msg.GetStringField() != "deep deep string" { t.Errorf("After round trip with oneof set to message, Union = %+v", m.Union) } } func TestInefficientPackedBool(t *testing.T) { // https://github.com/golang/protobuf/issues/76 inp := []byte{ 0x12, 0x02, // 0x12 = 2<<3|2; 2 bytes // Usually a bool should take a single byte, // but it is permitted to be any varint. 0xb9, 0x30, } if err := Unmarshal(inp, new(MoreRepeated)); err != nil { t.Error(err) } } // Benchmarks func testMsg() *GoTest { pb := initGoTest(true) const N = 1000 // Internally the library starts much smaller. pb.F_Int32Repeated = make([]int32, N) pb.F_DoubleRepeated = make([]float64, N) for i := 0; i < N; i++ { pb.F_Int32Repeated[i] = int32(i) pb.F_DoubleRepeated[i] = float64(i) } return pb } func bytesMsg() *GoTest { pb := initGoTest(true) buf := make([]byte, 4000) for i := range buf { buf[i] = byte(i) } pb.F_BytesDefaulted = buf return pb } func benchmarkMarshal(b *testing.B, pb Message, marshal func(Message) ([]byte, error)) { d, _ := marshal(pb) b.SetBytes(int64(len(d))) b.ResetTimer() for i := 0; i < b.N; i++ { marshal(pb) } } func benchmarkBufferMarshal(b *testing.B, pb Message) { p := NewBuffer(nil) benchmarkMarshal(b, pb, func(pb0 Message) ([]byte, error) { p.Reset() err := p.Marshal(pb0) return p.Bytes(), err }) } func benchmarkSize(b *testing.B, pb Message) { benchmarkMarshal(b, pb, func(pb0 Message) ([]byte, error) { Size(pb) return nil, nil }) } func newOf(pb Message) Message { in := reflect.ValueOf(pb) if in.IsNil() { return pb } return reflect.New(in.Type().Elem()).Interface().(Message) } func benchmarkUnmarshal(b *testing.B, pb Message, unmarshal func([]byte, Message) error) { d, _ := Marshal(pb) b.SetBytes(int64(len(d))) pbd := newOf(pb) b.ResetTimer() for i := 0; i < b.N; i++ { unmarshal(d, pbd) } } func benchmarkBufferUnmarshal(b *testing.B, pb Message) { p := NewBuffer(nil) benchmarkUnmarshal(b, pb, func(d []byte, pb0 Message) error { p.SetBuf(d) return p.Unmarshal(pb0) }) } // Benchmark{Marshal,BufferMarshal,Size,Unmarshal,BufferUnmarshal}{,Bytes} func BenchmarkMarshal(b *testing.B) { benchmarkMarshal(b, testMsg(), Marshal) } func BenchmarkBufferMarshal(b *testing.B) { benchmarkBufferMarshal(b, testMsg()) } func BenchmarkSize(b *testing.B) { benchmarkSize(b, testMsg()) } func BenchmarkUnmarshal(b *testing.B) { benchmarkUnmarshal(b, testMsg(), Unmarshal) } func BenchmarkBufferUnmarshal(b *testing.B) { benchmarkBufferUnmarshal(b, testMsg()) } func BenchmarkMarshalBytes(b *testing.B) { benchmarkMarshal(b, bytesMsg(), Marshal) } func BenchmarkBufferMarshalBytes(b *testing.B) { benchmarkBufferMarshal(b, bytesMsg()) } func BenchmarkSizeBytes(b *testing.B) { benchmarkSize(b, bytesMsg()) } func BenchmarkUnmarshalBytes(b *testing.B) { benchmarkUnmarshal(b, bytesMsg(), Unmarshal) } func BenchmarkBufferUnmarshalBytes(b *testing.B) { benchmarkBufferUnmarshal(b, bytesMsg()) } func BenchmarkUnmarshalUnrecognizedFields(b *testing.B) { b.StopTimer() pb := initGoTestField() skip := &GoSkipTest{ SkipInt32: Int32(32), SkipFixed32: Uint32(3232), SkipFixed64: Uint64(6464), SkipString: String("skipper"), Skipgroup: &GoSkipTest_SkipGroup{ GroupInt32: Int32(75), GroupString: String("wxyz"), }, } pbd := new(GoTestField) p := NewBuffer(nil) p.Marshal(pb) p.Marshal(skip) p2 := NewBuffer(nil) b.StartTimer() for i := 0; i < b.N; i++ { p2.SetBuf(p.Bytes()) p2.Unmarshal(pbd) } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/any_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2016 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "strings" "testing" "github.com/golang/protobuf/proto" pb "github.com/golang/protobuf/proto/proto3_proto" testpb "github.com/golang/protobuf/proto/testdata" anypb "github.com/golang/protobuf/ptypes/any" ) var ( expandedMarshaler = proto.TextMarshaler{ExpandAny: true} expandedCompactMarshaler = proto.TextMarshaler{Compact: true, ExpandAny: true} ) // anyEqual reports whether two messages which may be google.protobuf.Any or may // contain google.protobuf.Any fields are equal. We can't use proto.Equal for // comparison, because semantically equivalent messages may be marshaled to // binary in different tag order. Instead, trust that TextMarshaler with // ExpandAny option works and compare the text marshaling results. func anyEqual(got, want proto.Message) bool { // if messages are proto.Equal, no need to marshal. if proto.Equal(got, want) { return true } g := expandedMarshaler.Text(got) w := expandedMarshaler.Text(want) return g == w } type golden struct { m proto.Message t, c string } var goldenMessages = makeGolden() func makeGolden() []golden { nested := &pb.Nested{Bunny: "Monty"} nb, err := proto.Marshal(nested) if err != nil { panic(err) } m1 := &pb.Message{ Name: "David", ResultCount: 47, Anything: &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(nested), Value: nb}, } m2 := &pb.Message{ Name: "David", ResultCount: 47, Anything: &anypb.Any{TypeUrl: "http://[::1]/type.googleapis.com/" + proto.MessageName(nested), Value: nb}, } m3 := &pb.Message{ Name: "David", ResultCount: 47, Anything: &anypb.Any{TypeUrl: `type.googleapis.com/"/` + proto.MessageName(nested), Value: nb}, } m4 := &pb.Message{ Name: "David", ResultCount: 47, Anything: &anypb.Any{TypeUrl: "type.googleapis.com/a/path/" + proto.MessageName(nested), Value: nb}, } m5 := &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(nested), Value: nb} any1 := &testpb.MyMessage{Count: proto.Int32(47), Name: proto.String("David")} proto.SetExtension(any1, testpb.E_Ext_More, &testpb.Ext{Data: proto.String("foo")}) proto.SetExtension(any1, testpb.E_Ext_Text, proto.String("bar")) any1b, err := proto.Marshal(any1) if err != nil { panic(err) } any2 := &testpb.MyMessage{Count: proto.Int32(42), Bikeshed: testpb.MyMessage_GREEN.Enum(), RepBytes: [][]byte{[]byte("roboto")}} proto.SetExtension(any2, testpb.E_Ext_More, &testpb.Ext{Data: proto.String("baz")}) any2b, err := proto.Marshal(any2) if err != nil { panic(err) } m6 := &pb.Message{ Name: "David", ResultCount: 47, Anything: &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any1), Value: any1b}, ManyThings: []*anypb.Any{ &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any2), Value: any2b}, &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any1), Value: any1b}, }, } const ( m1Golden = ` name: "David" result_count: 47 anything: < [type.googleapis.com/proto3_proto.Nested]: < bunny: "Monty" > > ` m2Golden = ` name: "David" result_count: 47 anything: < ["http://[::1]/type.googleapis.com/proto3_proto.Nested"]: < bunny: "Monty" > > ` m3Golden = ` name: "David" result_count: 47 anything: < ["type.googleapis.com/\"/proto3_proto.Nested"]: < bunny: "Monty" > > ` m4Golden = ` name: "David" result_count: 47 anything: < [type.googleapis.com/a/path/proto3_proto.Nested]: < bunny: "Monty" > > ` m5Golden = ` [type.googleapis.com/proto3_proto.Nested]: < bunny: "Monty" > ` m6Golden = ` name: "David" result_count: 47 anything: < [type.googleapis.com/testdata.MyMessage]: < count: 47 name: "David" [testdata.Ext.more]: < data: "foo" > [testdata.Ext.text]: "bar" > > many_things: < [type.googleapis.com/testdata.MyMessage]: < count: 42 bikeshed: GREEN rep_bytes: "roboto" [testdata.Ext.more]: < data: "baz" > > > many_things: < [type.googleapis.com/testdata.MyMessage]: < count: 47 name: "David" [testdata.Ext.more]: < data: "foo" > [testdata.Ext.text]: "bar" > > ` ) return []golden{ {m1, strings.TrimSpace(m1Golden) + "\n", strings.TrimSpace(compact(m1Golden)) + " "}, {m2, strings.TrimSpace(m2Golden) + "\n", strings.TrimSpace(compact(m2Golden)) + " "}, {m3, strings.TrimSpace(m3Golden) + "\n", strings.TrimSpace(compact(m3Golden)) + " "}, {m4, strings.TrimSpace(m4Golden) + "\n", strings.TrimSpace(compact(m4Golden)) + " "}, {m5, strings.TrimSpace(m5Golden) + "\n", strings.TrimSpace(compact(m5Golden)) + " "}, {m6, strings.TrimSpace(m6Golden) + "\n", strings.TrimSpace(compact(m6Golden)) + " "}, } } func TestMarshalGolden(t *testing.T) { for _, tt := range goldenMessages { if got, want := expandedMarshaler.Text(tt.m), tt.t; got != want { t.Errorf("message %v: got:\n%s\nwant:\n%s", tt.m, got, want) } if got, want := expandedCompactMarshaler.Text(tt.m), tt.c; got != want { t.Errorf("message %v: got:\n`%s`\nwant:\n`%s`", tt.m, got, want) } } } func TestUnmarshalGolden(t *testing.T) { for _, tt := range goldenMessages { want := tt.m got := proto.Clone(tt.m) got.Reset() if err := proto.UnmarshalText(tt.t, got); err != nil { t.Errorf("failed to unmarshal\n%s\nerror: %v", tt.t, err) } if !anyEqual(got, want) { t.Errorf("message:\n%s\ngot:\n%s\nwant:\n%s", tt.t, got, want) } got.Reset() if err := proto.UnmarshalText(tt.c, got); err != nil { t.Errorf("failed to unmarshal\n%s\nerror: %v", tt.c, err) } if !anyEqual(got, want) { t.Errorf("message:\n%s\ngot:\n%s\nwant:\n%s", tt.c, got, want) } } } func TestMarshalUnknownAny(t *testing.T) { m := &pb.Message{ Anything: &anypb.Any{ TypeUrl: "foo", Value: []byte("bar"), }, } want := `anything: < type_url: "foo" value: "bar" > ` got := expandedMarshaler.Text(m) if got != want { t.Errorf("got\n`%s`\nwant\n`%s`", got, want) } } func TestAmbiguousAny(t *testing.T) { pb := &anypb.Any{} err := proto.UnmarshalText(` type_url: "ttt/proto3_proto.Nested" value: "\n\x05Monty" `, pb) t.Logf("result: %v (error: %v)", expandedMarshaler.Text(pb), err) if err != nil { t.Errorf("failed to parse ambiguous Any message: %v", err) } } func TestUnmarshalOverwriteAny(t *testing.T) { pb := &anypb.Any{} err := proto.UnmarshalText(` [type.googleapis.com/a/path/proto3_proto.Nested]: < bunny: "Monty" > [type.googleapis.com/a/path/proto3_proto.Nested]: < bunny: "Rabbit of Caerbannog" > `, pb) want := `line 7: Any message unpacked multiple times, or "type_url" already set` if err.Error() != want { t.Errorf("incorrect error.\nHave: %v\nWant: %v", err.Error(), want) } } func TestUnmarshalAnyMixAndMatch(t *testing.T) { pb := &anypb.Any{} err := proto.UnmarshalText(` value: "\n\x05Monty" [type.googleapis.com/a/path/proto3_proto.Nested]: < bunny: "Rabbit of Caerbannog" > `, pb) want := `line 5: Any message unpacked multiple times, or "value" already set` if err.Error() != want { t.Errorf("incorrect error.\nHave: %v\nWant: %v", err.Error(), want) } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/clone.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2011 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Protocol buffer deep copy and merge. // TODO: RawMessage. package proto import ( "log" "reflect" "strings" ) // Clone returns a deep copy of a protocol buffer. func Clone(pb Message) Message { in := reflect.ValueOf(pb) if in.IsNil() { return pb } out := reflect.New(in.Type().Elem()) // out is empty so a merge is a deep copy. mergeStruct(out.Elem(), in.Elem()) return out.Interface().(Message) } // Merge merges src into dst. // Required and optional fields that are set in src will be set to that value in dst. // Elements of repeated fields will be appended. // Merge panics if src and dst are not the same type, or if dst is nil. func Merge(dst, src Message) { in := reflect.ValueOf(src) out := reflect.ValueOf(dst) if out.IsNil() { panic("proto: nil destination") } if in.Type() != out.Type() { // Explicit test prior to mergeStruct so that mistyped nils will fail panic("proto: type mismatch") } if in.IsNil() { // Merging nil into non-nil is a quiet no-op return } mergeStruct(out.Elem(), in.Elem()) } func mergeStruct(out, in reflect.Value) { sprop := GetProperties(in.Type()) for i := 0; i < in.NumField(); i++ { f := in.Type().Field(i) if strings.HasPrefix(f.Name, "XXX_") { continue } mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i]) } if emIn, ok := extendable(in.Addr().Interface()); ok { emOut, _ := extendable(out.Addr().Interface()) mIn, muIn := emIn.extensionsRead() if mIn != nil { mOut := emOut.extensionsWrite() muIn.Lock() mergeExtension(mOut, mIn) muIn.Unlock() } } uf := in.FieldByName("XXX_unrecognized") if !uf.IsValid() { return } uin := uf.Bytes() if len(uin) > 0 { out.FieldByName("XXX_unrecognized").SetBytes(append([]byte(nil), uin...)) } } // mergeAny performs a merge between two values of the same type. // viaPtr indicates whether the values were indirected through a pointer (implying proto2). // prop is set if this is a struct field (it may be nil). func mergeAny(out, in reflect.Value, viaPtr bool, prop *Properties) { if in.Type() == protoMessageType { if !in.IsNil() { if out.IsNil() { out.Set(reflect.ValueOf(Clone(in.Interface().(Message)))) } else { Merge(out.Interface().(Message), in.Interface().(Message)) } } return } switch in.Kind() { case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64, reflect.String, reflect.Uint32, reflect.Uint64: if !viaPtr && isProto3Zero(in) { return } out.Set(in) case reflect.Interface: // Probably a oneof field; copy non-nil values. if in.IsNil() { return } // Allocate destination if it is not set, or set to a different type. // Otherwise we will merge as normal. if out.IsNil() || out.Elem().Type() != in.Elem().Type() { out.Set(reflect.New(in.Elem().Elem().Type())) // interface -> *T -> T -> new(T) } mergeAny(out.Elem(), in.Elem(), false, nil) case reflect.Map: if in.Len() == 0 { return } if out.IsNil() { out.Set(reflect.MakeMap(in.Type())) } // For maps with value types of *T or []byte we need to deep copy each value. elemKind := in.Type().Elem().Kind() for _, key := range in.MapKeys() { var val reflect.Value switch elemKind { case reflect.Ptr: val = reflect.New(in.Type().Elem().Elem()) mergeAny(val, in.MapIndex(key), false, nil) case reflect.Slice: val = in.MapIndex(key) val = reflect.ValueOf(append([]byte{}, val.Bytes()...)) default: val = in.MapIndex(key) } out.SetMapIndex(key, val) } case reflect.Ptr: if in.IsNil() { return } if out.IsNil() { out.Set(reflect.New(in.Elem().Type())) } mergeAny(out.Elem(), in.Elem(), true, nil) case reflect.Slice: if in.IsNil() { return } if in.Type().Elem().Kind() == reflect.Uint8 { // []byte is a scalar bytes field, not a repeated field. // Edge case: if this is in a proto3 message, a zero length // bytes field is considered the zero value, and should not // be merged. if prop != nil && prop.proto3 && in.Len() == 0 { return } // Make a deep copy. // Append to []byte{} instead of []byte(nil) so that we never end up // with a nil result. out.SetBytes(append([]byte{}, in.Bytes()...)) return } n := in.Len() if out.IsNil() { out.Set(reflect.MakeSlice(in.Type(), 0, n)) } switch in.Type().Elem().Kind() { case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64, reflect.String, reflect.Uint32, reflect.Uint64: out.Set(reflect.AppendSlice(out, in)) default: for i := 0; i < n; i++ { x := reflect.Indirect(reflect.New(in.Type().Elem())) mergeAny(x, in.Index(i), false, nil) out.Set(reflect.Append(out, x)) } } case reflect.Struct: mergeStruct(out, in) default: // unknown type, so not a protocol buffer log.Printf("proto: don't know how to copy %v", in) } } func mergeExtension(out, in map[int32]Extension) { for extNum, eIn := range in { eOut := Extension{desc: eIn.desc} if eIn.value != nil { v := reflect.New(reflect.TypeOf(eIn.value)).Elem() mergeAny(v, reflect.ValueOf(eIn.value), false, nil) eOut.value = v.Interface() } if eIn.enc != nil { eOut.enc = make([]byte, len(eIn.enc)) copy(eOut.enc, eIn.enc) } out[extNum] = eOut } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/clone_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2011 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "testing" "github.com/golang/protobuf/proto" proto3pb "github.com/golang/protobuf/proto/proto3_proto" pb "github.com/golang/protobuf/proto/testdata" ) var cloneTestMessage = &pb.MyMessage{ Count: proto.Int32(42), Name: proto.String("Dave"), Pet: []string{"bunny", "kitty", "horsey"}, Inner: &pb.InnerMessage{ Host: proto.String("niles"), Port: proto.Int32(9099), Connected: proto.Bool(true), }, Others: []*pb.OtherMessage{ { Value: []byte("some bytes"), }, }, Somegroup: &pb.MyMessage_SomeGroup{ GroupField: proto.Int32(6), }, RepBytes: [][]byte{[]byte("sham"), []byte("wow")}, } func init() { ext := &pb.Ext{ Data: proto.String("extension"), } if err := proto.SetExtension(cloneTestMessage, pb.E_Ext_More, ext); err != nil { panic("SetExtension: " + err.Error()) } } func TestClone(t *testing.T) { m := proto.Clone(cloneTestMessage).(*pb.MyMessage) if !proto.Equal(m, cloneTestMessage) { t.Errorf("Clone(%v) = %v", cloneTestMessage, m) } // Verify it was a deep copy. *m.Inner.Port++ if proto.Equal(m, cloneTestMessage) { t.Error("Mutating clone changed the original") } // Byte fields and repeated fields should be copied. if &m.Pet[0] == &cloneTestMessage.Pet[0] { t.Error("Pet: repeated field not copied") } if &m.Others[0] == &cloneTestMessage.Others[0] { t.Error("Others: repeated field not copied") } if &m.Others[0].Value[0] == &cloneTestMessage.Others[0].Value[0] { t.Error("Others[0].Value: bytes field not copied") } if &m.RepBytes[0] == &cloneTestMessage.RepBytes[0] { t.Error("RepBytes: repeated field not copied") } if &m.RepBytes[0][0] == &cloneTestMessage.RepBytes[0][0] { t.Error("RepBytes[0]: bytes field not copied") } } func TestCloneNil(t *testing.T) { var m *pb.MyMessage if c := proto.Clone(m); !proto.Equal(m, c) { t.Errorf("Clone(%v) = %v", m, c) } } var mergeTests = []struct { src, dst, want proto.Message }{ { src: &pb.MyMessage{ Count: proto.Int32(42), }, dst: &pb.MyMessage{ Name: proto.String("Dave"), }, want: &pb.MyMessage{ Count: proto.Int32(42), Name: proto.String("Dave"), }, }, { src: &pb.MyMessage{ Inner: &pb.InnerMessage{ Host: proto.String("hey"), Connected: proto.Bool(true), }, Pet: []string{"horsey"}, Others: []*pb.OtherMessage{ { Value: []byte("some bytes"), }, }, }, dst: &pb.MyMessage{ Inner: &pb.InnerMessage{ Host: proto.String("niles"), Port: proto.Int32(9099), }, Pet: []string{"bunny", "kitty"}, Others: []*pb.OtherMessage{ { Key: proto.Int64(31415926535), }, { // Explicitly test a src=nil field Inner: nil, }, }, }, want: &pb.MyMessage{ Inner: &pb.InnerMessage{ Host: proto.String("hey"), Connected: proto.Bool(true), Port: proto.Int32(9099), }, Pet: []string{"bunny", "kitty", "horsey"}, Others: []*pb.OtherMessage{ { Key: proto.Int64(31415926535), }, {}, { Value: []byte("some bytes"), }, }, }, }, { src: &pb.MyMessage{ RepBytes: [][]byte{[]byte("wow")}, }, dst: &pb.MyMessage{ Somegroup: &pb.MyMessage_SomeGroup{ GroupField: proto.Int32(6), }, RepBytes: [][]byte{[]byte("sham")}, }, want: &pb.MyMessage{ Somegroup: &pb.MyMessage_SomeGroup{ GroupField: proto.Int32(6), }, RepBytes: [][]byte{[]byte("sham"), []byte("wow")}, }, }, // Check that a scalar bytes field replaces rather than appends. { src: &pb.OtherMessage{Value: []byte("foo")}, dst: &pb.OtherMessage{Value: []byte("bar")}, want: &pb.OtherMessage{Value: []byte("foo")}, }, { src: &pb.MessageWithMap{ NameMapping: map[int32]string{6: "Nigel"}, MsgMapping: map[int64]*pb.FloatingPoint{ 0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)}, 0x4002: &pb.FloatingPoint{ F: proto.Float64(2.0), }, }, ByteMapping: map[bool][]byte{true: []byte("wowsa")}, }, dst: &pb.MessageWithMap{ NameMapping: map[int32]string{ 6: "Bruce", // should be overwritten 7: "Andrew", }, MsgMapping: map[int64]*pb.FloatingPoint{ 0x4002: &pb.FloatingPoint{ F: proto.Float64(3.0), Exact: proto.Bool(true), }, // the entire message should be overwritten }, }, want: &pb.MessageWithMap{ NameMapping: map[int32]string{ 6: "Nigel", 7: "Andrew", }, MsgMapping: map[int64]*pb.FloatingPoint{ 0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)}, 0x4002: &pb.FloatingPoint{ F: proto.Float64(2.0), }, }, ByteMapping: map[bool][]byte{true: []byte("wowsa")}, }, }, // proto3 shouldn't merge zero values, // in the same way that proto2 shouldn't merge nils. { src: &proto3pb.Message{ Name: "Aaron", Data: []byte(""), // zero value, but not nil }, dst: &proto3pb.Message{ HeightInCm: 176, Data: []byte("texas!"), }, want: &proto3pb.Message{ Name: "Aaron", HeightInCm: 176, Data: []byte("texas!"), }, }, // Oneof fields should merge by assignment. { src: &pb.Communique{ Union: &pb.Communique_Number{41}, }, dst: &pb.Communique{ Union: &pb.Communique_Name{"Bobby Tables"}, }, want: &pb.Communique{ Union: &pb.Communique_Number{41}, }, }, // Oneof nil is the same as not set. { src: &pb.Communique{}, dst: &pb.Communique{ Union: &pb.Communique_Name{"Bobby Tables"}, }, want: &pb.Communique{ Union: &pb.Communique_Name{"Bobby Tables"}, }, }, { src: &proto3pb.Message{ Terrain: map[string]*proto3pb.Nested{ "kay_a": &proto3pb.Nested{Cute: true}, // replace "kay_b": &proto3pb.Nested{Bunny: "rabbit"}, // insert }, }, dst: &proto3pb.Message{ Terrain: map[string]*proto3pb.Nested{ "kay_a": &proto3pb.Nested{Bunny: "lost"}, // replaced "kay_c": &proto3pb.Nested{Bunny: "bunny"}, // keep }, }, want: &proto3pb.Message{ Terrain: map[string]*proto3pb.Nested{ "kay_a": &proto3pb.Nested{Cute: true}, "kay_b": &proto3pb.Nested{Bunny: "rabbit"}, "kay_c": &proto3pb.Nested{Bunny: "bunny"}, }, }, }, } func TestMerge(t *testing.T) { for _, m := range mergeTests { got := proto.Clone(m.dst) proto.Merge(got, m.src) if !proto.Equal(got, m.want) { t.Errorf("Merge(%v, %v)\n got %v\nwant %v\n", m.dst, m.src, got, m.want) } } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/decode.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto /* * Routines for decoding protocol buffer data to construct in-memory representations. */ import ( "errors" "fmt" "io" "os" "reflect" ) // errOverflow is returned when an integer is too large to be represented. var errOverflow = errors.New("proto: integer overflow") // ErrInternalBadWireType is returned by generated code when an incorrect // wire type is encountered. It does not get returned to user code. var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof") // The fundamental decoders that interpret bytes on the wire. // Those that take integer types all return uint64 and are // therefore of type valueDecoder. // DecodeVarint reads a varint-encoded integer from the slice. // It returns the integer and the number of bytes consumed, or // zero if there is not enough. // This is the format for the // int32, int64, uint32, uint64, bool, and enum // protocol buffer types. func DecodeVarint(buf []byte) (x uint64, n int) { for shift := uint(0); shift < 64; shift += 7 { if n >= len(buf) { return 0, 0 } b := uint64(buf[n]) n++ x |= (b & 0x7F) << shift if (b & 0x80) == 0 { return x, n } } // The number is too large to represent in a 64-bit value. return 0, 0 } func (p *Buffer) decodeVarintSlow() (x uint64, err error) { i := p.index l := len(p.buf) for shift := uint(0); shift < 64; shift += 7 { if i >= l { err = io.ErrUnexpectedEOF return } b := p.buf[i] i++ x |= (uint64(b) & 0x7F) << shift if b < 0x80 { p.index = i return } } // The number is too large to represent in a 64-bit value. err = errOverflow return } // DecodeVarint reads a varint-encoded integer from the Buffer. // This is the format for the // int32, int64, uint32, uint64, bool, and enum // protocol buffer types. func (p *Buffer) DecodeVarint() (x uint64, err error) { i := p.index buf := p.buf if i >= len(buf) { return 0, io.ErrUnexpectedEOF } else if buf[i] < 0x80 { p.index++ return uint64(buf[i]), nil } else if len(buf)-i < 10 { return p.decodeVarintSlow() } var b uint64 // we already checked the first byte x = uint64(buf[i]) - 0x80 i++ b = uint64(buf[i]) i++ x += b << 7 if b&0x80 == 0 { goto done } x -= 0x80 << 7 b = uint64(buf[i]) i++ x += b << 14 if b&0x80 == 0 { goto done } x -= 0x80 << 14 b = uint64(buf[i]) i++ x += b << 21 if b&0x80 == 0 { goto done } x -= 0x80 << 21 b = uint64(buf[i]) i++ x += b << 28 if b&0x80 == 0 { goto done } x -= 0x80 << 28 b = uint64(buf[i]) i++ x += b << 35 if b&0x80 == 0 { goto done } x -= 0x80 << 35 b = uint64(buf[i]) i++ x += b << 42 if b&0x80 == 0 { goto done } x -= 0x80 << 42 b = uint64(buf[i]) i++ x += b << 49 if b&0x80 == 0 { goto done } x -= 0x80 << 49 b = uint64(buf[i]) i++ x += b << 56 if b&0x80 == 0 { goto done } x -= 0x80 << 56 b = uint64(buf[i]) i++ x += b << 63 if b&0x80 == 0 { goto done } // x -= 0x80 << 63 // Always zero. return 0, errOverflow done: p.index = i return x, nil } // DecodeFixed64 reads a 64-bit integer from the Buffer. // This is the format for the // fixed64, sfixed64, and double protocol buffer types. func (p *Buffer) DecodeFixed64() (x uint64, err error) { // x, err already 0 i := p.index + 8 if i < 0 || i > len(p.buf) { err = io.ErrUnexpectedEOF return } p.index = i x = uint64(p.buf[i-8]) x |= uint64(p.buf[i-7]) << 8 x |= uint64(p.buf[i-6]) << 16 x |= uint64(p.buf[i-5]) << 24 x |= uint64(p.buf[i-4]) << 32 x |= uint64(p.buf[i-3]) << 40 x |= uint64(p.buf[i-2]) << 48 x |= uint64(p.buf[i-1]) << 56 return } // DecodeFixed32 reads a 32-bit integer from the Buffer. // This is the format for the // fixed32, sfixed32, and float protocol buffer types. func (p *Buffer) DecodeFixed32() (x uint64, err error) { // x, err already 0 i := p.index + 4 if i < 0 || i > len(p.buf) { err = io.ErrUnexpectedEOF return } p.index = i x = uint64(p.buf[i-4]) x |= uint64(p.buf[i-3]) << 8 x |= uint64(p.buf[i-2]) << 16 x |= uint64(p.buf[i-1]) << 24 return } // DecodeZigzag64 reads a zigzag-encoded 64-bit integer // from the Buffer. // This is the format used for the sint64 protocol buffer type. func (p *Buffer) DecodeZigzag64() (x uint64, err error) { x, err = p.DecodeVarint() if err != nil { return } x = (x >> 1) ^ uint64((int64(x&1)<<63)>>63) return } // DecodeZigzag32 reads a zigzag-encoded 32-bit integer // from the Buffer. // This is the format used for the sint32 protocol buffer type. func (p *Buffer) DecodeZigzag32() (x uint64, err error) { x, err = p.DecodeVarint() if err != nil { return } x = uint64((uint32(x) >> 1) ^ uint32((int32(x&1)<<31)>>31)) return } // These are not ValueDecoders: they produce an array of bytes or a string. // bytes, embedded messages // DecodeRawBytes reads a count-delimited byte buffer from the Buffer. // This is the format used for the bytes protocol buffer // type and for embedded messages. func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err error) { n, err := p.DecodeVarint() if err != nil { return nil, err } nb := int(n) if nb < 0 { return nil, fmt.Errorf("proto: bad byte length %d", nb) } end := p.index + nb if end < p.index || end > len(p.buf) { return nil, io.ErrUnexpectedEOF } if !alloc { // todo: check if can get more uses of alloc=false buf = p.buf[p.index:end] p.index += nb return } buf = make([]byte, nb) copy(buf, p.buf[p.index:]) p.index += nb return } // DecodeStringBytes reads an encoded string from the Buffer. // This is the format used for the proto2 string type. func (p *Buffer) DecodeStringBytes() (s string, err error) { buf, err := p.DecodeRawBytes(false) if err != nil { return } return string(buf), nil } // Skip the next item in the buffer. Its wire type is decoded and presented as an argument. // If the protocol buffer has extensions, and the field matches, add it as an extension. // Otherwise, if the XXX_unrecognized field exists, append the skipped data there. func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error { oi := o.index err := o.skip(t, tag, wire) if err != nil { return err } if !unrecField.IsValid() { return nil } ptr := structPointer_Bytes(base, unrecField) // Add the skipped field to struct field obuf := o.buf o.buf = *ptr o.EncodeVarint(uint64(tag<<3 | wire)) *ptr = append(o.buf, obuf[oi:o.index]...) o.buf = obuf return nil } // Skip the next item in the buffer. Its wire type is decoded and presented as an argument. func (o *Buffer) skip(t reflect.Type, tag, wire int) error { var u uint64 var err error switch wire { case WireVarint: _, err = o.DecodeVarint() case WireFixed64: _, err = o.DecodeFixed64() case WireBytes: _, err = o.DecodeRawBytes(false) case WireFixed32: _, err = o.DecodeFixed32() case WireStartGroup: for { u, err = o.DecodeVarint() if err != nil { break } fwire := int(u & 0x7) if fwire == WireEndGroup { break } ftag := int(u >> 3) err = o.skip(t, ftag, fwire) if err != nil { break } } default: err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t) } return err } // Unmarshaler is the interface representing objects that can // unmarshal themselves. The method should reset the receiver before // decoding starts. The argument points to data that may be // overwritten, so implementations should not keep references to the // buffer. type Unmarshaler interface { Unmarshal([]byte) error } // Unmarshal parses the protocol buffer representation in buf and places the // decoded result in pb. If the struct underlying pb does not match // the data in buf, the results can be unpredictable. // // Unmarshal resets pb before starting to unmarshal, so any // existing data in pb is always removed. Use UnmarshalMerge // to preserve and append to existing data. func Unmarshal(buf []byte, pb Message) error { pb.Reset() return UnmarshalMerge(buf, pb) } // UnmarshalMerge parses the protocol buffer representation in buf and // writes the decoded result to pb. If the struct underlying pb does not match // the data in buf, the results can be unpredictable. // // UnmarshalMerge merges into existing data in pb. // Most code should use Unmarshal instead. func UnmarshalMerge(buf []byte, pb Message) error { // If the object can unmarshal itself, let it. if u, ok := pb.(Unmarshaler); ok { return u.Unmarshal(buf) } return NewBuffer(buf).Unmarshal(pb) } // DecodeMessage reads a count-delimited message from the Buffer. func (p *Buffer) DecodeMessage(pb Message) error { enc, err := p.DecodeRawBytes(false) if err != nil { return err } return NewBuffer(enc).Unmarshal(pb) } // DecodeGroup reads a tag-delimited group from the Buffer. func (p *Buffer) DecodeGroup(pb Message) error { typ, base, err := getbase(pb) if err != nil { return err } return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base) } // Unmarshal parses the protocol buffer representation in the // Buffer and places the decoded result in pb. If the struct // underlying pb does not match the data in the buffer, the results can be // unpredictable. // // Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal. func (p *Buffer) Unmarshal(pb Message) error { // If the object can unmarshal itself, let it. if u, ok := pb.(Unmarshaler); ok { err := u.Unmarshal(p.buf[p.index:]) p.index = len(p.buf) return err } typ, base, err := getbase(pb) if err != nil { return err } err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base) if collectStats { stats.Decode++ } return err } // unmarshalType does the work of unmarshaling a structure. func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error { var state errorState required, reqFields := prop.reqCount, uint64(0) var err error for err == nil && o.index < len(o.buf) { oi := o.index var u uint64 u, err = o.DecodeVarint() if err != nil { break } wire := int(u & 0x7) if wire == WireEndGroup { if is_group { if required > 0 { // Not enough information to determine the exact field. // (See below.) return &RequiredNotSetError{"{Unknown}"} } return nil // input is satisfied } return fmt.Errorf("proto: %s: wiretype end group for non-group", st) } tag := int(u >> 3) if tag <= 0 { return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire) } fieldnum, ok := prop.decoderTags.get(tag) if !ok { // Maybe it's an extension? if prop.extendable { if e, _ := extendable(structPointer_Interface(base, st)); isExtensionField(e, int32(tag)) { if err = o.skip(st, tag, wire); err == nil { extmap := e.extensionsWrite() ext := extmap[int32(tag)] // may be missing ext.enc = append(ext.enc, o.buf[oi:o.index]...) extmap[int32(tag)] = ext } continue } } // Maybe it's a oneof? if prop.oneofUnmarshaler != nil { m := structPointer_Interface(base, st).(Message) // First return value indicates whether tag is a oneof field. ok, err = prop.oneofUnmarshaler(m, tag, wire, o) if err == ErrInternalBadWireType { // Map the error to something more descriptive. // Do the formatting here to save generated code space. err = fmt.Errorf("bad wiretype for oneof field in %T", m) } if ok { continue } } err = o.skipAndSave(st, tag, wire, base, prop.unrecField) continue } p := prop.Prop[fieldnum] if p.dec == nil { fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name) continue } dec := p.dec if wire != WireStartGroup && wire != p.WireType { if wire == WireBytes && p.packedDec != nil { // a packable field dec = p.packedDec } else { err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType) continue } } decErr := dec(o, p, base) if decErr != nil && !state.shouldContinue(decErr, p) { err = decErr } if err == nil && p.Required { // Successfully decoded a required field. if tag <= 64 { // use bitmap for fields 1-64 to catch field reuse. var mask uint64 = 1 << uint64(tag-1) if reqFields&mask == 0 { // new required field reqFields |= mask required-- } } else { // This is imprecise. It can be fooled by a required field // with a tag > 64 that is encoded twice; that's very rare. // A fully correct implementation would require allocating // a data structure, which we would like to avoid. required-- } } } if err == nil { if is_group { return io.ErrUnexpectedEOF } if state.err != nil { return state.err } if required > 0 { // Not enough information to determine the exact field. If we use extra // CPU, we could determine the field only if the missing required field // has a tag <= 64 and we check reqFields. return &RequiredNotSetError{"{Unknown}"} } } return err } // Individual type decoders // For each, // u is the decoded value, // v is a pointer to the field (pointer) in the struct // Sizes of the pools to allocate inside the Buffer. // The goal is modest amortization and allocation // on at least 16-byte boundaries. const ( boolPoolSize = 16 uint32PoolSize = 8 uint64PoolSize = 4 ) // Decode a bool. func (o *Buffer) dec_bool(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } if len(o.bools) == 0 { o.bools = make([]bool, boolPoolSize) } o.bools[0] = u != 0 *structPointer_Bool(base, p.field) = &o.bools[0] o.bools = o.bools[1:] return nil } func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } *structPointer_BoolVal(base, p.field) = u != 0 return nil } // Decode an int32. func (o *Buffer) dec_int32(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } word32_Set(structPointer_Word32(base, p.field), o, uint32(u)) return nil } func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u)) return nil } // Decode an int64. func (o *Buffer) dec_int64(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } word64_Set(structPointer_Word64(base, p.field), o, u) return nil } func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } word64Val_Set(structPointer_Word64Val(base, p.field), o, u) return nil } // Decode a string. func (o *Buffer) dec_string(p *Properties, base structPointer) error { s, err := o.DecodeStringBytes() if err != nil { return err } *structPointer_String(base, p.field) = &s return nil } func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error { s, err := o.DecodeStringBytes() if err != nil { return err } *structPointer_StringVal(base, p.field) = s return nil } // Decode a slice of bytes ([]byte). func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error { b, err := o.DecodeRawBytes(true) if err != nil { return err } *structPointer_Bytes(base, p.field) = b return nil } // Decode a slice of bools ([]bool). func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } v := structPointer_BoolSlice(base, p.field) *v = append(*v, u != 0) return nil } // Decode a slice of bools ([]bool) in packed format. func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error { v := structPointer_BoolSlice(base, p.field) nn, err := o.DecodeVarint() if err != nil { return err } nb := int(nn) // number of bytes of encoded bools fin := o.index + nb if fin < o.index { return errOverflow } y := *v for o.index < fin { u, err := p.valDec(o) if err != nil { return err } y = append(y, u != 0) } *v = y return nil } // Decode a slice of int32s ([]int32). func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } structPointer_Word32Slice(base, p.field).Append(uint32(u)) return nil } // Decode a slice of int32s ([]int32) in packed format. func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error { v := structPointer_Word32Slice(base, p.field) nn, err := o.DecodeVarint() if err != nil { return err } nb := int(nn) // number of bytes of encoded int32s fin := o.index + nb if fin < o.index { return errOverflow } for o.index < fin { u, err := p.valDec(o) if err != nil { return err } v.Append(uint32(u)) } return nil } // Decode a slice of int64s ([]int64). func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error { u, err := p.valDec(o) if err != nil { return err } structPointer_Word64Slice(base, p.field).Append(u) return nil } // Decode a slice of int64s ([]int64) in packed format. func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error { v := structPointer_Word64Slice(base, p.field) nn, err := o.DecodeVarint() if err != nil { return err } nb := int(nn) // number of bytes of encoded int64s fin := o.index + nb if fin < o.index { return errOverflow } for o.index < fin { u, err := p.valDec(o) if err != nil { return err } v.Append(u) } return nil } // Decode a slice of strings ([]string). func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error { s, err := o.DecodeStringBytes() if err != nil { return err } v := structPointer_StringSlice(base, p.field) *v = append(*v, s) return nil } // Decode a slice of slice of bytes ([][]byte). func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error { b, err := o.DecodeRawBytes(true) if err != nil { return err } v := structPointer_BytesSlice(base, p.field) *v = append(*v, b) return nil } // Decode a map field. func (o *Buffer) dec_new_map(p *Properties, base structPointer) error { raw, err := o.DecodeRawBytes(false) if err != nil { return err } oi := o.index // index at the end of this map entry o.index -= len(raw) // move buffer back to start of map entry mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V if mptr.Elem().IsNil() { mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem())) } v := mptr.Elem() // map[K]V // Prepare addressable doubly-indirect placeholders for the key and value types. // See enc_new_map for why. keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K keybase := toStructPointer(keyptr.Addr()) // **K var valbase structPointer var valptr reflect.Value switch p.mtype.Elem().Kind() { case reflect.Slice: // []byte var dummy []byte valptr = reflect.ValueOf(&dummy) // *[]byte valbase = toStructPointer(valptr) // *[]byte case reflect.Ptr: // message; valptr is **Msg; need to allocate the intermediate pointer valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V valptr.Set(reflect.New(valptr.Type().Elem())) valbase = toStructPointer(valptr) default: // everything else valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V valbase = toStructPointer(valptr.Addr()) // **V } // Decode. // This parses a restricted wire format, namely the encoding of a message // with two fields. See enc_new_map for the format. for o.index < oi { // tagcode for key and value properties are always a single byte // because they have tags 1 and 2. tagcode := o.buf[o.index] o.index++ switch tagcode { case p.mkeyprop.tagcode[0]: if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil { return err } case p.mvalprop.tagcode[0]: if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil { return err } default: // TODO: Should we silently skip this instead? return fmt.Errorf("proto: bad map data tag %d", raw[0]) } } keyelem, valelem := keyptr.Elem(), valptr.Elem() if !keyelem.IsValid() { keyelem = reflect.Zero(p.mtype.Key()) } if !valelem.IsValid() { valelem = reflect.Zero(p.mtype.Elem()) } v.SetMapIndex(keyelem, valelem) return nil } // Decode a group. func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error { bas := structPointer_GetStructPointer(base, p.field) if structPointer_IsNil(bas) { // allocate new nested message bas = toStructPointer(reflect.New(p.stype)) structPointer_SetStructPointer(base, p.field, bas) } return o.unmarshalType(p.stype, p.sprop, true, bas) } // Decode an embedded message. func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) { raw, e := o.DecodeRawBytes(false) if e != nil { return e } bas := structPointer_GetStructPointer(base, p.field) if structPointer_IsNil(bas) { // allocate new nested message bas = toStructPointer(reflect.New(p.stype)) structPointer_SetStructPointer(base, p.field, bas) } // If the object can unmarshal itself, let it. if p.isUnmarshaler { iv := structPointer_Interface(bas, p.stype) return iv.(Unmarshaler).Unmarshal(raw) } obuf := o.buf oi := o.index o.buf = raw o.index = 0 err = o.unmarshalType(p.stype, p.sprop, false, bas) o.buf = obuf o.index = oi return err } // Decode a slice of embedded messages. func (o *Buffer) dec_slice_struct_message(p *Properties, base structPointer) error { return o.dec_slice_struct(p, false, base) } // Decode a slice of embedded groups. func (o *Buffer) dec_slice_struct_group(p *Properties, base structPointer) error { return o.dec_slice_struct(p, true, base) } // Decode a slice of structs ([]*struct). func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base structPointer) error { v := reflect.New(p.stype) bas := toStructPointer(v) structPointer_StructPointerSlice(base, p.field).Append(bas) if is_group { err := o.unmarshalType(p.stype, p.sprop, is_group, bas) return err } raw, err := o.DecodeRawBytes(false) if err != nil { return err } // If the object can unmarshal itself, let it. if p.isUnmarshaler { iv := v.Interface() return iv.(Unmarshaler).Unmarshal(raw) } obuf := o.buf oi := o.index o.buf = raw o.index = 0 err = o.unmarshalType(p.stype, p.sprop, is_group, bas) o.buf = obuf o.index = oi return err } ================================================ FILE: vendor/github.com/golang/protobuf/proto/decode_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +build go1.7 package proto_test import ( "fmt" "testing" "github.com/golang/protobuf/proto" tpb "github.com/golang/protobuf/proto/proto3_proto" ) var ( bytesBlackhole []byte msgBlackhole = new(tpb.Message) ) // BenchmarkVarint32ArraySmall shows the performance on an array of small int32 fields (1 and // 2 bytes long). func BenchmarkVarint32ArraySmall(b *testing.B) { for i := uint(1); i <= 10; i++ { dist := genInt32Dist([7]int{0, 3, 1}, 1<2GB. ErrTooLarge = errors.New("proto: message encodes to over 2 GB") ) // The fundamental encoders that put bytes on the wire. // Those that take integer types all accept uint64 and are // therefore of type valueEncoder. const maxVarintBytes = 10 // maximum length of a varint // maxMarshalSize is the largest allowed size of an encoded protobuf, // since C++ and Java use signed int32s for the size. const maxMarshalSize = 1<<31 - 1 // EncodeVarint returns the varint encoding of x. // This is the format for the // int32, int64, uint32, uint64, bool, and enum // protocol buffer types. // Not used by the package itself, but helpful to clients // wishing to use the same encoding. func EncodeVarint(x uint64) []byte { var buf [maxVarintBytes]byte var n int for n = 0; x > 127; n++ { buf[n] = 0x80 | uint8(x&0x7F) x >>= 7 } buf[n] = uint8(x) n++ return buf[0:n] } // EncodeVarint writes a varint-encoded integer to the Buffer. // This is the format for the // int32, int64, uint32, uint64, bool, and enum // protocol buffer types. func (p *Buffer) EncodeVarint(x uint64) error { for x >= 1<<7 { p.buf = append(p.buf, uint8(x&0x7f|0x80)) x >>= 7 } p.buf = append(p.buf, uint8(x)) return nil } // SizeVarint returns the varint encoding size of an integer. func SizeVarint(x uint64) int { return sizeVarint(x) } func sizeVarint(x uint64) (n int) { for { n++ x >>= 7 if x == 0 { break } } return n } // EncodeFixed64 writes a 64-bit integer to the Buffer. // This is the format for the // fixed64, sfixed64, and double protocol buffer types. func (p *Buffer) EncodeFixed64(x uint64) error { p.buf = append(p.buf, uint8(x), uint8(x>>8), uint8(x>>16), uint8(x>>24), uint8(x>>32), uint8(x>>40), uint8(x>>48), uint8(x>>56)) return nil } func sizeFixed64(x uint64) int { return 8 } // EncodeFixed32 writes a 32-bit integer to the Buffer. // This is the format for the // fixed32, sfixed32, and float protocol buffer types. func (p *Buffer) EncodeFixed32(x uint64) error { p.buf = append(p.buf, uint8(x), uint8(x>>8), uint8(x>>16), uint8(x>>24)) return nil } func sizeFixed32(x uint64) int { return 4 } // EncodeZigzag64 writes a zigzag-encoded 64-bit integer // to the Buffer. // This is the format used for the sint64 protocol buffer type. func (p *Buffer) EncodeZigzag64(x uint64) error { // use signed number to get arithmetic right shift. return p.EncodeVarint((x << 1) ^ uint64((int64(x) >> 63))) } func sizeZigzag64(x uint64) int { return sizeVarint((x << 1) ^ uint64((int64(x) >> 63))) } // EncodeZigzag32 writes a zigzag-encoded 32-bit integer // to the Buffer. // This is the format used for the sint32 protocol buffer type. func (p *Buffer) EncodeZigzag32(x uint64) error { // use signed number to get arithmetic right shift. return p.EncodeVarint(uint64((uint32(x) << 1) ^ uint32((int32(x) >> 31)))) } func sizeZigzag32(x uint64) int { return sizeVarint(uint64((uint32(x) << 1) ^ uint32((int32(x) >> 31)))) } // EncodeRawBytes writes a count-delimited byte buffer to the Buffer. // This is the format used for the bytes protocol buffer // type and for embedded messages. func (p *Buffer) EncodeRawBytes(b []byte) error { p.EncodeVarint(uint64(len(b))) p.buf = append(p.buf, b...) return nil } func sizeRawBytes(b []byte) int { return sizeVarint(uint64(len(b))) + len(b) } // EncodeStringBytes writes an encoded string to the Buffer. // This is the format used for the proto2 string type. func (p *Buffer) EncodeStringBytes(s string) error { p.EncodeVarint(uint64(len(s))) p.buf = append(p.buf, s...) return nil } func sizeStringBytes(s string) int { return sizeVarint(uint64(len(s))) + len(s) } // Marshaler is the interface representing objects that can marshal themselves. type Marshaler interface { Marshal() ([]byte, error) } // Marshal takes the protocol buffer // and encodes it into the wire format, returning the data. func Marshal(pb Message) ([]byte, error) { // Can the object marshal itself? if m, ok := pb.(Marshaler); ok { return m.Marshal() } p := NewBuffer(nil) err := p.Marshal(pb) if p.buf == nil && err == nil { // Return a non-nil slice on success. return []byte{}, nil } return p.buf, err } // EncodeMessage writes the protocol buffer to the Buffer, // prefixed by a varint-encoded length. func (p *Buffer) EncodeMessage(pb Message) error { t, base, err := getbase(pb) if structPointer_IsNil(base) { return ErrNil } if err == nil { var state errorState err = p.enc_len_struct(GetProperties(t.Elem()), base, &state) } return err } // Marshal takes the protocol buffer // and encodes it into the wire format, writing the result to the // Buffer. func (p *Buffer) Marshal(pb Message) error { // Can the object marshal itself? if m, ok := pb.(Marshaler); ok { data, err := m.Marshal() p.buf = append(p.buf, data...) return err } t, base, err := getbase(pb) if structPointer_IsNil(base) { return ErrNil } if err == nil { err = p.enc_struct(GetProperties(t.Elem()), base) } if collectStats { (stats).Encode++ // Parens are to work around a goimports bug. } if len(p.buf) > maxMarshalSize { return ErrTooLarge } return err } // Size returns the encoded size of a protocol buffer. func Size(pb Message) (n int) { // Can the object marshal itself? If so, Size is slow. // TODO: add Size to Marshaler, or add a Sizer interface. if m, ok := pb.(Marshaler); ok { b, _ := m.Marshal() return len(b) } t, base, err := getbase(pb) if structPointer_IsNil(base) { return 0 } if err == nil { n = size_struct(GetProperties(t.Elem()), base) } if collectStats { (stats).Size++ // Parens are to work around a goimports bug. } return } // Individual type encoders. // Encode a bool. func (o *Buffer) enc_bool(p *Properties, base structPointer) error { v := *structPointer_Bool(base, p.field) if v == nil { return ErrNil } x := 0 if *v { x = 1 } o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func (o *Buffer) enc_proto3_bool(p *Properties, base structPointer) error { v := *structPointer_BoolVal(base, p.field) if !v { return ErrNil } o.buf = append(o.buf, p.tagcode...) p.valEnc(o, 1) return nil } func size_bool(p *Properties, base structPointer) int { v := *structPointer_Bool(base, p.field) if v == nil { return 0 } return len(p.tagcode) + 1 // each bool takes exactly one byte } func size_proto3_bool(p *Properties, base structPointer) int { v := *structPointer_BoolVal(base, p.field) if !v && !p.oneof { return 0 } return len(p.tagcode) + 1 // each bool takes exactly one byte } // Encode an int32. func (o *Buffer) enc_int32(p *Properties, base structPointer) error { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { return ErrNil } x := int32(word32_Get(v)) // permit sign extension to use full 64-bit range o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func (o *Buffer) enc_proto3_int32(p *Properties, base structPointer) error { v := structPointer_Word32Val(base, p.field) x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range if x == 0 { return ErrNil } o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func size_int32(p *Properties, base structPointer) (n int) { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { return 0 } x := int32(word32_Get(v)) // permit sign extension to use full 64-bit range n += len(p.tagcode) n += p.valSize(uint64(x)) return } func size_proto3_int32(p *Properties, base structPointer) (n int) { v := structPointer_Word32Val(base, p.field) x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range if x == 0 && !p.oneof { return 0 } n += len(p.tagcode) n += p.valSize(uint64(x)) return } // Encode a uint32. // Exactly the same as int32, except for no sign extension. func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { return ErrNil } x := word32_Get(v) o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func (o *Buffer) enc_proto3_uint32(p *Properties, base structPointer) error { v := structPointer_Word32Val(base, p.field) x := word32Val_Get(v) if x == 0 { return ErrNil } o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func size_uint32(p *Properties, base structPointer) (n int) { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { return 0 } x := word32_Get(v) n += len(p.tagcode) n += p.valSize(uint64(x)) return } func size_proto3_uint32(p *Properties, base structPointer) (n int) { v := structPointer_Word32Val(base, p.field) x := word32Val_Get(v) if x == 0 && !p.oneof { return 0 } n += len(p.tagcode) n += p.valSize(uint64(x)) return } // Encode an int64. func (o *Buffer) enc_int64(p *Properties, base structPointer) error { v := structPointer_Word64(base, p.field) if word64_IsNil(v) { return ErrNil } x := word64_Get(v) o.buf = append(o.buf, p.tagcode...) p.valEnc(o, x) return nil } func (o *Buffer) enc_proto3_int64(p *Properties, base structPointer) error { v := structPointer_Word64Val(base, p.field) x := word64Val_Get(v) if x == 0 { return ErrNil } o.buf = append(o.buf, p.tagcode...) p.valEnc(o, x) return nil } func size_int64(p *Properties, base structPointer) (n int) { v := structPointer_Word64(base, p.field) if word64_IsNil(v) { return 0 } x := word64_Get(v) n += len(p.tagcode) n += p.valSize(x) return } func size_proto3_int64(p *Properties, base structPointer) (n int) { v := structPointer_Word64Val(base, p.field) x := word64Val_Get(v) if x == 0 && !p.oneof { return 0 } n += len(p.tagcode) n += p.valSize(x) return } // Encode a string. func (o *Buffer) enc_string(p *Properties, base structPointer) error { v := *structPointer_String(base, p.field) if v == nil { return ErrNil } x := *v o.buf = append(o.buf, p.tagcode...) o.EncodeStringBytes(x) return nil } func (o *Buffer) enc_proto3_string(p *Properties, base structPointer) error { v := *structPointer_StringVal(base, p.field) if v == "" { return ErrNil } o.buf = append(o.buf, p.tagcode...) o.EncodeStringBytes(v) return nil } func size_string(p *Properties, base structPointer) (n int) { v := *structPointer_String(base, p.field) if v == nil { return 0 } x := *v n += len(p.tagcode) n += sizeStringBytes(x) return } func size_proto3_string(p *Properties, base structPointer) (n int) { v := *structPointer_StringVal(base, p.field) if v == "" && !p.oneof { return 0 } n += len(p.tagcode) n += sizeStringBytes(v) return } // All protocol buffer fields are nillable, but be careful. func isNil(v reflect.Value) bool { switch v.Kind() { case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: return v.IsNil() } return false } // Encode a message struct. func (o *Buffer) enc_struct_message(p *Properties, base structPointer) error { var state errorState structp := structPointer_GetStructPointer(base, p.field) if structPointer_IsNil(structp) { return ErrNil } // Can the object marshal itself? if p.isMarshaler { m := structPointer_Interface(structp, p.stype).(Marshaler) data, err := m.Marshal() if err != nil && !state.shouldContinue(err, nil) { return err } o.buf = append(o.buf, p.tagcode...) o.EncodeRawBytes(data) return state.err } o.buf = append(o.buf, p.tagcode...) return o.enc_len_struct(p.sprop, structp, &state) } func size_struct_message(p *Properties, base structPointer) int { structp := structPointer_GetStructPointer(base, p.field) if structPointer_IsNil(structp) { return 0 } // Can the object marshal itself? if p.isMarshaler { m := structPointer_Interface(structp, p.stype).(Marshaler) data, _ := m.Marshal() n0 := len(p.tagcode) n1 := sizeRawBytes(data) return n0 + n1 } n0 := len(p.tagcode) n1 := size_struct(p.sprop, structp) n2 := sizeVarint(uint64(n1)) // size of encoded length return n0 + n1 + n2 } // Encode a group struct. func (o *Buffer) enc_struct_group(p *Properties, base structPointer) error { var state errorState b := structPointer_GetStructPointer(base, p.field) if structPointer_IsNil(b) { return ErrNil } o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) err := o.enc_struct(p.sprop, b) if err != nil && !state.shouldContinue(err, nil) { return err } o.EncodeVarint(uint64((p.Tag << 3) | WireEndGroup)) return state.err } func size_struct_group(p *Properties, base structPointer) (n int) { b := structPointer_GetStructPointer(base, p.field) if structPointer_IsNil(b) { return 0 } n += sizeVarint(uint64((p.Tag << 3) | WireStartGroup)) n += size_struct(p.sprop, b) n += sizeVarint(uint64((p.Tag << 3) | WireEndGroup)) return } // Encode a slice of bools ([]bool). func (o *Buffer) enc_slice_bool(p *Properties, base structPointer) error { s := *structPointer_BoolSlice(base, p.field) l := len(s) if l == 0 { return ErrNil } for _, x := range s { o.buf = append(o.buf, p.tagcode...) v := uint64(0) if x { v = 1 } p.valEnc(o, v) } return nil } func size_slice_bool(p *Properties, base structPointer) int { s := *structPointer_BoolSlice(base, p.field) l := len(s) if l == 0 { return 0 } return l * (len(p.tagcode) + 1) // each bool takes exactly one byte } // Encode a slice of bools ([]bool) in packed format. func (o *Buffer) enc_slice_packed_bool(p *Properties, base structPointer) error { s := *structPointer_BoolSlice(base, p.field) l := len(s) if l == 0 { return ErrNil } o.buf = append(o.buf, p.tagcode...) o.EncodeVarint(uint64(l)) // each bool takes exactly one byte for _, x := range s { v := uint64(0) if x { v = 1 } p.valEnc(o, v) } return nil } func size_slice_packed_bool(p *Properties, base structPointer) (n int) { s := *structPointer_BoolSlice(base, p.field) l := len(s) if l == 0 { return 0 } n += len(p.tagcode) n += sizeVarint(uint64(l)) n += l // each bool takes exactly one byte return } // Encode a slice of bytes ([]byte). func (o *Buffer) enc_slice_byte(p *Properties, base structPointer) error { s := *structPointer_Bytes(base, p.field) if s == nil { return ErrNil } o.buf = append(o.buf, p.tagcode...) o.EncodeRawBytes(s) return nil } func (o *Buffer) enc_proto3_slice_byte(p *Properties, base structPointer) error { s := *structPointer_Bytes(base, p.field) if len(s) == 0 { return ErrNil } o.buf = append(o.buf, p.tagcode...) o.EncodeRawBytes(s) return nil } func size_slice_byte(p *Properties, base structPointer) (n int) { s := *structPointer_Bytes(base, p.field) if s == nil && !p.oneof { return 0 } n += len(p.tagcode) n += sizeRawBytes(s) return } func size_proto3_slice_byte(p *Properties, base structPointer) (n int) { s := *structPointer_Bytes(base, p.field) if len(s) == 0 && !p.oneof { return 0 } n += len(p.tagcode) n += sizeRawBytes(s) return } // Encode a slice of int32s ([]int32). func (o *Buffer) enc_slice_int32(p *Properties, base structPointer) error { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return ErrNil } for i := 0; i < l; i++ { o.buf = append(o.buf, p.tagcode...) x := int32(s.Index(i)) // permit sign extension to use full 64-bit range p.valEnc(o, uint64(x)) } return nil } func size_slice_int32(p *Properties, base structPointer) (n int) { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return 0 } for i := 0; i < l; i++ { n += len(p.tagcode) x := int32(s.Index(i)) // permit sign extension to use full 64-bit range n += p.valSize(uint64(x)) } return } // Encode a slice of int32s ([]int32) in packed format. func (o *Buffer) enc_slice_packed_int32(p *Properties, base structPointer) error { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return ErrNil } // TODO: Reuse a Buffer. buf := NewBuffer(nil) for i := 0; i < l; i++ { x := int32(s.Index(i)) // permit sign extension to use full 64-bit range p.valEnc(buf, uint64(x)) } o.buf = append(o.buf, p.tagcode...) o.EncodeVarint(uint64(len(buf.buf))) o.buf = append(o.buf, buf.buf...) return nil } func size_slice_packed_int32(p *Properties, base structPointer) (n int) { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return 0 } var bufSize int for i := 0; i < l; i++ { x := int32(s.Index(i)) // permit sign extension to use full 64-bit range bufSize += p.valSize(uint64(x)) } n += len(p.tagcode) n += sizeVarint(uint64(bufSize)) n += bufSize return } // Encode a slice of uint32s ([]uint32). // Exactly the same as int32, except for no sign extension. func (o *Buffer) enc_slice_uint32(p *Properties, base structPointer) error { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return ErrNil } for i := 0; i < l; i++ { o.buf = append(o.buf, p.tagcode...) x := s.Index(i) p.valEnc(o, uint64(x)) } return nil } func size_slice_uint32(p *Properties, base structPointer) (n int) { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return 0 } for i := 0; i < l; i++ { n += len(p.tagcode) x := s.Index(i) n += p.valSize(uint64(x)) } return } // Encode a slice of uint32s ([]uint32) in packed format. // Exactly the same as int32, except for no sign extension. func (o *Buffer) enc_slice_packed_uint32(p *Properties, base structPointer) error { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return ErrNil } // TODO: Reuse a Buffer. buf := NewBuffer(nil) for i := 0; i < l; i++ { p.valEnc(buf, uint64(s.Index(i))) } o.buf = append(o.buf, p.tagcode...) o.EncodeVarint(uint64(len(buf.buf))) o.buf = append(o.buf, buf.buf...) return nil } func size_slice_packed_uint32(p *Properties, base structPointer) (n int) { s := structPointer_Word32Slice(base, p.field) l := s.Len() if l == 0 { return 0 } var bufSize int for i := 0; i < l; i++ { bufSize += p.valSize(uint64(s.Index(i))) } n += len(p.tagcode) n += sizeVarint(uint64(bufSize)) n += bufSize return } // Encode a slice of int64s ([]int64). func (o *Buffer) enc_slice_int64(p *Properties, base structPointer) error { s := structPointer_Word64Slice(base, p.field) l := s.Len() if l == 0 { return ErrNil } for i := 0; i < l; i++ { o.buf = append(o.buf, p.tagcode...) p.valEnc(o, s.Index(i)) } return nil } func size_slice_int64(p *Properties, base structPointer) (n int) { s := structPointer_Word64Slice(base, p.field) l := s.Len() if l == 0 { return 0 } for i := 0; i < l; i++ { n += len(p.tagcode) n += p.valSize(s.Index(i)) } return } // Encode a slice of int64s ([]int64) in packed format. func (o *Buffer) enc_slice_packed_int64(p *Properties, base structPointer) error { s := structPointer_Word64Slice(base, p.field) l := s.Len() if l == 0 { return ErrNil } // TODO: Reuse a Buffer. buf := NewBuffer(nil) for i := 0; i < l; i++ { p.valEnc(buf, s.Index(i)) } o.buf = append(o.buf, p.tagcode...) o.EncodeVarint(uint64(len(buf.buf))) o.buf = append(o.buf, buf.buf...) return nil } func size_slice_packed_int64(p *Properties, base structPointer) (n int) { s := structPointer_Word64Slice(base, p.field) l := s.Len() if l == 0 { return 0 } var bufSize int for i := 0; i < l; i++ { bufSize += p.valSize(s.Index(i)) } n += len(p.tagcode) n += sizeVarint(uint64(bufSize)) n += bufSize return } // Encode a slice of slice of bytes ([][]byte). func (o *Buffer) enc_slice_slice_byte(p *Properties, base structPointer) error { ss := *structPointer_BytesSlice(base, p.field) l := len(ss) if l == 0 { return ErrNil } for i := 0; i < l; i++ { o.buf = append(o.buf, p.tagcode...) o.EncodeRawBytes(ss[i]) } return nil } func size_slice_slice_byte(p *Properties, base structPointer) (n int) { ss := *structPointer_BytesSlice(base, p.field) l := len(ss) if l == 0 { return 0 } n += l * len(p.tagcode) for i := 0; i < l; i++ { n += sizeRawBytes(ss[i]) } return } // Encode a slice of strings ([]string). func (o *Buffer) enc_slice_string(p *Properties, base structPointer) error { ss := *structPointer_StringSlice(base, p.field) l := len(ss) for i := 0; i < l; i++ { o.buf = append(o.buf, p.tagcode...) o.EncodeStringBytes(ss[i]) } return nil } func size_slice_string(p *Properties, base structPointer) (n int) { ss := *structPointer_StringSlice(base, p.field) l := len(ss) n += l * len(p.tagcode) for i := 0; i < l; i++ { n += sizeStringBytes(ss[i]) } return } // Encode a slice of message structs ([]*struct). func (o *Buffer) enc_slice_struct_message(p *Properties, base structPointer) error { var state errorState s := structPointer_StructPointerSlice(base, p.field) l := s.Len() for i := 0; i < l; i++ { structp := s.Index(i) if structPointer_IsNil(structp) { return errRepeatedHasNil } // Can the object marshal itself? if p.isMarshaler { m := structPointer_Interface(structp, p.stype).(Marshaler) data, err := m.Marshal() if err != nil && !state.shouldContinue(err, nil) { return err } o.buf = append(o.buf, p.tagcode...) o.EncodeRawBytes(data) continue } o.buf = append(o.buf, p.tagcode...) err := o.enc_len_struct(p.sprop, structp, &state) if err != nil && !state.shouldContinue(err, nil) { if err == ErrNil { return errRepeatedHasNil } return err } } return state.err } func size_slice_struct_message(p *Properties, base structPointer) (n int) { s := structPointer_StructPointerSlice(base, p.field) l := s.Len() n += l * len(p.tagcode) for i := 0; i < l; i++ { structp := s.Index(i) if structPointer_IsNil(structp) { return // return the size up to this point } // Can the object marshal itself? if p.isMarshaler { m := structPointer_Interface(structp, p.stype).(Marshaler) data, _ := m.Marshal() n += sizeRawBytes(data) continue } n0 := size_struct(p.sprop, structp) n1 := sizeVarint(uint64(n0)) // size of encoded length n += n0 + n1 } return } // Encode a slice of group structs ([]*struct). func (o *Buffer) enc_slice_struct_group(p *Properties, base structPointer) error { var state errorState s := structPointer_StructPointerSlice(base, p.field) l := s.Len() for i := 0; i < l; i++ { b := s.Index(i) if structPointer_IsNil(b) { return errRepeatedHasNil } o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) err := o.enc_struct(p.sprop, b) if err != nil && !state.shouldContinue(err, nil) { if err == ErrNil { return errRepeatedHasNil } return err } o.EncodeVarint(uint64((p.Tag << 3) | WireEndGroup)) } return state.err } func size_slice_struct_group(p *Properties, base structPointer) (n int) { s := structPointer_StructPointerSlice(base, p.field) l := s.Len() n += l * sizeVarint(uint64((p.Tag<<3)|WireStartGroup)) n += l * sizeVarint(uint64((p.Tag<<3)|WireEndGroup)) for i := 0; i < l; i++ { b := s.Index(i) if structPointer_IsNil(b) { return // return size up to this point } n += size_struct(p.sprop, b) } return } // Encode an extension map. func (o *Buffer) enc_map(p *Properties, base structPointer) error { exts := structPointer_ExtMap(base, p.field) if err := encodeExtensionsMap(*exts); err != nil { return err } return o.enc_map_body(*exts) } func (o *Buffer) enc_exts(p *Properties, base structPointer) error { exts := structPointer_Extensions(base, p.field) v, mu := exts.extensionsRead() if v == nil { return nil } mu.Lock() defer mu.Unlock() if err := encodeExtensionsMap(v); err != nil { return err } return o.enc_map_body(v) } func (o *Buffer) enc_map_body(v map[int32]Extension) error { // Fast-path for common cases: zero or one extensions. if len(v) <= 1 { for _, e := range v { o.buf = append(o.buf, e.enc...) } return nil } // Sort keys to provide a deterministic encoding. keys := make([]int, 0, len(v)) for k := range v { keys = append(keys, int(k)) } sort.Ints(keys) for _, k := range keys { o.buf = append(o.buf, v[int32(k)].enc...) } return nil } func size_map(p *Properties, base structPointer) int { v := structPointer_ExtMap(base, p.field) return extensionsMapSize(*v) } func size_exts(p *Properties, base structPointer) int { v := structPointer_Extensions(base, p.field) return extensionsSize(v) } // Encode a map field. func (o *Buffer) enc_new_map(p *Properties, base structPointer) error { var state errorState // XXX: or do we need to plumb this through? /* A map defined as map map_field = N; is encoded in the same way as message MapFieldEntry { key_type key = 1; value_type value = 2; } repeated MapFieldEntry map_field = N; */ v := structPointer_NewAt(base, p.field, p.mtype).Elem() // map[K]V if v.Len() == 0 { return nil } keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) enc := func() error { if err := p.mkeyprop.enc(o, p.mkeyprop, keybase); err != nil { return err } if err := p.mvalprop.enc(o, p.mvalprop, valbase); err != nil && err != ErrNil { return err } return nil } // Don't sort map keys. It is not required by the spec, and C++ doesn't do it. for _, key := range v.MapKeys() { val := v.MapIndex(key) keycopy.Set(key) valcopy.Set(val) o.buf = append(o.buf, p.tagcode...) if err := o.enc_len_thing(enc, &state); err != nil { return err } } return nil } func size_new_map(p *Properties, base structPointer) int { v := structPointer_NewAt(base, p.field, p.mtype).Elem() // map[K]V keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) n := 0 for _, key := range v.MapKeys() { val := v.MapIndex(key) keycopy.Set(key) valcopy.Set(val) // Tag codes for key and val are the responsibility of the sub-sizer. keysize := p.mkeyprop.size(p.mkeyprop, keybase) valsize := p.mvalprop.size(p.mvalprop, valbase) entry := keysize + valsize // Add on tag code and length of map entry itself. n += len(p.tagcode) + sizeVarint(uint64(entry)) + entry } return n } // mapEncodeScratch returns a new reflect.Value matching the map's value type, // and a structPointer suitable for passing to an encoder or sizer. func mapEncodeScratch(mapType reflect.Type) (keycopy, valcopy reflect.Value, keybase, valbase structPointer) { // Prepare addressable doubly-indirect placeholders for the key and value types. // This is needed because the element-type encoders expect **T, but the map iteration produces T. keycopy = reflect.New(mapType.Key()).Elem() // addressable K keyptr := reflect.New(reflect.PtrTo(keycopy.Type())).Elem() // addressable *K keyptr.Set(keycopy.Addr()) // keybase = toStructPointer(keyptr.Addr()) // **K // Value types are more varied and require special handling. switch mapType.Elem().Kind() { case reflect.Slice: // []byte var dummy []byte valcopy = reflect.ValueOf(&dummy).Elem() // addressable []byte valbase = toStructPointer(valcopy.Addr()) case reflect.Ptr: // message; the generated field type is map[K]*Msg (so V is *Msg), // so we only need one level of indirection. valcopy = reflect.New(mapType.Elem()).Elem() // addressable V valbase = toStructPointer(valcopy.Addr()) default: // everything else valcopy = reflect.New(mapType.Elem()).Elem() // addressable V valptr := reflect.New(reflect.PtrTo(valcopy.Type())).Elem() // addressable *V valptr.Set(valcopy.Addr()) // valbase = toStructPointer(valptr.Addr()) // **V } return } // Encode a struct. func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { var state errorState // Encode fields in tag order so that decoders may use optimizations // that depend on the ordering. // https://developers.google.com/protocol-buffers/docs/encoding#order for _, i := range prop.order { p := prop.Prop[i] if p.enc != nil { err := p.enc(o, p, base) if err != nil { if err == ErrNil { if p.Required && state.err == nil { state.err = &RequiredNotSetError{p.Name} } } else if err == errRepeatedHasNil { // Give more context to nil values in repeated fields. return errors.New("repeated field " + p.OrigName + " has nil element") } else if !state.shouldContinue(err, p) { return err } } if len(o.buf) > maxMarshalSize { return ErrTooLarge } } } // Do oneof fields. if prop.oneofMarshaler != nil { m := structPointer_Interface(base, prop.stype).(Message) if err := prop.oneofMarshaler(m, o); err == ErrNil { return errOneofHasNil } else if err != nil { return err } } // Add unrecognized fields at the end. if prop.unrecField.IsValid() { v := *structPointer_Bytes(base, prop.unrecField) if len(o.buf)+len(v) > maxMarshalSize { return ErrTooLarge } if len(v) > 0 { o.buf = append(o.buf, v...) } } return state.err } func size_struct(prop *StructProperties, base structPointer) (n int) { for _, i := range prop.order { p := prop.Prop[i] if p.size != nil { n += p.size(p, base) } } // Add unrecognized fields at the end. if prop.unrecField.IsValid() { v := *structPointer_Bytes(base, prop.unrecField) n += len(v) } // Factor in any oneof fields. if prop.oneofSizer != nil { m := structPointer_Interface(base, prop.stype).(Message) n += prop.oneofSizer(m) } return } var zeroes [20]byte // longer than any conceivable sizeVarint // Encode a struct, preceded by its encoded length (as a varint). func (o *Buffer) enc_len_struct(prop *StructProperties, base structPointer, state *errorState) error { return o.enc_len_thing(func() error { return o.enc_struct(prop, base) }, state) } // Encode something, preceded by its encoded length (as a varint). func (o *Buffer) enc_len_thing(enc func() error, state *errorState) error { iLen := len(o.buf) o.buf = append(o.buf, 0, 0, 0, 0) // reserve four bytes for length iMsg := len(o.buf) err := enc() if err != nil && !state.shouldContinue(err, nil) { return err } lMsg := len(o.buf) - iMsg lLen := sizeVarint(uint64(lMsg)) switch x := lLen - (iMsg - iLen); { case x > 0: // actual length is x bytes larger than the space we reserved // Move msg x bytes right. o.buf = append(o.buf, zeroes[:x]...) copy(o.buf[iMsg+x:], o.buf[iMsg:iMsg+lMsg]) case x < 0: // actual length is x bytes smaller than the space we reserved // Move msg x bytes left. copy(o.buf[iMsg+x:], o.buf[iMsg:iMsg+lMsg]) o.buf = o.buf[:len(o.buf)+x] // x is negative } // Encode the length in the reserved space. o.buf = o.buf[:iLen] o.EncodeVarint(uint64(lMsg)) o.buf = o.buf[:len(o.buf)+lMsg] return state.err } // errorState maintains the first error that occurs and updates that error // with additional context. type errorState struct { err error } // shouldContinue reports whether encoding should continue upon encountering the // given error. If the error is RequiredNotSetError, shouldContinue returns true // and, if this is the first appearance of that error, remembers it for future // reporting. // // If prop is not nil, it may update any error with additional context about the // field with the error. func (s *errorState) shouldContinue(err error, prop *Properties) bool { // Ignore unset required fields. reqNotSet, ok := err.(*RequiredNotSetError) if !ok { return false } if s.err == nil { if prop != nil { err = &RequiredNotSetError{prop.Name + "." + reqNotSet.field} } s.err = err } return true } ================================================ FILE: vendor/github.com/golang/protobuf/proto/encode_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +build go1.7 package proto_test import ( "strconv" "testing" "github.com/golang/protobuf/proto" tpb "github.com/golang/protobuf/proto/proto3_proto" "github.com/golang/protobuf/ptypes" ) var ( blackhole []byte ) // BenchmarkAny creates increasingly large arbitrary Any messages. The type is always the // same. func BenchmarkAny(b *testing.B) { data := make([]byte, 1<<20) quantum := 1 << 10 for i := uint(0); i <= 10; i++ { b.Run(strconv.Itoa(quantum<= len(o.buf) { break } } return value.Interface(), nil } // GetExtensions returns a slice of the extensions present in pb that are also listed in es. // The returned slice has the same length as es; missing extensions will appear as nil elements. func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) { epb, ok := extendable(pb) if !ok { return nil, errors.New("proto: not an extendable proto") } extensions = make([]interface{}, len(es)) for i, e := range es { extensions[i], err = GetExtension(epb, e) if err == ErrMissingExtension { err = nil } if err != nil { return } } return } // ExtensionDescs returns a new slice containing pb's extension descriptors, in undefined order. // For non-registered extensions, ExtensionDescs returns an incomplete descriptor containing // just the Field field, which defines the extension's field number. func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { epb, ok := extendable(pb) if !ok { return nil, fmt.Errorf("proto: %T is not an extendable proto.Message", pb) } registeredExtensions := RegisteredExtensions(pb) emap, mu := epb.extensionsRead() if emap == nil { return nil, nil } mu.Lock() defer mu.Unlock() extensions := make([]*ExtensionDesc, 0, len(emap)) for extid, e := range emap { desc := e.desc if desc == nil { desc = registeredExtensions[extid] if desc == nil { desc = &ExtensionDesc{Field: extid} } } extensions = append(extensions, desc) } return extensions, nil } // SetExtension sets the specified extension of pb to the specified value. func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error { epb, ok := extendable(pb) if !ok { return errors.New("proto: not an extendable proto") } if err := checkExtensionTypes(epb, extension); err != nil { return err } typ := reflect.TypeOf(extension.ExtensionType) if typ != reflect.TypeOf(value) { return errors.New("proto: bad extension value type") } // nil extension values need to be caught early, because the // encoder can't distinguish an ErrNil due to a nil extension // from an ErrNil due to a missing field. Extensions are // always optional, so the encoder would just swallow the error // and drop all the extensions from the encoded message. if reflect.ValueOf(value).IsNil() { return fmt.Errorf("proto: SetExtension called with nil value of type %T", value) } extmap := epb.extensionsWrite() extmap[extension.Field] = Extension{desc: extension, value: value} return nil } // ClearAllExtensions clears all extensions from pb. func ClearAllExtensions(pb Message) { epb, ok := extendable(pb) if !ok { return } m := epb.extensionsWrite() for k := range m { delete(m, k) } } // A global registry of extensions. // The generated code will register the generated descriptors by calling RegisterExtension. var extensionMaps = make(map[reflect.Type]map[int32]*ExtensionDesc) // RegisterExtension is called from the generated code. func RegisterExtension(desc *ExtensionDesc) { st := reflect.TypeOf(desc.ExtendedType).Elem() m := extensionMaps[st] if m == nil { m = make(map[int32]*ExtensionDesc) extensionMaps[st] = m } if _, ok := m[desc.Field]; ok { panic("proto: duplicate extension registered: " + st.String() + " " + strconv.Itoa(int(desc.Field))) } m[desc.Field] = desc } // RegisteredExtensions returns a map of the registered extensions of a // protocol buffer struct, indexed by the extension number. // The argument pb should be a nil pointer to the struct type. func RegisteredExtensions(pb Message) map[int32]*ExtensionDesc { return extensionMaps[reflect.TypeOf(pb).Elem()] } ================================================ FILE: vendor/github.com/golang/protobuf/proto/extensions_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2014 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "bytes" "fmt" "reflect" "sort" "testing" "github.com/golang/protobuf/proto" pb "github.com/golang/protobuf/proto/testdata" "golang.org/x/sync/errgroup" ) func TestGetExtensionsWithMissingExtensions(t *testing.T) { msg := &pb.MyMessage{} ext1 := &pb.Ext{} if err := proto.SetExtension(msg, pb.E_Ext_More, ext1); err != nil { t.Fatalf("Could not set ext1: %s", err) } exts, err := proto.GetExtensions(msg, []*proto.ExtensionDesc{ pb.E_Ext_More, pb.E_Ext_Text, }) if err != nil { t.Fatalf("GetExtensions() failed: %s", err) } if exts[0] != ext1 { t.Errorf("ext1 not in returned extensions: %T %v", exts[0], exts[0]) } if exts[1] != nil { t.Errorf("ext2 in returned extensions: %T %v", exts[1], exts[1]) } } func TestExtensionDescsWithMissingExtensions(t *testing.T) { msg := &pb.MyMessage{Count: proto.Int32(0)} extdesc1 := pb.E_Ext_More if descs, err := proto.ExtensionDescs(msg); len(descs) != 0 || err != nil { t.Errorf("proto.ExtensionDescs: got %d descs, error %v; want 0, nil", len(descs), err) } ext1 := &pb.Ext{} if err := proto.SetExtension(msg, extdesc1, ext1); err != nil { t.Fatalf("Could not set ext1: %s", err) } extdesc2 := &proto.ExtensionDesc{ ExtendedType: (*pb.MyMessage)(nil), ExtensionType: (*bool)(nil), Field: 123456789, Name: "a.b", Tag: "varint,123456789,opt", } ext2 := proto.Bool(false) if err := proto.SetExtension(msg, extdesc2, ext2); err != nil { t.Fatalf("Could not set ext2: %s", err) } b, err := proto.Marshal(msg) if err != nil { t.Fatalf("Could not marshal msg: %v", err) } if err := proto.Unmarshal(b, msg); err != nil { t.Fatalf("Could not unmarshal into msg: %v", err) } descs, err := proto.ExtensionDescs(msg) if err != nil { t.Fatalf("proto.ExtensionDescs: got error %v", err) } sortExtDescs(descs) wantDescs := []*proto.ExtensionDesc{extdesc1, &proto.ExtensionDesc{Field: extdesc2.Field}} if !reflect.DeepEqual(descs, wantDescs) { t.Errorf("proto.ExtensionDescs(msg) sorted extension ids: got %+v, want %+v", descs, wantDescs) } } type ExtensionDescSlice []*proto.ExtensionDesc func (s ExtensionDescSlice) Len() int { return len(s) } func (s ExtensionDescSlice) Less(i, j int) bool { return s[i].Field < s[j].Field } func (s ExtensionDescSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func sortExtDescs(s []*proto.ExtensionDesc) { sort.Sort(ExtensionDescSlice(s)) } func TestGetExtensionStability(t *testing.T) { check := func(m *pb.MyMessage) bool { ext1, err := proto.GetExtension(m, pb.E_Ext_More) if err != nil { t.Fatalf("GetExtension() failed: %s", err) } ext2, err := proto.GetExtension(m, pb.E_Ext_More) if err != nil { t.Fatalf("GetExtension() failed: %s", err) } return ext1 == ext2 } msg := &pb.MyMessage{Count: proto.Int32(4)} ext0 := &pb.Ext{} if err := proto.SetExtension(msg, pb.E_Ext_More, ext0); err != nil { t.Fatalf("Could not set ext1: %s", ext0) } if !check(msg) { t.Errorf("GetExtension() not stable before marshaling") } bb, err := proto.Marshal(msg) if err != nil { t.Fatalf("Marshal() failed: %s", err) } msg1 := &pb.MyMessage{} err = proto.Unmarshal(bb, msg1) if err != nil { t.Fatalf("Unmarshal() failed: %s", err) } if !check(msg1) { t.Errorf("GetExtension() not stable after unmarshaling") } } func TestGetExtensionDefaults(t *testing.T) { var setFloat64 float64 = 1 var setFloat32 float32 = 2 var setInt32 int32 = 3 var setInt64 int64 = 4 var setUint32 uint32 = 5 var setUint64 uint64 = 6 var setBool = true var setBool2 = false var setString = "Goodnight string" var setBytes = []byte("Goodnight bytes") var setEnum = pb.DefaultsMessage_TWO type testcase struct { ext *proto.ExtensionDesc // Extension we are testing. want interface{} // Expected value of extension, or nil (meaning that GetExtension will fail). def interface{} // Expected value of extension after ClearExtension(). } tests := []testcase{ {pb.E_NoDefaultDouble, setFloat64, nil}, {pb.E_NoDefaultFloat, setFloat32, nil}, {pb.E_NoDefaultInt32, setInt32, nil}, {pb.E_NoDefaultInt64, setInt64, nil}, {pb.E_NoDefaultUint32, setUint32, nil}, {pb.E_NoDefaultUint64, setUint64, nil}, {pb.E_NoDefaultSint32, setInt32, nil}, {pb.E_NoDefaultSint64, setInt64, nil}, {pb.E_NoDefaultFixed32, setUint32, nil}, {pb.E_NoDefaultFixed64, setUint64, nil}, {pb.E_NoDefaultSfixed32, setInt32, nil}, {pb.E_NoDefaultSfixed64, setInt64, nil}, {pb.E_NoDefaultBool, setBool, nil}, {pb.E_NoDefaultBool, setBool2, nil}, {pb.E_NoDefaultString, setString, nil}, {pb.E_NoDefaultBytes, setBytes, nil}, {pb.E_NoDefaultEnum, setEnum, nil}, {pb.E_DefaultDouble, setFloat64, float64(3.1415)}, {pb.E_DefaultFloat, setFloat32, float32(3.14)}, {pb.E_DefaultInt32, setInt32, int32(42)}, {pb.E_DefaultInt64, setInt64, int64(43)}, {pb.E_DefaultUint32, setUint32, uint32(44)}, {pb.E_DefaultUint64, setUint64, uint64(45)}, {pb.E_DefaultSint32, setInt32, int32(46)}, {pb.E_DefaultSint64, setInt64, int64(47)}, {pb.E_DefaultFixed32, setUint32, uint32(48)}, {pb.E_DefaultFixed64, setUint64, uint64(49)}, {pb.E_DefaultSfixed32, setInt32, int32(50)}, {pb.E_DefaultSfixed64, setInt64, int64(51)}, {pb.E_DefaultBool, setBool, true}, {pb.E_DefaultBool, setBool2, true}, {pb.E_DefaultString, setString, "Hello, string"}, {pb.E_DefaultBytes, setBytes, []byte("Hello, bytes")}, {pb.E_DefaultEnum, setEnum, pb.DefaultsMessage_ONE}, } checkVal := func(test testcase, msg *pb.DefaultsMessage, valWant interface{}) error { val, err := proto.GetExtension(msg, test.ext) if err != nil { if valWant != nil { return fmt.Errorf("GetExtension(): %s", err) } if want := proto.ErrMissingExtension; err != want { return fmt.Errorf("Unexpected error: got %v, want %v", err, want) } return nil } // All proto2 extension values are either a pointer to a value or a slice of values. ty := reflect.TypeOf(val) tyWant := reflect.TypeOf(test.ext.ExtensionType) if got, want := ty, tyWant; got != want { return fmt.Errorf("unexpected reflect.TypeOf(): got %v want %v", got, want) } tye := ty.Elem() tyeWant := tyWant.Elem() if got, want := tye, tyeWant; got != want { return fmt.Errorf("unexpected reflect.TypeOf().Elem(): got %v want %v", got, want) } // Check the name of the type of the value. // If it is an enum it will be type int32 with the name of the enum. if got, want := tye.Name(), tye.Name(); got != want { return fmt.Errorf("unexpected reflect.TypeOf().Elem().Name(): got %v want %v", got, want) } // Check that value is what we expect. // If we have a pointer in val, get the value it points to. valExp := val if ty.Kind() == reflect.Ptr { valExp = reflect.ValueOf(val).Elem().Interface() } if got, want := valExp, valWant; !reflect.DeepEqual(got, want) { return fmt.Errorf("unexpected reflect.DeepEqual(): got %v want %v", got, want) } return nil } setTo := func(test testcase) interface{} { setTo := reflect.ValueOf(test.want) if typ := reflect.TypeOf(test.ext.ExtensionType); typ.Kind() == reflect.Ptr { setTo = reflect.New(typ).Elem() setTo.Set(reflect.New(setTo.Type().Elem())) setTo.Elem().Set(reflect.ValueOf(test.want)) } return setTo.Interface() } for _, test := range tests { msg := &pb.DefaultsMessage{} name := test.ext.Name // Check the initial value. if err := checkVal(test, msg, test.def); err != nil { t.Errorf("%s: %v", name, err) } // Set the per-type value and check value. name = fmt.Sprintf("%s (set to %T %v)", name, test.want, test.want) if err := proto.SetExtension(msg, test.ext, setTo(test)); err != nil { t.Errorf("%s: SetExtension(): %v", name, err) continue } if err := checkVal(test, msg, test.want); err != nil { t.Errorf("%s: %v", name, err) continue } // Set and check the value. name += " (cleared)" proto.ClearExtension(msg, test.ext) if err := checkVal(test, msg, test.def); err != nil { t.Errorf("%s: %v", name, err) } } } func TestExtensionsRoundTrip(t *testing.T) { msg := &pb.MyMessage{} ext1 := &pb.Ext{ Data: proto.String("hi"), } ext2 := &pb.Ext{ Data: proto.String("there"), } exists := proto.HasExtension(msg, pb.E_Ext_More) if exists { t.Error("Extension More present unexpectedly") } if err := proto.SetExtension(msg, pb.E_Ext_More, ext1); err != nil { t.Error(err) } if err := proto.SetExtension(msg, pb.E_Ext_More, ext2); err != nil { t.Error(err) } e, err := proto.GetExtension(msg, pb.E_Ext_More) if err != nil { t.Error(err) } x, ok := e.(*pb.Ext) if !ok { t.Errorf("e has type %T, expected testdata.Ext", e) } else if *x.Data != "there" { t.Errorf("SetExtension failed to overwrite, got %+v, not 'there'", x) } proto.ClearExtension(msg, pb.E_Ext_More) if _, err = proto.GetExtension(msg, pb.E_Ext_More); err != proto.ErrMissingExtension { t.Errorf("got %v, expected ErrMissingExtension", e) } if _, err := proto.GetExtension(msg, pb.E_X215); err == nil { t.Error("expected bad extension error, got nil") } if err := proto.SetExtension(msg, pb.E_X215, 12); err == nil { t.Error("expected extension err") } if err := proto.SetExtension(msg, pb.E_Ext_More, 12); err == nil { t.Error("expected some sort of type mismatch error, got nil") } } func TestNilExtension(t *testing.T) { msg := &pb.MyMessage{ Count: proto.Int32(1), } if err := proto.SetExtension(msg, pb.E_Ext_Text, proto.String("hello")); err != nil { t.Fatal(err) } if err := proto.SetExtension(msg, pb.E_Ext_More, (*pb.Ext)(nil)); err == nil { t.Error("expected SetExtension to fail due to a nil extension") } else if want := "proto: SetExtension called with nil value of type *testdata.Ext"; err.Error() != want { t.Errorf("expected error %v, got %v", want, err) } // Note: if the behavior of Marshal is ever changed to ignore nil extensions, update // this test to verify that E_Ext_Text is properly propagated through marshal->unmarshal. } func TestMarshalUnmarshalRepeatedExtension(t *testing.T) { // Add a repeated extension to the result. tests := []struct { name string ext []*pb.ComplexExtension }{ { "two fields", []*pb.ComplexExtension{ {First: proto.Int32(7)}, {Second: proto.Int32(11)}, }, }, { "repeated field", []*pb.ComplexExtension{ {Third: []int32{1000}}, {Third: []int32{2000}}, }, }, { "two fields and repeated field", []*pb.ComplexExtension{ {Third: []int32{1000}}, {First: proto.Int32(9)}, {Second: proto.Int32(21)}, {Third: []int32{2000}}, }, }, } for _, test := range tests { // Marshal message with a repeated extension. msg1 := new(pb.OtherMessage) err := proto.SetExtension(msg1, pb.E_RComplex, test.ext) if err != nil { t.Fatalf("[%s] Error setting extension: %v", test.name, err) } b, err := proto.Marshal(msg1) if err != nil { t.Fatalf("[%s] Error marshaling message: %v", test.name, err) } // Unmarshal and read the merged proto. msg2 := new(pb.OtherMessage) err = proto.Unmarshal(b, msg2) if err != nil { t.Fatalf("[%s] Error unmarshaling message: %v", test.name, err) } e, err := proto.GetExtension(msg2, pb.E_RComplex) if err != nil { t.Fatalf("[%s] Error getting extension: %v", test.name, err) } ext := e.([]*pb.ComplexExtension) if ext == nil { t.Fatalf("[%s] Invalid extension", test.name) } if !reflect.DeepEqual(ext, test.ext) { t.Errorf("[%s] Wrong value for ComplexExtension: got: %v want: %v\n", test.name, ext, test.ext) } } } func TestUnmarshalRepeatingNonRepeatedExtension(t *testing.T) { // We may see multiple instances of the same extension in the wire // format. For example, the proto compiler may encode custom options in // this way. Here, we verify that we merge the extensions together. tests := []struct { name string ext []*pb.ComplexExtension }{ { "two fields", []*pb.ComplexExtension{ {First: proto.Int32(7)}, {Second: proto.Int32(11)}, }, }, { "repeated field", []*pb.ComplexExtension{ {Third: []int32{1000}}, {Third: []int32{2000}}, }, }, { "two fields and repeated field", []*pb.ComplexExtension{ {Third: []int32{1000}}, {First: proto.Int32(9)}, {Second: proto.Int32(21)}, {Third: []int32{2000}}, }, }, } for _, test := range tests { var buf bytes.Buffer var want pb.ComplexExtension // Generate a serialized representation of a repeated extension // by catenating bytes together. for i, e := range test.ext { // Merge to create the wanted proto. proto.Merge(&want, e) // serialize the message msg := new(pb.OtherMessage) err := proto.SetExtension(msg, pb.E_Complex, e) if err != nil { t.Fatalf("[%s] Error setting extension %d: %v", test.name, i, err) } b, err := proto.Marshal(msg) if err != nil { t.Fatalf("[%s] Error marshaling message %d: %v", test.name, i, err) } buf.Write(b) } // Unmarshal and read the merged proto. msg2 := new(pb.OtherMessage) err := proto.Unmarshal(buf.Bytes(), msg2) if err != nil { t.Fatalf("[%s] Error unmarshaling message: %v", test.name, err) } e, err := proto.GetExtension(msg2, pb.E_Complex) if err != nil { t.Fatalf("[%s] Error getting extension: %v", test.name, err) } ext := e.(*pb.ComplexExtension) if ext == nil { t.Fatalf("[%s] Invalid extension", test.name) } if !reflect.DeepEqual(*ext, want) { t.Errorf("[%s] Wrong value for ComplexExtension: got: %s want: %s\n", test.name, ext, want) } } } func TestClearAllExtensions(t *testing.T) { // unregistered extension desc := &proto.ExtensionDesc{ ExtendedType: (*pb.MyMessage)(nil), ExtensionType: (*bool)(nil), Field: 101010100, Name: "emptyextension", Tag: "varint,0,opt", } m := &pb.MyMessage{} if proto.HasExtension(m, desc) { t.Errorf("proto.HasExtension(%s): got true, want false", proto.MarshalTextString(m)) } if err := proto.SetExtension(m, desc, proto.Bool(true)); err != nil { t.Errorf("proto.SetExtension(m, desc, true): got error %q, want nil", err) } if !proto.HasExtension(m, desc) { t.Errorf("proto.HasExtension(%s): got false, want true", proto.MarshalTextString(m)) } proto.ClearAllExtensions(m) if proto.HasExtension(m, desc) { t.Errorf("proto.HasExtension(%s): got true, want false", proto.MarshalTextString(m)) } } func TestMarshalRace(t *testing.T) { // unregistered extension desc := &proto.ExtensionDesc{ ExtendedType: (*pb.MyMessage)(nil), ExtensionType: (*bool)(nil), Field: 101010100, Name: "emptyextension", Tag: "varint,0,opt", } m := &pb.MyMessage{Count: proto.Int32(4)} if err := proto.SetExtension(m, desc, proto.Bool(true)); err != nil { t.Errorf("proto.SetExtension(m, desc, true): got error %q, want nil", err) } var g errgroup.Group for n := 3; n > 0; n-- { g.Go(func() error { _, err := proto.Marshal(m) return err }) } if err := g.Wait(); err != nil { t.Fatal(err) } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/lib.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Package proto converts data structures to and from the wire format of protocol buffers. It works in concert with the Go source code generated for .proto files by the protocol compiler. A summary of the properties of the protocol buffer interface for a protocol buffer variable v: - Names are turned from camel_case to CamelCase for export. - There are no methods on v to set fields; just treat them as structure fields. - There are getters that return a field's value if set, and return the field's default value if unset. The getters work even if the receiver is a nil message. - The zero value for a struct is its correct initialization state. All desired fields must be set before marshaling. - A Reset() method will restore a protobuf struct to its zero state. - Non-repeated fields are pointers to the values; nil means unset. That is, optional or required field int32 f becomes F *int32. - Repeated fields are slices. - Helper functions are available to aid the setting of fields. msg.Foo = proto.String("hello") // set field - Constants are defined to hold the default values of all fields that have them. They have the form Default_StructName_FieldName. Because the getter methods handle defaulted values, direct use of these constants should be rare. - Enums are given type names and maps from names to values. Enum values are prefixed by the enclosing message's name, or by the enum's type name if it is a top-level enum. Enum types have a String method, and a Enum method to assist in message construction. - Nested messages, groups and enums have type names prefixed with the name of the surrounding message type. - Extensions are given descriptor names that start with E_, followed by an underscore-delimited list of the nested messages that contain it (if any) followed by the CamelCased name of the extension field itself. HasExtension, ClearExtension, GetExtension and SetExtension are functions for manipulating extensions. - Oneof field sets are given a single field in their message, with distinguished wrapper types for each possible field value. - Marshal and Unmarshal are functions to encode and decode the wire format. When the .proto file specifies `syntax="proto3"`, there are some differences: - Non-repeated fields of non-message type are values instead of pointers. - Enum types do not get an Enum method. The simplest way to describe this is to see an example. Given file test.proto, containing package example; enum FOO { X = 17; } message Test { required string label = 1; optional int32 type = 2 [default=77]; repeated int64 reps = 3; optional group OptionalGroup = 4 { required string RequiredField = 5; } oneof union { int32 number = 6; string name = 7; } } The resulting file, test.pb.go, is: package example import proto "github.com/golang/protobuf/proto" import math "math" type FOO int32 const ( FOO_X FOO = 17 ) var FOO_name = map[int32]string{ 17: "X", } var FOO_value = map[string]int32{ "X": 17, } func (x FOO) Enum() *FOO { p := new(FOO) *p = x return p } func (x FOO) String() string { return proto.EnumName(FOO_name, int32(x)) } func (x *FOO) UnmarshalJSON(data []byte) error { value, err := proto.UnmarshalJSONEnum(FOO_value, data) if err != nil { return err } *x = FOO(value) return nil } type Test struct { Label *string `protobuf:"bytes,1,req,name=label" json:"label,omitempty"` Type *int32 `protobuf:"varint,2,opt,name=type,def=77" json:"type,omitempty"` Reps []int64 `protobuf:"varint,3,rep,name=reps" json:"reps,omitempty"` Optionalgroup *Test_OptionalGroup `protobuf:"group,4,opt,name=OptionalGroup" json:"optionalgroup,omitempty"` // Types that are valid to be assigned to Union: // *Test_Number // *Test_Name Union isTest_Union `protobuf_oneof:"union"` XXX_unrecognized []byte `json:"-"` } func (m *Test) Reset() { *m = Test{} } func (m *Test) String() string { return proto.CompactTextString(m) } func (*Test) ProtoMessage() {} type isTest_Union interface { isTest_Union() } type Test_Number struct { Number int32 `protobuf:"varint,6,opt,name=number"` } type Test_Name struct { Name string `protobuf:"bytes,7,opt,name=name"` } func (*Test_Number) isTest_Union() {} func (*Test_Name) isTest_Union() {} func (m *Test) GetUnion() isTest_Union { if m != nil { return m.Union } return nil } const Default_Test_Type int32 = 77 func (m *Test) GetLabel() string { if m != nil && m.Label != nil { return *m.Label } return "" } func (m *Test) GetType() int32 { if m != nil && m.Type != nil { return *m.Type } return Default_Test_Type } func (m *Test) GetOptionalgroup() *Test_OptionalGroup { if m != nil { return m.Optionalgroup } return nil } type Test_OptionalGroup struct { RequiredField *string `protobuf:"bytes,5,req" json:"RequiredField,omitempty"` } func (m *Test_OptionalGroup) Reset() { *m = Test_OptionalGroup{} } func (m *Test_OptionalGroup) String() string { return proto.CompactTextString(m) } func (m *Test_OptionalGroup) GetRequiredField() string { if m != nil && m.RequiredField != nil { return *m.RequiredField } return "" } func (m *Test) GetNumber() int32 { if x, ok := m.GetUnion().(*Test_Number); ok { return x.Number } return 0 } func (m *Test) GetName() string { if x, ok := m.GetUnion().(*Test_Name); ok { return x.Name } return "" } func init() { proto.RegisterEnum("example.FOO", FOO_name, FOO_value) } To create and play with a Test object: package main import ( "log" "github.com/golang/protobuf/proto" pb "./example.pb" ) func main() { test := &pb.Test{ Label: proto.String("hello"), Type: proto.Int32(17), Reps: []int64{1, 2, 3}, Optionalgroup: &pb.Test_OptionalGroup{ RequiredField: proto.String("good bye"), }, Union: &pb.Test_Name{"fred"}, } data, err := proto.Marshal(test) if err != nil { log.Fatal("marshaling error: ", err) } newTest := &pb.Test{} err = proto.Unmarshal(data, newTest) if err != nil { log.Fatal("unmarshaling error: ", err) } // Now test and newTest contain the same data. if test.GetLabel() != newTest.GetLabel() { log.Fatalf("data mismatch %q != %q", test.GetLabel(), newTest.GetLabel()) } // Use a type switch to determine which oneof was set. switch u := test.Union.(type) { case *pb.Test_Number: // u.Number contains the number. case *pb.Test_Name: // u.Name contains the string. } // etc. } */ package proto import ( "encoding/json" "fmt" "log" "reflect" "sort" "strconv" "sync" ) // Message is implemented by generated protocol buffer messages. type Message interface { Reset() String() string ProtoMessage() } // Stats records allocation details about the protocol buffer encoders // and decoders. Useful for tuning the library itself. type Stats struct { Emalloc uint64 // mallocs in encode Dmalloc uint64 // mallocs in decode Encode uint64 // number of encodes Decode uint64 // number of decodes Chit uint64 // number of cache hits Cmiss uint64 // number of cache misses Size uint64 // number of sizes } // Set to true to enable stats collection. const collectStats = false var stats Stats // GetStats returns a copy of the global Stats structure. func GetStats() Stats { return stats } // A Buffer is a buffer manager for marshaling and unmarshaling // protocol buffers. It may be reused between invocations to // reduce memory usage. It is not necessary to use a Buffer; // the global functions Marshal and Unmarshal create a // temporary Buffer and are fine for most applications. type Buffer struct { buf []byte // encode/decode byte stream index int // read point // pools of basic types to amortize allocation. bools []bool uint32s []uint32 uint64s []uint64 // extra pools, only used with pointer_reflect.go int32s []int32 int64s []int64 float32s []float32 float64s []float64 } // NewBuffer allocates a new Buffer and initializes its internal data to // the contents of the argument slice. func NewBuffer(e []byte) *Buffer { return &Buffer{buf: e} } // Reset resets the Buffer, ready for marshaling a new protocol buffer. func (p *Buffer) Reset() { p.buf = p.buf[0:0] // for reading/writing p.index = 0 // for reading } // SetBuf replaces the internal buffer with the slice, // ready for unmarshaling the contents of the slice. func (p *Buffer) SetBuf(s []byte) { p.buf = s p.index = 0 } // Bytes returns the contents of the Buffer. func (p *Buffer) Bytes() []byte { return p.buf } /* * Helper routines for simplifying the creation of optional fields of basic type. */ // Bool is a helper routine that allocates a new bool value // to store v and returns a pointer to it. func Bool(v bool) *bool { return &v } // Int32 is a helper routine that allocates a new int32 value // to store v and returns a pointer to it. func Int32(v int32) *int32 { return &v } // Int is a helper routine that allocates a new int32 value // to store v and returns a pointer to it, but unlike Int32 // its argument value is an int. func Int(v int) *int32 { p := new(int32) *p = int32(v) return p } // Int64 is a helper routine that allocates a new int64 value // to store v and returns a pointer to it. func Int64(v int64) *int64 { return &v } // Float32 is a helper routine that allocates a new float32 value // to store v and returns a pointer to it. func Float32(v float32) *float32 { return &v } // Float64 is a helper routine that allocates a new float64 value // to store v and returns a pointer to it. func Float64(v float64) *float64 { return &v } // Uint32 is a helper routine that allocates a new uint32 value // to store v and returns a pointer to it. func Uint32(v uint32) *uint32 { return &v } // Uint64 is a helper routine that allocates a new uint64 value // to store v and returns a pointer to it. func Uint64(v uint64) *uint64 { return &v } // String is a helper routine that allocates a new string value // to store v and returns a pointer to it. func String(v string) *string { return &v } // EnumName is a helper function to simplify printing protocol buffer enums // by name. Given an enum map and a value, it returns a useful string. func EnumName(m map[int32]string, v int32) string { s, ok := m[v] if ok { return s } return strconv.Itoa(int(v)) } // UnmarshalJSONEnum is a helper function to simplify recovering enum int values // from their JSON-encoded representation. Given a map from the enum's symbolic // names to its int values, and a byte buffer containing the JSON-encoded // value, it returns an int32 that can be cast to the enum type by the caller. // // The function can deal with both JSON representations, numeric and symbolic. func UnmarshalJSONEnum(m map[string]int32, data []byte, enumName string) (int32, error) { if data[0] == '"' { // New style: enums are strings. var repr string if err := json.Unmarshal(data, &repr); err != nil { return -1, err } val, ok := m[repr] if !ok { return 0, fmt.Errorf("unrecognized enum %s value %q", enumName, repr) } return val, nil } // Old style: enums are ints. var val int32 if err := json.Unmarshal(data, &val); err != nil { return 0, fmt.Errorf("cannot unmarshal %#q into enum %s", data, enumName) } return val, nil } // DebugPrint dumps the encoded data in b in a debugging format with a header // including the string s. Used in testing but made available for general debugging. func (p *Buffer) DebugPrint(s string, b []byte) { var u uint64 obuf := p.buf index := p.index p.buf = b p.index = 0 depth := 0 fmt.Printf("\n--- %s ---\n", s) out: for { for i := 0; i < depth; i++ { fmt.Print(" ") } index := p.index if index == len(p.buf) { break } op, err := p.DecodeVarint() if err != nil { fmt.Printf("%3d: fetching op err %v\n", index, err) break out } tag := op >> 3 wire := op & 7 switch wire { default: fmt.Printf("%3d: t=%3d unknown wire=%d\n", index, tag, wire) break out case WireBytes: var r []byte r, err = p.DecodeRawBytes(false) if err != nil { break out } fmt.Printf("%3d: t=%3d bytes [%d]", index, tag, len(r)) if len(r) <= 6 { for i := 0; i < len(r); i++ { fmt.Printf(" %.2x", r[i]) } } else { for i := 0; i < 3; i++ { fmt.Printf(" %.2x", r[i]) } fmt.Printf(" ..") for i := len(r) - 3; i < len(r); i++ { fmt.Printf(" %.2x", r[i]) } } fmt.Printf("\n") case WireFixed32: u, err = p.DecodeFixed32() if err != nil { fmt.Printf("%3d: t=%3d fix32 err %v\n", index, tag, err) break out } fmt.Printf("%3d: t=%3d fix32 %d\n", index, tag, u) case WireFixed64: u, err = p.DecodeFixed64() if err != nil { fmt.Printf("%3d: t=%3d fix64 err %v\n", index, tag, err) break out } fmt.Printf("%3d: t=%3d fix64 %d\n", index, tag, u) case WireVarint: u, err = p.DecodeVarint() if err != nil { fmt.Printf("%3d: t=%3d varint err %v\n", index, tag, err) break out } fmt.Printf("%3d: t=%3d varint %d\n", index, tag, u) case WireStartGroup: fmt.Printf("%3d: t=%3d start\n", index, tag) depth++ case WireEndGroup: depth-- fmt.Printf("%3d: t=%3d end\n", index, tag) } } if depth != 0 { fmt.Printf("%3d: start-end not balanced %d\n", p.index, depth) } fmt.Printf("\n") p.buf = obuf p.index = index } // SetDefaults sets unset protocol buffer fields to their default values. // It only modifies fields that are both unset and have defined defaults. // It recursively sets default values in any non-nil sub-messages. func SetDefaults(pb Message) { setDefaults(reflect.ValueOf(pb), true, false) } // v is a pointer to a struct. func setDefaults(v reflect.Value, recur, zeros bool) { v = v.Elem() defaultMu.RLock() dm, ok := defaults[v.Type()] defaultMu.RUnlock() if !ok { dm = buildDefaultMessage(v.Type()) defaultMu.Lock() defaults[v.Type()] = dm defaultMu.Unlock() } for _, sf := range dm.scalars { f := v.Field(sf.index) if !f.IsNil() { // field already set continue } dv := sf.value if dv == nil && !zeros { // no explicit default, and don't want to set zeros continue } fptr := f.Addr().Interface() // **T // TODO: Consider batching the allocations we do here. switch sf.kind { case reflect.Bool: b := new(bool) if dv != nil { *b = dv.(bool) } *(fptr.(**bool)) = b case reflect.Float32: f := new(float32) if dv != nil { *f = dv.(float32) } *(fptr.(**float32)) = f case reflect.Float64: f := new(float64) if dv != nil { *f = dv.(float64) } *(fptr.(**float64)) = f case reflect.Int32: // might be an enum if ft := f.Type(); ft != int32PtrType { // enum f.Set(reflect.New(ft.Elem())) if dv != nil { f.Elem().SetInt(int64(dv.(int32))) } } else { // int32 field i := new(int32) if dv != nil { *i = dv.(int32) } *(fptr.(**int32)) = i } case reflect.Int64: i := new(int64) if dv != nil { *i = dv.(int64) } *(fptr.(**int64)) = i case reflect.String: s := new(string) if dv != nil { *s = dv.(string) } *(fptr.(**string)) = s case reflect.Uint8: // exceptional case: []byte var b []byte if dv != nil { db := dv.([]byte) b = make([]byte, len(db)) copy(b, db) } else { b = []byte{} } *(fptr.(*[]byte)) = b case reflect.Uint32: u := new(uint32) if dv != nil { *u = dv.(uint32) } *(fptr.(**uint32)) = u case reflect.Uint64: u := new(uint64) if dv != nil { *u = dv.(uint64) } *(fptr.(**uint64)) = u default: log.Printf("proto: can't set default for field %v (sf.kind=%v)", f, sf.kind) } } for _, ni := range dm.nested { f := v.Field(ni) // f is *T or []*T or map[T]*T switch f.Kind() { case reflect.Ptr: if f.IsNil() { continue } setDefaults(f, recur, zeros) case reflect.Slice: for i := 0; i < f.Len(); i++ { e := f.Index(i) if e.IsNil() { continue } setDefaults(e, recur, zeros) } case reflect.Map: for _, k := range f.MapKeys() { e := f.MapIndex(k) if e.IsNil() { continue } setDefaults(e, recur, zeros) } } } } var ( // defaults maps a protocol buffer struct type to a slice of the fields, // with its scalar fields set to their proto-declared non-zero default values. defaultMu sync.RWMutex defaults = make(map[reflect.Type]defaultMessage) int32PtrType = reflect.TypeOf((*int32)(nil)) ) // defaultMessage represents information about the default values of a message. type defaultMessage struct { scalars []scalarField nested []int // struct field index of nested messages } type scalarField struct { index int // struct field index kind reflect.Kind // element type (the T in *T or []T) value interface{} // the proto-declared default value, or nil } // t is a struct type. func buildDefaultMessage(t reflect.Type) (dm defaultMessage) { sprop := GetProperties(t) for _, prop := range sprop.Prop { fi, ok := sprop.decoderTags.get(prop.Tag) if !ok { // XXX_unrecognized continue } ft := t.Field(fi).Type sf, nested, err := fieldDefault(ft, prop) switch { case err != nil: log.Print(err) case nested: dm.nested = append(dm.nested, fi) case sf != nil: sf.index = fi dm.scalars = append(dm.scalars, *sf) } } return dm } // fieldDefault returns the scalarField for field type ft. // sf will be nil if the field can not have a default. // nestedMessage will be true if this is a nested message. // Note that sf.index is not set on return. func fieldDefault(ft reflect.Type, prop *Properties) (sf *scalarField, nestedMessage bool, err error) { var canHaveDefault bool switch ft.Kind() { case reflect.Ptr: if ft.Elem().Kind() == reflect.Struct { nestedMessage = true } else { canHaveDefault = true // proto2 scalar field } case reflect.Slice: switch ft.Elem().Kind() { case reflect.Ptr: nestedMessage = true // repeated message case reflect.Uint8: canHaveDefault = true // bytes field } case reflect.Map: if ft.Elem().Kind() == reflect.Ptr { nestedMessage = true // map with message values } } if !canHaveDefault { if nestedMessage { return nil, true, nil } return nil, false, nil } // We now know that ft is a pointer or slice. sf = &scalarField{kind: ft.Elem().Kind()} // scalar fields without defaults if !prop.HasDefault { return sf, false, nil } // a scalar field: either *T or []byte switch ft.Elem().Kind() { case reflect.Bool: x, err := strconv.ParseBool(prop.Default) if err != nil { return nil, false, fmt.Errorf("proto: bad default bool %q: %v", prop.Default, err) } sf.value = x case reflect.Float32: x, err := strconv.ParseFloat(prop.Default, 32) if err != nil { return nil, false, fmt.Errorf("proto: bad default float32 %q: %v", prop.Default, err) } sf.value = float32(x) case reflect.Float64: x, err := strconv.ParseFloat(prop.Default, 64) if err != nil { return nil, false, fmt.Errorf("proto: bad default float64 %q: %v", prop.Default, err) } sf.value = x case reflect.Int32: x, err := strconv.ParseInt(prop.Default, 10, 32) if err != nil { return nil, false, fmt.Errorf("proto: bad default int32 %q: %v", prop.Default, err) } sf.value = int32(x) case reflect.Int64: x, err := strconv.ParseInt(prop.Default, 10, 64) if err != nil { return nil, false, fmt.Errorf("proto: bad default int64 %q: %v", prop.Default, err) } sf.value = x case reflect.String: sf.value = prop.Default case reflect.Uint8: // []byte (not *uint8) sf.value = []byte(prop.Default) case reflect.Uint32: x, err := strconv.ParseUint(prop.Default, 10, 32) if err != nil { return nil, false, fmt.Errorf("proto: bad default uint32 %q: %v", prop.Default, err) } sf.value = uint32(x) case reflect.Uint64: x, err := strconv.ParseUint(prop.Default, 10, 64) if err != nil { return nil, false, fmt.Errorf("proto: bad default uint64 %q: %v", prop.Default, err) } sf.value = x default: return nil, false, fmt.Errorf("proto: unhandled def kind %v", ft.Elem().Kind()) } return sf, false, nil } // Map fields may have key types of non-float scalars, strings and enums. // The easiest way to sort them in some deterministic order is to use fmt. // If this turns out to be inefficient we can always consider other options, // such as doing a Schwartzian transform. func mapKeys(vs []reflect.Value) sort.Interface { s := mapKeySorter{ vs: vs, // default Less function: textual comparison less: func(a, b reflect.Value) bool { return fmt.Sprint(a.Interface()) < fmt.Sprint(b.Interface()) }, } // Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps; // numeric keys are sorted numerically. if len(vs) == 0 { return s } switch vs[0].Kind() { case reflect.Int32, reflect.Int64: s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() } case reflect.Uint32, reflect.Uint64: s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() } } return s } type mapKeySorter struct { vs []reflect.Value less func(a, b reflect.Value) bool } func (s mapKeySorter) Len() int { return len(s.vs) } func (s mapKeySorter) Swap(i, j int) { s.vs[i], s.vs[j] = s.vs[j], s.vs[i] } func (s mapKeySorter) Less(i, j int) bool { return s.less(s.vs[i], s.vs[j]) } // isProto3Zero reports whether v is a zero proto3 value. func isProto3Zero(v reflect.Value) bool { switch v.Kind() { case reflect.Bool: return !v.Bool() case reflect.Int32, reflect.Int64: return v.Int() == 0 case reflect.Uint32, reflect.Uint64: return v.Uint() == 0 case reflect.Float32, reflect.Float64: return v.Float() == 0 case reflect.String: return v.String() == "" } return false } // ProtoPackageIsVersion2 is referenced from generated protocol buffer files // to assert that that code is compatible with this version of the proto package. const ProtoPackageIsVersion2 = true // ProtoPackageIsVersion1 is referenced from generated protocol buffer files // to assert that that code is compatible with this version of the proto package. const ProtoPackageIsVersion1 = true ================================================ FILE: vendor/github.com/golang/protobuf/proto/map_test.go ================================================ package proto_test import ( "fmt" "testing" "github.com/golang/protobuf/proto" ppb "github.com/golang/protobuf/proto/proto3_proto" ) func marshalled() []byte { m := &ppb.IntMaps{} for i := 0; i < 1000; i++ { m.Maps = append(m.Maps, &ppb.IntMap{ Rtt: map[int32]int32{1: 2}, }) } b, err := proto.Marshal(m) if err != nil { panic(fmt.Sprintf("Can't marshal %+v: %v", m, err)) } return b } func BenchmarkConcurrentMapUnmarshal(b *testing.B) { in := marshalled() b.RunParallel(func(pb *testing.PB) { for pb.Next() { var out ppb.IntMaps if err := proto.Unmarshal(in, &out); err != nil { b.Errorf("Can't unmarshal ppb.IntMaps: %v", err) } } }) } func BenchmarkSequentialMapUnmarshal(b *testing.B) { in := marshalled() b.ResetTimer() for i := 0; i < b.N; i++ { var out ppb.IntMaps if err := proto.Unmarshal(in, &out); err != nil { b.Errorf("Can't unmarshal ppb.IntMaps: %v", err) } } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/message_set.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto /* * Support for message sets. */ import ( "bytes" "encoding/json" "errors" "fmt" "reflect" "sort" ) // errNoMessageTypeID occurs when a protocol buffer does not have a message type ID. // A message type ID is required for storing a protocol buffer in a message set. var errNoMessageTypeID = errors.New("proto does not have a message type ID") // The first two types (_MessageSet_Item and messageSet) // model what the protocol compiler produces for the following protocol message: // message MessageSet { // repeated group Item = 1 { // required int32 type_id = 2; // required string message = 3; // }; // } // That is the MessageSet wire format. We can't use a proto to generate these // because that would introduce a circular dependency between it and this package. type _MessageSet_Item struct { TypeId *int32 `protobuf:"varint,2,req,name=type_id"` Message []byte `protobuf:"bytes,3,req,name=message"` } type messageSet struct { Item []*_MessageSet_Item `protobuf:"group,1,rep"` XXX_unrecognized []byte // TODO: caching? } // Make sure messageSet is a Message. var _ Message = (*messageSet)(nil) // messageTypeIder is an interface satisfied by a protocol buffer type // that may be stored in a MessageSet. type messageTypeIder interface { MessageTypeId() int32 } func (ms *messageSet) find(pb Message) *_MessageSet_Item { mti, ok := pb.(messageTypeIder) if !ok { return nil } id := mti.MessageTypeId() for _, item := range ms.Item { if *item.TypeId == id { return item } } return nil } func (ms *messageSet) Has(pb Message) bool { if ms.find(pb) != nil { return true } return false } func (ms *messageSet) Unmarshal(pb Message) error { if item := ms.find(pb); item != nil { return Unmarshal(item.Message, pb) } if _, ok := pb.(messageTypeIder); !ok { return errNoMessageTypeID } return nil // TODO: return error instead? } func (ms *messageSet) Marshal(pb Message) error { msg, err := Marshal(pb) if err != nil { return err } if item := ms.find(pb); item != nil { // reuse existing item item.Message = msg return nil } mti, ok := pb.(messageTypeIder) if !ok { return errNoMessageTypeID } mtid := mti.MessageTypeId() ms.Item = append(ms.Item, &_MessageSet_Item{ TypeId: &mtid, Message: msg, }) return nil } func (ms *messageSet) Reset() { *ms = messageSet{} } func (ms *messageSet) String() string { return CompactTextString(ms) } func (*messageSet) ProtoMessage() {} // Support for the message_set_wire_format message option. func skipVarint(buf []byte) []byte { i := 0 for ; buf[i]&0x80 != 0; i++ { } return buf[i+1:] } // MarshalMessageSet encodes the extension map represented by m in the message set wire format. // It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option. func MarshalMessageSet(exts interface{}) ([]byte, error) { var m map[int32]Extension switch exts := exts.(type) { case *XXX_InternalExtensions: if err := encodeExtensions(exts); err != nil { return nil, err } m, _ = exts.extensionsRead() case map[int32]Extension: if err := encodeExtensionsMap(exts); err != nil { return nil, err } m = exts default: return nil, errors.New("proto: not an extension map") } // Sort extension IDs to provide a deterministic encoding. // See also enc_map in encode.go. ids := make([]int, 0, len(m)) for id := range m { ids = append(ids, int(id)) } sort.Ints(ids) ms := &messageSet{Item: make([]*_MessageSet_Item, 0, len(m))} for _, id := range ids { e := m[int32(id)] // Remove the wire type and field number varint, as well as the length varint. msg := skipVarint(skipVarint(e.enc)) ms.Item = append(ms.Item, &_MessageSet_Item{ TypeId: Int32(int32(id)), Message: msg, }) } return Marshal(ms) } // UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format. // It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option. func UnmarshalMessageSet(buf []byte, exts interface{}) error { var m map[int32]Extension switch exts := exts.(type) { case *XXX_InternalExtensions: m = exts.extensionsWrite() case map[int32]Extension: m = exts default: return errors.New("proto: not an extension map") } ms := new(messageSet) if err := Unmarshal(buf, ms); err != nil { return err } for _, item := range ms.Item { id := *item.TypeId msg := item.Message // Restore wire type and field number varint, plus length varint. // Be careful to preserve duplicate items. b := EncodeVarint(uint64(id)<<3 | WireBytes) if ext, ok := m[id]; ok { // Existing data; rip off the tag and length varint // so we join the new data correctly. // We can assume that ext.enc is set because we are unmarshaling. o := ext.enc[len(b):] // skip wire type and field number _, n := DecodeVarint(o) // calculate length of length varint o = o[n:] // skip length varint msg = append(o, msg...) // join old data and new data } b = append(b, EncodeVarint(uint64(len(msg)))...) b = append(b, msg...) m[id] = Extension{enc: b} } return nil } // MarshalMessageSetJSON encodes the extension map represented by m in JSON format. // It is called by generated MarshalJSON methods on protocol buffer messages with the message_set_wire_format option. func MarshalMessageSetJSON(exts interface{}) ([]byte, error) { var m map[int32]Extension switch exts := exts.(type) { case *XXX_InternalExtensions: m, _ = exts.extensionsRead() case map[int32]Extension: m = exts default: return nil, errors.New("proto: not an extension map") } var b bytes.Buffer b.WriteByte('{') // Process the map in key order for deterministic output. ids := make([]int32, 0, len(m)) for id := range m { ids = append(ids, id) } sort.Sort(int32Slice(ids)) // int32Slice defined in text.go for i, id := range ids { ext := m[id] if i > 0 { b.WriteByte(',') } msd, ok := messageSetMap[id] if !ok { // Unknown type; we can't render it, so skip it. continue } fmt.Fprintf(&b, `"[%s]":`, msd.name) x := ext.value if x == nil { x = reflect.New(msd.t.Elem()).Interface() if err := Unmarshal(ext.enc, x.(Message)); err != nil { return nil, err } } d, err := json.Marshal(x) if err != nil { return nil, err } b.Write(d) } b.WriteByte('}') return b.Bytes(), nil } // UnmarshalMessageSetJSON decodes the extension map encoded in buf in JSON format. // It is called by generated UnmarshalJSON methods on protocol buffer messages with the message_set_wire_format option. func UnmarshalMessageSetJSON(buf []byte, exts interface{}) error { // Common-case fast path. if len(buf) == 0 || bytes.Equal(buf, []byte("{}")) { return nil } // This is fairly tricky, and it's not clear that it is needed. return errors.New("TODO: UnmarshalMessageSetJSON not yet implemented") } // A global registry of types that can be used in a MessageSet. var messageSetMap = make(map[int32]messageSetDesc) type messageSetDesc struct { t reflect.Type // pointer to struct name string } // RegisterMessageSetType is called from the generated code. func RegisterMessageSetType(m Message, fieldNum int32, name string) { messageSetMap[fieldNum] = messageSetDesc{ t: reflect.TypeOf(m), name: name, } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/message_set_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2014 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto import ( "bytes" "testing" ) func TestUnmarshalMessageSetWithDuplicate(t *testing.T) { // Check that a repeated message set entry will be concatenated. in := &messageSet{ Item: []*_MessageSet_Item{ {TypeId: Int32(12345), Message: []byte("hoo")}, {TypeId: Int32(12345), Message: []byte("hah")}, }, } b, err := Marshal(in) if err != nil { t.Fatalf("Marshal: %v", err) } t.Logf("Marshaled bytes: %q", b) var extensions XXX_InternalExtensions if err := UnmarshalMessageSet(b, &extensions); err != nil { t.Fatalf("UnmarshalMessageSet: %v", err) } ext, ok := extensions.p.extensionMap[12345] if !ok { t.Fatalf("Didn't retrieve extension 12345; map is %v", extensions.p.extensionMap) } // Skip wire type/field number and length varints. got := skipVarint(skipVarint(ext.enc)) if want := []byte("hoohah"); !bytes.Equal(got, want) { t.Errorf("Combined extension is %q, want %q", got, want) } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/pointer_reflect.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2012 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +build appengine js // This file contains an implementation of proto field accesses using package reflect. // It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can // be used on App Engine. package proto import ( "math" "reflect" ) // A structPointer is a pointer to a struct. type structPointer struct { v reflect.Value } // toStructPointer returns a structPointer equivalent to the given reflect value. // The reflect value must itself be a pointer to a struct. func toStructPointer(v reflect.Value) structPointer { return structPointer{v} } // IsNil reports whether p is nil. func structPointer_IsNil(p structPointer) bool { return p.v.IsNil() } // Interface returns the struct pointer as an interface value. func structPointer_Interface(p structPointer, _ reflect.Type) interface{} { return p.v.Interface() } // A field identifies a field in a struct, accessible from a structPointer. // In this implementation, a field is identified by the sequence of field indices // passed to reflect's FieldByIndex. type field []int // toField returns a field equivalent to the given reflect field. func toField(f *reflect.StructField) field { return f.Index } // invalidField is an invalid field identifier. var invalidField = field(nil) // IsValid reports whether the field identifier is valid. func (f field) IsValid() bool { return f != nil } // field returns the given field in the struct as a reflect value. func structPointer_field(p structPointer, f field) reflect.Value { // Special case: an extension map entry with a value of type T // passes a *T to the struct-handling code with a zero field, // expecting that it will be treated as equivalent to *struct{ X T }, // which has the same memory layout. We have to handle that case // specially, because reflect will panic if we call FieldByIndex on a // non-struct. if f == nil { return p.v.Elem() } return p.v.Elem().FieldByIndex(f) } // ifield returns the given field in the struct as an interface value. func structPointer_ifield(p structPointer, f field) interface{} { return structPointer_field(p, f).Addr().Interface() } // Bytes returns the address of a []byte field in the struct. func structPointer_Bytes(p structPointer, f field) *[]byte { return structPointer_ifield(p, f).(*[]byte) } // BytesSlice returns the address of a [][]byte field in the struct. func structPointer_BytesSlice(p structPointer, f field) *[][]byte { return structPointer_ifield(p, f).(*[][]byte) } // Bool returns the address of a *bool field in the struct. func structPointer_Bool(p structPointer, f field) **bool { return structPointer_ifield(p, f).(**bool) } // BoolVal returns the address of a bool field in the struct. func structPointer_BoolVal(p structPointer, f field) *bool { return structPointer_ifield(p, f).(*bool) } // BoolSlice returns the address of a []bool field in the struct. func structPointer_BoolSlice(p structPointer, f field) *[]bool { return structPointer_ifield(p, f).(*[]bool) } // String returns the address of a *string field in the struct. func structPointer_String(p structPointer, f field) **string { return structPointer_ifield(p, f).(**string) } // StringVal returns the address of a string field in the struct. func structPointer_StringVal(p structPointer, f field) *string { return structPointer_ifield(p, f).(*string) } // StringSlice returns the address of a []string field in the struct. func structPointer_StringSlice(p structPointer, f field) *[]string { return structPointer_ifield(p, f).(*[]string) } // Extensions returns the address of an extension map field in the struct. func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { return structPointer_ifield(p, f).(*XXX_InternalExtensions) } // ExtMap returns the address of an extension map field in the struct. func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { return structPointer_ifield(p, f).(*map[int32]Extension) } // NewAt returns the reflect.Value for a pointer to a field in the struct. func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value { return structPointer_field(p, f).Addr() } // SetStructPointer writes a *struct field in the struct. func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { structPointer_field(p, f).Set(q.v) } // GetStructPointer reads a *struct field in the struct. func structPointer_GetStructPointer(p structPointer, f field) structPointer { return structPointer{structPointer_field(p, f)} } // StructPointerSlice the address of a []*struct field in the struct. func structPointer_StructPointerSlice(p structPointer, f field) structPointerSlice { return structPointerSlice{structPointer_field(p, f)} } // A structPointerSlice represents the address of a slice of pointers to structs // (themselves messages or groups). That is, v.Type() is *[]*struct{...}. type structPointerSlice struct { v reflect.Value } func (p structPointerSlice) Len() int { return p.v.Len() } func (p structPointerSlice) Index(i int) structPointer { return structPointer{p.v.Index(i)} } func (p structPointerSlice) Append(q structPointer) { p.v.Set(reflect.Append(p.v, q.v)) } var ( int32Type = reflect.TypeOf(int32(0)) uint32Type = reflect.TypeOf(uint32(0)) float32Type = reflect.TypeOf(float32(0)) int64Type = reflect.TypeOf(int64(0)) uint64Type = reflect.TypeOf(uint64(0)) float64Type = reflect.TypeOf(float64(0)) ) // A word32 represents a field of type *int32, *uint32, *float32, or *enum. // That is, v.Type() is *int32, *uint32, *float32, or *enum and v is assignable. type word32 struct { v reflect.Value } // IsNil reports whether p is nil. func word32_IsNil(p word32) bool { return p.v.IsNil() } // Set sets p to point at a newly allocated word with bits set to x. func word32_Set(p word32, o *Buffer, x uint32) { t := p.v.Type().Elem() switch t { case int32Type: if len(o.int32s) == 0 { o.int32s = make([]int32, uint32PoolSize) } o.int32s[0] = int32(x) p.v.Set(reflect.ValueOf(&o.int32s[0])) o.int32s = o.int32s[1:] return case uint32Type: if len(o.uint32s) == 0 { o.uint32s = make([]uint32, uint32PoolSize) } o.uint32s[0] = x p.v.Set(reflect.ValueOf(&o.uint32s[0])) o.uint32s = o.uint32s[1:] return case float32Type: if len(o.float32s) == 0 { o.float32s = make([]float32, uint32PoolSize) } o.float32s[0] = math.Float32frombits(x) p.v.Set(reflect.ValueOf(&o.float32s[0])) o.float32s = o.float32s[1:] return } // must be enum p.v.Set(reflect.New(t)) p.v.Elem().SetInt(int64(int32(x))) } // Get gets the bits pointed at by p, as a uint32. func word32_Get(p word32) uint32 { elem := p.v.Elem() switch elem.Kind() { case reflect.Int32: return uint32(elem.Int()) case reflect.Uint32: return uint32(elem.Uint()) case reflect.Float32: return math.Float32bits(float32(elem.Float())) } panic("unreachable") } // Word32 returns a reference to a *int32, *uint32, *float32, or *enum field in the struct. func structPointer_Word32(p structPointer, f field) word32 { return word32{structPointer_field(p, f)} } // A word32Val represents a field of type int32, uint32, float32, or enum. // That is, v.Type() is int32, uint32, float32, or enum and v is assignable. type word32Val struct { v reflect.Value } // Set sets *p to x. func word32Val_Set(p word32Val, x uint32) { switch p.v.Type() { case int32Type: p.v.SetInt(int64(x)) return case uint32Type: p.v.SetUint(uint64(x)) return case float32Type: p.v.SetFloat(float64(math.Float32frombits(x))) return } // must be enum p.v.SetInt(int64(int32(x))) } // Get gets the bits pointed at by p, as a uint32. func word32Val_Get(p word32Val) uint32 { elem := p.v switch elem.Kind() { case reflect.Int32: return uint32(elem.Int()) case reflect.Uint32: return uint32(elem.Uint()) case reflect.Float32: return math.Float32bits(float32(elem.Float())) } panic("unreachable") } // Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct. func structPointer_Word32Val(p structPointer, f field) word32Val { return word32Val{structPointer_field(p, f)} } // A word32Slice is a slice of 32-bit values. // That is, v.Type() is []int32, []uint32, []float32, or []enum. type word32Slice struct { v reflect.Value } func (p word32Slice) Append(x uint32) { n, m := p.v.Len(), p.v.Cap() if n < m { p.v.SetLen(n + 1) } else { t := p.v.Type().Elem() p.v.Set(reflect.Append(p.v, reflect.Zero(t))) } elem := p.v.Index(n) switch elem.Kind() { case reflect.Int32: elem.SetInt(int64(int32(x))) case reflect.Uint32: elem.SetUint(uint64(x)) case reflect.Float32: elem.SetFloat(float64(math.Float32frombits(x))) } } func (p word32Slice) Len() int { return p.v.Len() } func (p word32Slice) Index(i int) uint32 { elem := p.v.Index(i) switch elem.Kind() { case reflect.Int32: return uint32(elem.Int()) case reflect.Uint32: return uint32(elem.Uint()) case reflect.Float32: return math.Float32bits(float32(elem.Float())) } panic("unreachable") } // Word32Slice returns a reference to a []int32, []uint32, []float32, or []enum field in the struct. func structPointer_Word32Slice(p structPointer, f field) word32Slice { return word32Slice{structPointer_field(p, f)} } // word64 is like word32 but for 64-bit values. type word64 struct { v reflect.Value } func word64_Set(p word64, o *Buffer, x uint64) { t := p.v.Type().Elem() switch t { case int64Type: if len(o.int64s) == 0 { o.int64s = make([]int64, uint64PoolSize) } o.int64s[0] = int64(x) p.v.Set(reflect.ValueOf(&o.int64s[0])) o.int64s = o.int64s[1:] return case uint64Type: if len(o.uint64s) == 0 { o.uint64s = make([]uint64, uint64PoolSize) } o.uint64s[0] = x p.v.Set(reflect.ValueOf(&o.uint64s[0])) o.uint64s = o.uint64s[1:] return case float64Type: if len(o.float64s) == 0 { o.float64s = make([]float64, uint64PoolSize) } o.float64s[0] = math.Float64frombits(x) p.v.Set(reflect.ValueOf(&o.float64s[0])) o.float64s = o.float64s[1:] return } panic("unreachable") } func word64_IsNil(p word64) bool { return p.v.IsNil() } func word64_Get(p word64) uint64 { elem := p.v.Elem() switch elem.Kind() { case reflect.Int64: return uint64(elem.Int()) case reflect.Uint64: return elem.Uint() case reflect.Float64: return math.Float64bits(elem.Float()) } panic("unreachable") } func structPointer_Word64(p structPointer, f field) word64 { return word64{structPointer_field(p, f)} } // word64Val is like word32Val but for 64-bit values. type word64Val struct { v reflect.Value } func word64Val_Set(p word64Val, o *Buffer, x uint64) { switch p.v.Type() { case int64Type: p.v.SetInt(int64(x)) return case uint64Type: p.v.SetUint(x) return case float64Type: p.v.SetFloat(math.Float64frombits(x)) return } panic("unreachable") } func word64Val_Get(p word64Val) uint64 { elem := p.v switch elem.Kind() { case reflect.Int64: return uint64(elem.Int()) case reflect.Uint64: return elem.Uint() case reflect.Float64: return math.Float64bits(elem.Float()) } panic("unreachable") } func structPointer_Word64Val(p structPointer, f field) word64Val { return word64Val{structPointer_field(p, f)} } type word64Slice struct { v reflect.Value } func (p word64Slice) Append(x uint64) { n, m := p.v.Len(), p.v.Cap() if n < m { p.v.SetLen(n + 1) } else { t := p.v.Type().Elem() p.v.Set(reflect.Append(p.v, reflect.Zero(t))) } elem := p.v.Index(n) switch elem.Kind() { case reflect.Int64: elem.SetInt(int64(int64(x))) case reflect.Uint64: elem.SetUint(uint64(x)) case reflect.Float64: elem.SetFloat(float64(math.Float64frombits(x))) } } func (p word64Slice) Len() int { return p.v.Len() } func (p word64Slice) Index(i int) uint64 { elem := p.v.Index(i) switch elem.Kind() { case reflect.Int64: return uint64(elem.Int()) case reflect.Uint64: return uint64(elem.Uint()) case reflect.Float64: return math.Float64bits(float64(elem.Float())) } panic("unreachable") } func structPointer_Word64Slice(p structPointer, f field) word64Slice { return word64Slice{structPointer_field(p, f)} } ================================================ FILE: vendor/github.com/golang/protobuf/proto/pointer_unsafe.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2012 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +build !appengine,!js // This file contains the implementation of the proto field accesses using package unsafe. package proto import ( "reflect" "unsafe" ) // NOTE: These type_Foo functions would more idiomatically be methods, // but Go does not allow methods on pointer types, and we must preserve // some pointer type for the garbage collector. We use these // funcs with clunky names as our poor approximation to methods. // // An alternative would be // type structPointer struct { p unsafe.Pointer } // but that does not registerize as well. // A structPointer is a pointer to a struct. type structPointer unsafe.Pointer // toStructPointer returns a structPointer equivalent to the given reflect value. func toStructPointer(v reflect.Value) structPointer { return structPointer(unsafe.Pointer(v.Pointer())) } // IsNil reports whether p is nil. func structPointer_IsNil(p structPointer) bool { return p == nil } // Interface returns the struct pointer, assumed to have element type t, // as an interface value. func structPointer_Interface(p structPointer, t reflect.Type) interface{} { return reflect.NewAt(t, unsafe.Pointer(p)).Interface() } // A field identifies a field in a struct, accessible from a structPointer. // In this implementation, a field is identified by its byte offset from the start of the struct. type field uintptr // toField returns a field equivalent to the given reflect field. func toField(f *reflect.StructField) field { return field(f.Offset) } // invalidField is an invalid field identifier. const invalidField = ^field(0) // IsValid reports whether the field identifier is valid. func (f field) IsValid() bool { return f != ^field(0) } // Bytes returns the address of a []byte field in the struct. func structPointer_Bytes(p structPointer, f field) *[]byte { return (*[]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // BytesSlice returns the address of a [][]byte field in the struct. func structPointer_BytesSlice(p structPointer, f field) *[][]byte { return (*[][]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // Bool returns the address of a *bool field in the struct. func structPointer_Bool(p structPointer, f field) **bool { return (**bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // BoolVal returns the address of a bool field in the struct. func structPointer_BoolVal(p structPointer, f field) *bool { return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // BoolSlice returns the address of a []bool field in the struct. func structPointer_BoolSlice(p structPointer, f field) *[]bool { return (*[]bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // String returns the address of a *string field in the struct. func structPointer_String(p structPointer, f field) **string { return (**string)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // StringVal returns the address of a string field in the struct. func structPointer_StringVal(p structPointer, f field) *string { return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // StringSlice returns the address of a []string field in the struct. func structPointer_StringSlice(p structPointer, f field) *[]string { return (*[]string)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // ExtMap returns the address of an extension map field in the struct. func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { return (*XXX_InternalExtensions)(unsafe.Pointer(uintptr(p) + uintptr(f))) } func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { return (*map[int32]Extension)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // NewAt returns the reflect.Value for a pointer to a field in the struct. func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value { return reflect.NewAt(typ, unsafe.Pointer(uintptr(p)+uintptr(f))) } // SetStructPointer writes a *struct field in the struct. func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) = q } // GetStructPointer reads a *struct field in the struct. func structPointer_GetStructPointer(p structPointer, f field) structPointer { return *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // StructPointerSlice the address of a []*struct field in the struct. func structPointer_StructPointerSlice(p structPointer, f field) *structPointerSlice { return (*structPointerSlice)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // A structPointerSlice represents a slice of pointers to structs (themselves submessages or groups). type structPointerSlice []structPointer func (v *structPointerSlice) Len() int { return len(*v) } func (v *structPointerSlice) Index(i int) structPointer { return (*v)[i] } func (v *structPointerSlice) Append(p structPointer) { *v = append(*v, p) } // A word32 is the address of a "pointer to 32-bit value" field. type word32 **uint32 // IsNil reports whether *v is nil. func word32_IsNil(p word32) bool { return *p == nil } // Set sets *v to point at a newly allocated word set to x. func word32_Set(p word32, o *Buffer, x uint32) { if len(o.uint32s) == 0 { o.uint32s = make([]uint32, uint32PoolSize) } o.uint32s[0] = x *p = &o.uint32s[0] o.uint32s = o.uint32s[1:] } // Get gets the value pointed at by *v. func word32_Get(p word32) uint32 { return **p } // Word32 returns the address of a *int32, *uint32, *float32, or *enum field in the struct. func structPointer_Word32(p structPointer, f field) word32 { return word32((**uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } // A word32Val is the address of a 32-bit value field. type word32Val *uint32 // Set sets *p to x. func word32Val_Set(p word32Val, x uint32) { *p = x } // Get gets the value pointed at by p. func word32Val_Get(p word32Val) uint32 { return *p } // Word32Val returns the address of a *int32, *uint32, *float32, or *enum field in the struct. func structPointer_Word32Val(p structPointer, f field) word32Val { return word32Val((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } // A word32Slice is a slice of 32-bit values. type word32Slice []uint32 func (v *word32Slice) Append(x uint32) { *v = append(*v, x) } func (v *word32Slice) Len() int { return len(*v) } func (v *word32Slice) Index(i int) uint32 { return (*v)[i] } // Word32Slice returns the address of a []int32, []uint32, []float32, or []enum field in the struct. func structPointer_Word32Slice(p structPointer, f field) *word32Slice { return (*word32Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) } // word64 is like word32 but for 64-bit values. type word64 **uint64 func word64_Set(p word64, o *Buffer, x uint64) { if len(o.uint64s) == 0 { o.uint64s = make([]uint64, uint64PoolSize) } o.uint64s[0] = x *p = &o.uint64s[0] o.uint64s = o.uint64s[1:] } func word64_IsNil(p word64) bool { return *p == nil } func word64_Get(p word64) uint64 { return **p } func structPointer_Word64(p structPointer, f field) word64 { return word64((**uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } // word64Val is like word32Val but for 64-bit values. type word64Val *uint64 func word64Val_Set(p word64Val, o *Buffer, x uint64) { *p = x } func word64Val_Get(p word64Val) uint64 { return *p } func structPointer_Word64Val(p structPointer, f field) word64Val { return word64Val((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } // word64Slice is like word32Slice but for 64-bit values. type word64Slice []uint64 func (v *word64Slice) Append(x uint64) { *v = append(*v, x) } func (v *word64Slice) Len() int { return len(*v) } func (v *word64Slice) Index(i int) uint64 { return (*v)[i] } func structPointer_Word64Slice(p structPointer, f field) *word64Slice { return (*word64Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) } ================================================ FILE: vendor/github.com/golang/protobuf/proto/properties.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto /* * Routines for encoding data into the wire format for protocol buffers. */ import ( "fmt" "log" "os" "reflect" "sort" "strconv" "strings" "sync" ) const debug bool = false // Constants that identify the encoding of a value on the wire. const ( WireVarint = 0 WireFixed64 = 1 WireBytes = 2 WireStartGroup = 3 WireEndGroup = 4 WireFixed32 = 5 ) const startSize = 10 // initial slice/string sizes // Encoders are defined in encode.go // An encoder outputs the full representation of a field, including its // tag and encoder type. type encoder func(p *Buffer, prop *Properties, base structPointer) error // A valueEncoder encodes a single integer in a particular encoding. type valueEncoder func(o *Buffer, x uint64) error // Sizers are defined in encode.go // A sizer returns the encoded size of a field, including its tag and encoder // type. type sizer func(prop *Properties, base structPointer) int // A valueSizer returns the encoded size of a single integer in a particular // encoding. type valueSizer func(x uint64) int // Decoders are defined in decode.go // A decoder creates a value from its wire representation. // Unrecognized subelements are saved in unrec. type decoder func(p *Buffer, prop *Properties, base structPointer) error // A valueDecoder decodes a single integer in a particular encoding. type valueDecoder func(o *Buffer) (x uint64, err error) // A oneofMarshaler does the marshaling for all oneof fields in a message. type oneofMarshaler func(Message, *Buffer) error // A oneofUnmarshaler does the unmarshaling for a oneof field in a message. type oneofUnmarshaler func(Message, int, int, *Buffer) (bool, error) // A oneofSizer does the sizing for all oneof fields in a message. type oneofSizer func(Message) int // tagMap is an optimization over map[int]int for typical protocol buffer // use-cases. Encoded protocol buffers are often in tag order with small tag // numbers. type tagMap struct { fastTags []int slowTags map[int]int } // tagMapFastLimit is the upper bound on the tag number that will be stored in // the tagMap slice rather than its map. const tagMapFastLimit = 1024 func (p *tagMap) get(t int) (int, bool) { if t > 0 && t < tagMapFastLimit { if t >= len(p.fastTags) { return 0, false } fi := p.fastTags[t] return fi, fi >= 0 } fi, ok := p.slowTags[t] return fi, ok } func (p *tagMap) put(t int, fi int) { if t > 0 && t < tagMapFastLimit { for len(p.fastTags) < t+1 { p.fastTags = append(p.fastTags, -1) } p.fastTags[t] = fi return } if p.slowTags == nil { p.slowTags = make(map[int]int) } p.slowTags[t] = fi } // StructProperties represents properties for all the fields of a struct. // decoderTags and decoderOrigNames should only be used by the decoder. type StructProperties struct { Prop []*Properties // properties for each field reqCount int // required count decoderTags tagMap // map from proto tag to struct field number decoderOrigNames map[string]int // map from original name to struct field number order []int // list of struct field numbers in tag order unrecField field // field id of the XXX_unrecognized []byte field extendable bool // is this an extendable proto oneofMarshaler oneofMarshaler oneofUnmarshaler oneofUnmarshaler oneofSizer oneofSizer stype reflect.Type // OneofTypes contains information about the oneof fields in this message. // It is keyed by the original name of a field. OneofTypes map[string]*OneofProperties } // OneofProperties represents information about a specific field in a oneof. type OneofProperties struct { Type reflect.Type // pointer to generated struct type for this oneof field Field int // struct field number of the containing oneof in the message Prop *Properties } // Implement the sorting interface so we can sort the fields in tag order, as recommended by the spec. // See encode.go, (*Buffer).enc_struct. func (sp *StructProperties) Len() int { return len(sp.order) } func (sp *StructProperties) Less(i, j int) bool { return sp.Prop[sp.order[i]].Tag < sp.Prop[sp.order[j]].Tag } func (sp *StructProperties) Swap(i, j int) { sp.order[i], sp.order[j] = sp.order[j], sp.order[i] } // Properties represents the protocol-specific behavior of a single struct field. type Properties struct { Name string // name of the field, for error messages OrigName string // original name before protocol compiler (always set) JSONName string // name to use for JSON; determined by protoc Wire string WireType int Tag int Required bool Optional bool Repeated bool Packed bool // relevant for repeated primitives only Enum string // set for enum types only proto3 bool // whether this is known to be a proto3 field; set for []byte only oneof bool // whether this is a oneof field Default string // default value HasDefault bool // whether an explicit default was provided def_uint64 uint64 enc encoder valEnc valueEncoder // set for bool and numeric types only field field tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType) tagbuf [8]byte stype reflect.Type // set for struct types only sprop *StructProperties // set for struct types only isMarshaler bool isUnmarshaler bool mtype reflect.Type // set for map types only mkeyprop *Properties // set for map types only mvalprop *Properties // set for map types only size sizer valSize valueSizer // set for bool and numeric types only dec decoder valDec valueDecoder // set for bool and numeric types only // If this is a packable field, this will be the decoder for the packed version of the field. packedDec decoder } // String formats the properties in the protobuf struct field tag style. func (p *Properties) String() string { s := p.Wire s = "," s += strconv.Itoa(p.Tag) if p.Required { s += ",req" } if p.Optional { s += ",opt" } if p.Repeated { s += ",rep" } if p.Packed { s += ",packed" } s += ",name=" + p.OrigName if p.JSONName != p.OrigName { s += ",json=" + p.JSONName } if p.proto3 { s += ",proto3" } if p.oneof { s += ",oneof" } if len(p.Enum) > 0 { s += ",enum=" + p.Enum } if p.HasDefault { s += ",def=" + p.Default } return s } // Parse populates p by parsing a string in the protobuf struct field tag style. func (p *Properties) Parse(s string) { // "bytes,49,opt,name=foo,def=hello!" fields := strings.Split(s, ",") // breaks def=, but handled below. if len(fields) < 2 { fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s) return } p.Wire = fields[0] switch p.Wire { case "varint": p.WireType = WireVarint p.valEnc = (*Buffer).EncodeVarint p.valDec = (*Buffer).DecodeVarint p.valSize = sizeVarint case "fixed32": p.WireType = WireFixed32 p.valEnc = (*Buffer).EncodeFixed32 p.valDec = (*Buffer).DecodeFixed32 p.valSize = sizeFixed32 case "fixed64": p.WireType = WireFixed64 p.valEnc = (*Buffer).EncodeFixed64 p.valDec = (*Buffer).DecodeFixed64 p.valSize = sizeFixed64 case "zigzag32": p.WireType = WireVarint p.valEnc = (*Buffer).EncodeZigzag32 p.valDec = (*Buffer).DecodeZigzag32 p.valSize = sizeZigzag32 case "zigzag64": p.WireType = WireVarint p.valEnc = (*Buffer).EncodeZigzag64 p.valDec = (*Buffer).DecodeZigzag64 p.valSize = sizeZigzag64 case "bytes", "group": p.WireType = WireBytes // no numeric converter for non-numeric types default: fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s) return } var err error p.Tag, err = strconv.Atoi(fields[1]) if err != nil { return } for i := 2; i < len(fields); i++ { f := fields[i] switch { case f == "req": p.Required = true case f == "opt": p.Optional = true case f == "rep": p.Repeated = true case f == "packed": p.Packed = true case strings.HasPrefix(f, "name="): p.OrigName = f[5:] case strings.HasPrefix(f, "json="): p.JSONName = f[5:] case strings.HasPrefix(f, "enum="): p.Enum = f[5:] case f == "proto3": p.proto3 = true case f == "oneof": p.oneof = true case strings.HasPrefix(f, "def="): p.HasDefault = true p.Default = f[4:] // rest of string if i+1 < len(fields) { // Commas aren't escaped, and def is always last. p.Default += "," + strings.Join(fields[i+1:], ",") break } } } } func logNoSliceEnc(t1, t2 reflect.Type) { fmt.Fprintf(os.Stderr, "proto: no slice oenc for %T = []%T\n", t1, t2) } var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() // Initialize the fields for encoding and decoding. func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { p.enc = nil p.dec = nil p.size = nil switch t1 := typ; t1.Kind() { default: fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) // proto3 scalar types case reflect.Bool: p.enc = (*Buffer).enc_proto3_bool p.dec = (*Buffer).dec_proto3_bool p.size = size_proto3_bool case reflect.Int32: p.enc = (*Buffer).enc_proto3_int32 p.dec = (*Buffer).dec_proto3_int32 p.size = size_proto3_int32 case reflect.Uint32: p.enc = (*Buffer).enc_proto3_uint32 p.dec = (*Buffer).dec_proto3_int32 // can reuse p.size = size_proto3_uint32 case reflect.Int64, reflect.Uint64: p.enc = (*Buffer).enc_proto3_int64 p.dec = (*Buffer).dec_proto3_int64 p.size = size_proto3_int64 case reflect.Float32: p.enc = (*Buffer).enc_proto3_uint32 // can just treat them as bits p.dec = (*Buffer).dec_proto3_int32 p.size = size_proto3_uint32 case reflect.Float64: p.enc = (*Buffer).enc_proto3_int64 // can just treat them as bits p.dec = (*Buffer).dec_proto3_int64 p.size = size_proto3_int64 case reflect.String: p.enc = (*Buffer).enc_proto3_string p.dec = (*Buffer).dec_proto3_string p.size = size_proto3_string case reflect.Ptr: switch t2 := t1.Elem(); t2.Kind() { default: fmt.Fprintf(os.Stderr, "proto: no encoder function for %v -> %v\n", t1, t2) break case reflect.Bool: p.enc = (*Buffer).enc_bool p.dec = (*Buffer).dec_bool p.size = size_bool case reflect.Int32: p.enc = (*Buffer).enc_int32 p.dec = (*Buffer).dec_int32 p.size = size_int32 case reflect.Uint32: p.enc = (*Buffer).enc_uint32 p.dec = (*Buffer).dec_int32 // can reuse p.size = size_uint32 case reflect.Int64, reflect.Uint64: p.enc = (*Buffer).enc_int64 p.dec = (*Buffer).dec_int64 p.size = size_int64 case reflect.Float32: p.enc = (*Buffer).enc_uint32 // can just treat them as bits p.dec = (*Buffer).dec_int32 p.size = size_uint32 case reflect.Float64: p.enc = (*Buffer).enc_int64 // can just treat them as bits p.dec = (*Buffer).dec_int64 p.size = size_int64 case reflect.String: p.enc = (*Buffer).enc_string p.dec = (*Buffer).dec_string p.size = size_string case reflect.Struct: p.stype = t1.Elem() p.isMarshaler = isMarshaler(t1) p.isUnmarshaler = isUnmarshaler(t1) if p.Wire == "bytes" { p.enc = (*Buffer).enc_struct_message p.dec = (*Buffer).dec_struct_message p.size = size_struct_message } else { p.enc = (*Buffer).enc_struct_group p.dec = (*Buffer).dec_struct_group p.size = size_struct_group } } case reflect.Slice: switch t2 := t1.Elem(); t2.Kind() { default: logNoSliceEnc(t1, t2) break case reflect.Bool: if p.Packed { p.enc = (*Buffer).enc_slice_packed_bool p.size = size_slice_packed_bool } else { p.enc = (*Buffer).enc_slice_bool p.size = size_slice_bool } p.dec = (*Buffer).dec_slice_bool p.packedDec = (*Buffer).dec_slice_packed_bool case reflect.Int32: if p.Packed { p.enc = (*Buffer).enc_slice_packed_int32 p.size = size_slice_packed_int32 } else { p.enc = (*Buffer).enc_slice_int32 p.size = size_slice_int32 } p.dec = (*Buffer).dec_slice_int32 p.packedDec = (*Buffer).dec_slice_packed_int32 case reflect.Uint32: if p.Packed { p.enc = (*Buffer).enc_slice_packed_uint32 p.size = size_slice_packed_uint32 } else { p.enc = (*Buffer).enc_slice_uint32 p.size = size_slice_uint32 } p.dec = (*Buffer).dec_slice_int32 p.packedDec = (*Buffer).dec_slice_packed_int32 case reflect.Int64, reflect.Uint64: if p.Packed { p.enc = (*Buffer).enc_slice_packed_int64 p.size = size_slice_packed_int64 } else { p.enc = (*Buffer).enc_slice_int64 p.size = size_slice_int64 } p.dec = (*Buffer).dec_slice_int64 p.packedDec = (*Buffer).dec_slice_packed_int64 case reflect.Uint8: p.dec = (*Buffer).dec_slice_byte if p.proto3 { p.enc = (*Buffer).enc_proto3_slice_byte p.size = size_proto3_slice_byte } else { p.enc = (*Buffer).enc_slice_byte p.size = size_slice_byte } case reflect.Float32, reflect.Float64: switch t2.Bits() { case 32: // can just treat them as bits if p.Packed { p.enc = (*Buffer).enc_slice_packed_uint32 p.size = size_slice_packed_uint32 } else { p.enc = (*Buffer).enc_slice_uint32 p.size = size_slice_uint32 } p.dec = (*Buffer).dec_slice_int32 p.packedDec = (*Buffer).dec_slice_packed_int32 case 64: // can just treat them as bits if p.Packed { p.enc = (*Buffer).enc_slice_packed_int64 p.size = size_slice_packed_int64 } else { p.enc = (*Buffer).enc_slice_int64 p.size = size_slice_int64 } p.dec = (*Buffer).dec_slice_int64 p.packedDec = (*Buffer).dec_slice_packed_int64 default: logNoSliceEnc(t1, t2) break } case reflect.String: p.enc = (*Buffer).enc_slice_string p.dec = (*Buffer).dec_slice_string p.size = size_slice_string case reflect.Ptr: switch t3 := t2.Elem(); t3.Kind() { default: fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T -> %T\n", t1, t2, t3) break case reflect.Struct: p.stype = t2.Elem() p.isMarshaler = isMarshaler(t2) p.isUnmarshaler = isUnmarshaler(t2) if p.Wire == "bytes" { p.enc = (*Buffer).enc_slice_struct_message p.dec = (*Buffer).dec_slice_struct_message p.size = size_slice_struct_message } else { p.enc = (*Buffer).enc_slice_struct_group p.dec = (*Buffer).dec_slice_struct_group p.size = size_slice_struct_group } } case reflect.Slice: switch t2.Elem().Kind() { default: fmt.Fprintf(os.Stderr, "proto: no slice elem oenc for %T -> %T -> %T\n", t1, t2, t2.Elem()) break case reflect.Uint8: p.enc = (*Buffer).enc_slice_slice_byte p.dec = (*Buffer).dec_slice_slice_byte p.size = size_slice_slice_byte } } case reflect.Map: p.enc = (*Buffer).enc_new_map p.dec = (*Buffer).dec_new_map p.size = size_new_map p.mtype = t1 p.mkeyprop = &Properties{} p.mkeyprop.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) p.mvalprop = &Properties{} vtype := p.mtype.Elem() if vtype.Kind() != reflect.Ptr && vtype.Kind() != reflect.Slice { // The value type is not a message (*T) or bytes ([]byte), // so we need encoders for the pointer to this type. vtype = reflect.PtrTo(vtype) } p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) } // precalculate tag code wire := p.WireType if p.Packed { wire = WireBytes } x := uint32(p.Tag)<<3 | uint32(wire) i := 0 for i = 0; x > 127; i++ { p.tagbuf[i] = 0x80 | uint8(x&0x7F) x >>= 7 } p.tagbuf[i] = uint8(x) p.tagcode = p.tagbuf[0 : i+1] if p.stype != nil { if lockGetProp { p.sprop = GetProperties(p.stype) } else { p.sprop = getPropertiesLocked(p.stype) } } } var ( marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() ) // isMarshaler reports whether type t implements Marshaler. func isMarshaler(t reflect.Type) bool { // We're checking for (likely) pointer-receiver methods // so if t is not a pointer, something is very wrong. // The calls above only invoke isMarshaler on pointer types. if t.Kind() != reflect.Ptr { panic("proto: misuse of isMarshaler") } return t.Implements(marshalerType) } // isUnmarshaler reports whether type t implements Unmarshaler. func isUnmarshaler(t reflect.Type) bool { // We're checking for (likely) pointer-receiver methods // so if t is not a pointer, something is very wrong. // The calls above only invoke isUnmarshaler on pointer types. if t.Kind() != reflect.Ptr { panic("proto: misuse of isUnmarshaler") } return t.Implements(unmarshalerType) } // Init populates the properties from a protocol buffer struct tag. func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { p.init(typ, name, tag, f, true) } func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructField, lockGetProp bool) { // "bytes,49,opt,def=hello!" p.Name = name p.OrigName = name if f != nil { p.field = toField(f) } if tag == "" { return } p.Parse(tag) p.setEncAndDec(typ, f, lockGetProp) } var ( propertiesMu sync.RWMutex propertiesMap = make(map[reflect.Type]*StructProperties) ) // GetProperties returns the list of properties for the type represented by t. // t must represent a generated struct type of a protocol message. func GetProperties(t reflect.Type) *StructProperties { if t.Kind() != reflect.Struct { panic("proto: type must have kind struct") } // Most calls to GetProperties in a long-running program will be // retrieving details for types we have seen before. propertiesMu.RLock() sprop, ok := propertiesMap[t] propertiesMu.RUnlock() if ok { if collectStats { stats.Chit++ } return sprop } propertiesMu.Lock() sprop = getPropertiesLocked(t) propertiesMu.Unlock() return sprop } // getPropertiesLocked requires that propertiesMu is held. func getPropertiesLocked(t reflect.Type) *StructProperties { if prop, ok := propertiesMap[t]; ok { if collectStats { stats.Chit++ } return prop } if collectStats { stats.Cmiss++ } prop := new(StructProperties) // in case of recursive protos, fill this in now. propertiesMap[t] = prop // build properties prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) || reflect.PtrTo(t).Implements(extendableProtoV1Type) prop.unrecField = invalidField prop.Prop = make([]*Properties, t.NumField()) prop.order = make([]int, t.NumField()) for i := 0; i < t.NumField(); i++ { f := t.Field(i) p := new(Properties) name := f.Name p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) if f.Name == "XXX_InternalExtensions" { // special case p.enc = (*Buffer).enc_exts p.dec = nil // not needed p.size = size_exts } else if f.Name == "XXX_extensions" { // special case p.enc = (*Buffer).enc_map p.dec = nil // not needed p.size = size_map } else if f.Name == "XXX_unrecognized" { // special case prop.unrecField = toField(&f) } oneof := f.Tag.Get("protobuf_oneof") // special case if oneof != "" { // Oneof fields don't use the traditional protobuf tag. p.OrigName = oneof } prop.Prop[i] = p prop.order[i] = i if debug { print(i, " ", f.Name, " ", t.String(), " ") if p.Tag > 0 { print(p.String()) } print("\n") } if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") && oneof == "" { fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]") } } // Re-order prop.order. sort.Sort(prop) type oneofMessage interface { XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) } if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok { var oots []interface{} prop.oneofMarshaler, prop.oneofUnmarshaler, prop.oneofSizer, oots = om.XXX_OneofFuncs() prop.stype = t // Interpret oneof metadata. prop.OneofTypes = make(map[string]*OneofProperties) for _, oot := range oots { oop := &OneofProperties{ Type: reflect.ValueOf(oot).Type(), // *T Prop: new(Properties), } sft := oop.Type.Elem().Field(0) oop.Prop.Name = sft.Name oop.Prop.Parse(sft.Tag.Get("protobuf")) // There will be exactly one interface field that // this new value is assignable to. for i := 0; i < t.NumField(); i++ { f := t.Field(i) if f.Type.Kind() != reflect.Interface { continue } if !oop.Type.AssignableTo(f.Type) { continue } oop.Field = i break } prop.OneofTypes[oop.Prop.OrigName] = oop } } // build required counts // build tags reqCount := 0 prop.decoderOrigNames = make(map[string]int) for i, p := range prop.Prop { if strings.HasPrefix(p.Name, "XXX_") { // Internal fields should not appear in tags/origNames maps. // They are handled specially when encoding and decoding. continue } if p.Required { reqCount++ } prop.decoderTags.put(p.Tag, i) prop.decoderOrigNames[p.OrigName] = i } prop.reqCount = reqCount return prop } // Return the Properties object for the x[0]'th field of the structure. func propByIndex(t reflect.Type, x []int) *Properties { if len(x) != 1 { fmt.Fprintf(os.Stderr, "proto: field index dimension %d (not 1) for type %s\n", len(x), t) return nil } prop := GetProperties(t) return prop.Prop[x[0]] } // Get the address and type of a pointer to a struct from an interface. func getbase(pb Message) (t reflect.Type, b structPointer, err error) { if pb == nil { err = ErrNil return } // get the reflect type of the pointer to the struct. t = reflect.TypeOf(pb) // get the address of the struct. value := reflect.ValueOf(pb) b = toStructPointer(value) return } // A global registry of enum types. // The generated code will register the generated maps by calling RegisterEnum. var enumValueMaps = make(map[string]map[string]int32) // RegisterEnum is called from the generated code to install the enum descriptor // maps into the global table to aid parsing text format protocol buffers. func RegisterEnum(typeName string, unusedNameMap map[int32]string, valueMap map[string]int32) { if _, ok := enumValueMaps[typeName]; ok { panic("proto: duplicate enum registered: " + typeName) } enumValueMaps[typeName] = valueMap } // EnumValueMap returns the mapping from names to integers of the // enum type enumType, or a nil if not found. func EnumValueMap(enumType string) map[string]int32 { return enumValueMaps[enumType] } // A registry of all linked message types. // The string is a fully-qualified proto name ("pkg.Message"). var ( protoTypes = make(map[string]reflect.Type) revProtoTypes = make(map[reflect.Type]string) ) // RegisterType is called from generated code and maps from the fully qualified // proto name to the type (pointer to struct) of the protocol buffer. func RegisterType(x Message, name string) { if _, ok := protoTypes[name]; ok { // TODO: Some day, make this a panic. log.Printf("proto: duplicate proto type registered: %s", name) return } t := reflect.TypeOf(x) protoTypes[name] = t revProtoTypes[t] = name } // MessageName returns the fully-qualified proto name for the given message type. func MessageName(x Message) string { type xname interface { XXX_MessageName() string } if m, ok := x.(xname); ok { return m.XXX_MessageName() } return revProtoTypes[reflect.TypeOf(x)] } // MessageType returns the message type (pointer to struct) for a named message. func MessageType(name string) reflect.Type { return protoTypes[name] } // A registry of all linked proto files. var ( protoFiles = make(map[string][]byte) // file name => fileDescriptor ) // RegisterFile is called from generated code and maps from the // full file name of a .proto file to its compressed FileDescriptorProto. func RegisterFile(filename string, fileDescriptor []byte) { protoFiles[filename] = fileDescriptor } // FileDescriptor returns the compressed FileDescriptorProto for a .proto file. func FileDescriptor(filename string) []byte { return protoFiles[filename] } ================================================ FILE: vendor/github.com/golang/protobuf/proto/proto3_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2014 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "testing" "github.com/golang/protobuf/proto" pb "github.com/golang/protobuf/proto/proto3_proto" tpb "github.com/golang/protobuf/proto/testdata" ) func TestProto3ZeroValues(t *testing.T) { tests := []struct { desc string m proto.Message }{ {"zero message", &pb.Message{}}, {"empty bytes field", &pb.Message{Data: []byte{}}}, } for _, test := range tests { b, err := proto.Marshal(test.m) if err != nil { t.Errorf("%s: proto.Marshal: %v", test.desc, err) continue } if len(b) > 0 { t.Errorf("%s: Encoding is non-empty: %q", test.desc, b) } } } func TestRoundTripProto3(t *testing.T) { m := &pb.Message{ Name: "David", // (2 | 1<<3): 0x0a 0x05 "David" Hilarity: pb.Message_PUNS, // (0 | 2<<3): 0x10 0x01 HeightInCm: 178, // (0 | 3<<3): 0x18 0xb2 0x01 Data: []byte("roboto"), // (2 | 4<<3): 0x20 0x06 "roboto" ResultCount: 47, // (0 | 7<<3): 0x38 0x2f TrueScotsman: true, // (0 | 8<<3): 0x40 0x01 Score: 8.1, // (5 | 9<<3): 0x4d <8.1> Key: []uint64{1, 0xdeadbeef}, Nested: &pb.Nested{ Bunny: "Monty", }, } t.Logf(" m: %v", m) b, err := proto.Marshal(m) if err != nil { t.Fatalf("proto.Marshal: %v", err) } t.Logf(" b: %q", b) m2 := new(pb.Message) if err := proto.Unmarshal(b, m2); err != nil { t.Fatalf("proto.Unmarshal: %v", err) } t.Logf("m2: %v", m2) if !proto.Equal(m, m2) { t.Errorf("proto.Equal returned false:\n m: %v\nm2: %v", m, m2) } } func TestGettersForBasicTypesExist(t *testing.T) { var m pb.Message if got := m.GetNested().GetBunny(); got != "" { t.Errorf("m.GetNested().GetBunny() = %q, want empty string", got) } if got := m.GetNested().GetCute(); got { t.Errorf("m.GetNested().GetCute() = %t, want false", got) } } func TestProto3SetDefaults(t *testing.T) { in := &pb.Message{ Terrain: map[string]*pb.Nested{ "meadow": new(pb.Nested), }, Proto2Field: new(tpb.SubDefaults), Proto2Value: map[string]*tpb.SubDefaults{ "badlands": new(tpb.SubDefaults), }, } got := proto.Clone(in).(*pb.Message) proto.SetDefaults(got) // There are no defaults in proto3. Everything should be the zero value, but // we need to remember to set defaults for nested proto2 messages. want := &pb.Message{ Terrain: map[string]*pb.Nested{ "meadow": new(pb.Nested), }, Proto2Field: &tpb.SubDefaults{N: proto.Int64(7)}, Proto2Value: map[string]*tpb.SubDefaults{ "badlands": &tpb.SubDefaults{N: proto.Int64(7)}, }, } if !proto.Equal(got, want) { t.Errorf("with in = %v\nproto.SetDefaults(in) =>\ngot %v\nwant %v", in, got, want) } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/size2_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2012 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto import ( "testing" ) // This is a separate file and package from size_test.go because that one uses // generated messages and thus may not be in package proto without having a circular // dependency, whereas this file tests unexported details of size.go. func TestVarintSize(t *testing.T) { // Check the edge cases carefully. testCases := []struct { n uint64 size int }{ {0, 1}, {1, 1}, {127, 1}, {128, 2}, {16383, 2}, {16384, 3}, {1<<63 - 1, 9}, {1 << 63, 10}, } for _, tc := range testCases { size := sizeVarint(tc.n) if size != tc.size { t.Errorf("sizeVarint(%d) = %d, want %d", tc.n, size, tc.size) } } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/size_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2012 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "log" "strings" "testing" . "github.com/golang/protobuf/proto" proto3pb "github.com/golang/protobuf/proto/proto3_proto" pb "github.com/golang/protobuf/proto/testdata" ) var messageWithExtension1 = &pb.MyMessage{Count: Int32(7)} // messageWithExtension2 is in equal_test.go. var messageWithExtension3 = &pb.MyMessage{Count: Int32(8)} func init() { if err := SetExtension(messageWithExtension1, pb.E_Ext_More, &pb.Ext{Data: String("Abbott")}); err != nil { log.Panicf("SetExtension: %v", err) } if err := SetExtension(messageWithExtension3, pb.E_Ext_More, &pb.Ext{Data: String("Costello")}); err != nil { log.Panicf("SetExtension: %v", err) } // Force messageWithExtension3 to have the extension encoded. Marshal(messageWithExtension3) } var SizeTests = []struct { desc string pb Message }{ {"empty", &pb.OtherMessage{}}, // Basic types. {"bool", &pb.Defaults{F_Bool: Bool(true)}}, {"int32", &pb.Defaults{F_Int32: Int32(12)}}, {"negative int32", &pb.Defaults{F_Int32: Int32(-1)}}, {"small int64", &pb.Defaults{F_Int64: Int64(1)}}, {"big int64", &pb.Defaults{F_Int64: Int64(1 << 20)}}, {"negative int64", &pb.Defaults{F_Int64: Int64(-1)}}, {"fixed32", &pb.Defaults{F_Fixed32: Uint32(71)}}, {"fixed64", &pb.Defaults{F_Fixed64: Uint64(72)}}, {"uint32", &pb.Defaults{F_Uint32: Uint32(123)}}, {"uint64", &pb.Defaults{F_Uint64: Uint64(124)}}, {"float", &pb.Defaults{F_Float: Float32(12.6)}}, {"double", &pb.Defaults{F_Double: Float64(13.9)}}, {"string", &pb.Defaults{F_String: String("niles")}}, {"bytes", &pb.Defaults{F_Bytes: []byte("wowsa")}}, {"bytes, empty", &pb.Defaults{F_Bytes: []byte{}}}, {"sint32", &pb.Defaults{F_Sint32: Int32(65)}}, {"sint64", &pb.Defaults{F_Sint64: Int64(67)}}, {"enum", &pb.Defaults{F_Enum: pb.Defaults_BLUE.Enum()}}, // Repeated. {"empty repeated bool", &pb.MoreRepeated{Bools: []bool{}}}, {"repeated bool", &pb.MoreRepeated{Bools: []bool{false, true, true, false}}}, {"packed repeated bool", &pb.MoreRepeated{BoolsPacked: []bool{false, true, true, false, true, true, true}}}, {"repeated int32", &pb.MoreRepeated{Ints: []int32{1, 12203, 1729, -1}}}, {"repeated int32 packed", &pb.MoreRepeated{IntsPacked: []int32{1, 12203, 1729}}}, {"repeated int64 packed", &pb.MoreRepeated{Int64SPacked: []int64{ // Need enough large numbers to verify that the header is counting the number of bytes // for the field, not the number of elements. 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, }}}, {"repeated string", &pb.MoreRepeated{Strings: []string{"r", "ken", "gri"}}}, {"repeated fixed", &pb.MoreRepeated{Fixeds: []uint32{1, 2, 3, 4}}}, // Nested. {"nested", &pb.OldMessage{Nested: &pb.OldMessage_Nested{Name: String("whatever")}}}, {"group", &pb.GroupOld{G: &pb.GroupOld_G{X: Int32(12345)}}}, // Other things. {"unrecognized", &pb.MoreRepeated{XXX_unrecognized: []byte{13<<3 | 0, 4}}}, {"extension (unencoded)", messageWithExtension1}, {"extension (encoded)", messageWithExtension3}, // proto3 message {"proto3 empty", &proto3pb.Message{}}, {"proto3 bool", &proto3pb.Message{TrueScotsman: true}}, {"proto3 int64", &proto3pb.Message{ResultCount: 1}}, {"proto3 uint32", &proto3pb.Message{HeightInCm: 123}}, {"proto3 float", &proto3pb.Message{Score: 12.6}}, {"proto3 string", &proto3pb.Message{Name: "Snezana"}}, {"proto3 bytes", &proto3pb.Message{Data: []byte("wowsa")}}, {"proto3 bytes, empty", &proto3pb.Message{Data: []byte{}}}, {"proto3 enum", &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, {"proto3 map field with empty bytes", &proto3pb.MessageWithMap{ByteMapping: map[bool][]byte{false: []byte{}}}}, {"map field", &pb.MessageWithMap{NameMapping: map[int32]string{1: "Rob", 7: "Andrew"}}}, {"map field with message", &pb.MessageWithMap{MsgMapping: map[int64]*pb.FloatingPoint{0x7001: &pb.FloatingPoint{F: Float64(2.0)}}}}, {"map field with bytes", &pb.MessageWithMap{ByteMapping: map[bool][]byte{true: []byte("this time for sure")}}}, {"map field with empty bytes", &pb.MessageWithMap{ByteMapping: map[bool][]byte{true: []byte{}}}}, {"map field with big entry", &pb.MessageWithMap{NameMapping: map[int32]string{8: strings.Repeat("x", 125)}}}, {"map field with big key and val", &pb.MessageWithMap{StrToStr: map[string]string{strings.Repeat("x", 70): strings.Repeat("y", 70)}}}, {"map field with big numeric key", &pb.MessageWithMap{NameMapping: map[int32]string{0xf00d: "om nom nom"}}}, {"oneof not set", &pb.Oneof{}}, {"oneof bool", &pb.Oneof{Union: &pb.Oneof_F_Bool{true}}}, {"oneof zero int32", &pb.Oneof{Union: &pb.Oneof_F_Int32{0}}}, {"oneof big int32", &pb.Oneof{Union: &pb.Oneof_F_Int32{1 << 20}}}, {"oneof int64", &pb.Oneof{Union: &pb.Oneof_F_Int64{42}}}, {"oneof fixed32", &pb.Oneof{Union: &pb.Oneof_F_Fixed32{43}}}, {"oneof fixed64", &pb.Oneof{Union: &pb.Oneof_F_Fixed64{44}}}, {"oneof uint32", &pb.Oneof{Union: &pb.Oneof_F_Uint32{45}}}, {"oneof uint64", &pb.Oneof{Union: &pb.Oneof_F_Uint64{46}}}, {"oneof float", &pb.Oneof{Union: &pb.Oneof_F_Float{47.1}}}, {"oneof double", &pb.Oneof{Union: &pb.Oneof_F_Double{48.9}}}, {"oneof string", &pb.Oneof{Union: &pb.Oneof_F_String{"Rhythmic Fman"}}}, {"oneof bytes", &pb.Oneof{Union: &pb.Oneof_F_Bytes{[]byte("let go")}}}, {"oneof sint32", &pb.Oneof{Union: &pb.Oneof_F_Sint32{50}}}, {"oneof sint64", &pb.Oneof{Union: &pb.Oneof_F_Sint64{51}}}, {"oneof enum", &pb.Oneof{Union: &pb.Oneof_F_Enum{pb.MyMessage_BLUE}}}, {"message for oneof", &pb.GoTestField{Label: String("k"), Type: String("v")}}, {"oneof message", &pb.Oneof{Union: &pb.Oneof_F_Message{&pb.GoTestField{Label: String("k"), Type: String("v")}}}}, {"oneof group", &pb.Oneof{Union: &pb.Oneof_FGroup{&pb.Oneof_F_Group{X: Int32(52)}}}}, {"oneof largest tag", &pb.Oneof{Union: &pb.Oneof_F_Largest_Tag{1}}}, {"multiple oneofs", &pb.Oneof{Union: &pb.Oneof_F_Int32{1}, Tormato: &pb.Oneof_Value{2}}}, } func TestSize(t *testing.T) { for _, tc := range SizeTests { size := Size(tc.pb) b, err := Marshal(tc.pb) if err != nil { t.Errorf("%v: Marshal failed: %v", tc.desc, err) continue } if size != len(b) { t.Errorf("%v: Size(%v) = %d, want %d", tc.desc, tc.pb, size, len(b)) t.Logf("%v: bytes: %#v", tc.desc, b) } } } ================================================ FILE: vendor/github.com/golang/protobuf/proto/text.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto // Functions for writing the text protocol buffer format. import ( "bufio" "bytes" "encoding" "errors" "fmt" "io" "log" "math" "reflect" "sort" "strings" ) var ( newline = []byte("\n") spaces = []byte(" ") gtNewline = []byte(">\n") endBraceNewline = []byte("}\n") backslashN = []byte{'\\', 'n'} backslashR = []byte{'\\', 'r'} backslashT = []byte{'\\', 't'} backslashDQ = []byte{'\\', '"'} backslashBS = []byte{'\\', '\\'} posInf = []byte("inf") negInf = []byte("-inf") nan = []byte("nan") ) type writer interface { io.Writer WriteByte(byte) error } // textWriter is an io.Writer that tracks its indentation level. type textWriter struct { ind int complete bool // if the current position is a complete line compact bool // whether to write out as a one-liner w writer } func (w *textWriter) WriteString(s string) (n int, err error) { if !strings.Contains(s, "\n") { if !w.compact && w.complete { w.writeIndent() } w.complete = false return io.WriteString(w.w, s) } // WriteString is typically called without newlines, so this // codepath and its copy are rare. We copy to avoid // duplicating all of Write's logic here. return w.Write([]byte(s)) } func (w *textWriter) Write(p []byte) (n int, err error) { newlines := bytes.Count(p, newline) if newlines == 0 { if !w.compact && w.complete { w.writeIndent() } n, err = w.w.Write(p) w.complete = false return n, err } frags := bytes.SplitN(p, newline, newlines+1) if w.compact { for i, frag := range frags { if i > 0 { if err := w.w.WriteByte(' '); err != nil { return n, err } n++ } nn, err := w.w.Write(frag) n += nn if err != nil { return n, err } } return n, nil } for i, frag := range frags { if w.complete { w.writeIndent() } nn, err := w.w.Write(frag) n += nn if err != nil { return n, err } if i+1 < len(frags) { if err := w.w.WriteByte('\n'); err != nil { return n, err } n++ } } w.complete = len(frags[len(frags)-1]) == 0 return n, nil } func (w *textWriter) WriteByte(c byte) error { if w.compact && c == '\n' { c = ' ' } if !w.compact && w.complete { w.writeIndent() } err := w.w.WriteByte(c) w.complete = c == '\n' return err } func (w *textWriter) indent() { w.ind++ } func (w *textWriter) unindent() { if w.ind == 0 { log.Print("proto: textWriter unindented too far") return } w.ind-- } func writeName(w *textWriter, props *Properties) error { if _, err := w.WriteString(props.OrigName); err != nil { return err } if props.Wire != "group" { return w.WriteByte(':') } return nil } // raw is the interface satisfied by RawMessage. type raw interface { Bytes() []byte } func requiresQuotes(u string) bool { // When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted. for _, ch := range u { switch { case ch == '.' || ch == '/' || ch == '_': continue case '0' <= ch && ch <= '9': continue case 'A' <= ch && ch <= 'Z': continue case 'a' <= ch && ch <= 'z': continue default: return true } } return false } // isAny reports whether sv is a google.protobuf.Any message func isAny(sv reflect.Value) bool { type wkt interface { XXX_WellKnownType() string } t, ok := sv.Addr().Interface().(wkt) return ok && t.XXX_WellKnownType() == "Any" } // writeProto3Any writes an expanded google.protobuf.Any message. // // It returns (false, nil) if sv value can't be unmarshaled (e.g. because // required messages are not linked in). // // It returns (true, error) when sv was written in expanded format or an error // was encountered. func (tm *TextMarshaler) writeProto3Any(w *textWriter, sv reflect.Value) (bool, error) { turl := sv.FieldByName("TypeUrl") val := sv.FieldByName("Value") if !turl.IsValid() || !val.IsValid() { return true, errors.New("proto: invalid google.protobuf.Any message") } b, ok := val.Interface().([]byte) if !ok { return true, errors.New("proto: invalid google.protobuf.Any message") } parts := strings.Split(turl.String(), "/") mt := MessageType(parts[len(parts)-1]) if mt == nil { return false, nil } m := reflect.New(mt.Elem()) if err := Unmarshal(b, m.Interface().(Message)); err != nil { return false, nil } w.Write([]byte("[")) u := turl.String() if requiresQuotes(u) { writeString(w, u) } else { w.Write([]byte(u)) } if w.compact { w.Write([]byte("]:<")) } else { w.Write([]byte("]: <\n")) w.ind++ } if err := tm.writeStruct(w, m.Elem()); err != nil { return true, err } if w.compact { w.Write([]byte("> ")) } else { w.ind-- w.Write([]byte(">\n")) } return true, nil } func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { if tm.ExpandAny && isAny(sv) { if canExpand, err := tm.writeProto3Any(w, sv); canExpand { return err } } st := sv.Type() sprops := GetProperties(st) for i := 0; i < sv.NumField(); i++ { fv := sv.Field(i) props := sprops.Prop[i] name := st.Field(i).Name if strings.HasPrefix(name, "XXX_") { // There are two XXX_ fields: // XXX_unrecognized []byte // XXX_extensions map[int32]proto.Extension // The first is handled here; // the second is handled at the bottom of this function. if name == "XXX_unrecognized" && !fv.IsNil() { if err := writeUnknownStruct(w, fv.Interface().([]byte)); err != nil { return err } } continue } if fv.Kind() == reflect.Ptr && fv.IsNil() { // Field not filled in. This could be an optional field or // a required field that wasn't filled in. Either way, there // isn't anything we can show for it. continue } if fv.Kind() == reflect.Slice && fv.IsNil() { // Repeated field that is empty, or a bytes field that is unused. continue } if props.Repeated && fv.Kind() == reflect.Slice { // Repeated field. for j := 0; j < fv.Len(); j++ { if err := writeName(w, props); err != nil { return err } if !w.compact { if err := w.WriteByte(' '); err != nil { return err } } v := fv.Index(j) if v.Kind() == reflect.Ptr && v.IsNil() { // A nil message in a repeated field is not valid, // but we can handle that more gracefully than panicking. if _, err := w.Write([]byte("\n")); err != nil { return err } continue } if err := tm.writeAny(w, v, props); err != nil { return err } if err := w.WriteByte('\n'); err != nil { return err } } continue } if fv.Kind() == reflect.Map { // Map fields are rendered as a repeated struct with key/value fields. keys := fv.MapKeys() sort.Sort(mapKeys(keys)) for _, key := range keys { val := fv.MapIndex(key) if err := writeName(w, props); err != nil { return err } if !w.compact { if err := w.WriteByte(' '); err != nil { return err } } // open struct if err := w.WriteByte('<'); err != nil { return err } if !w.compact { if err := w.WriteByte('\n'); err != nil { return err } } w.indent() // key if _, err := w.WriteString("key:"); err != nil { return err } if !w.compact { if err := w.WriteByte(' '); err != nil { return err } } if err := tm.writeAny(w, key, props.mkeyprop); err != nil { return err } if err := w.WriteByte('\n'); err != nil { return err } // nil values aren't legal, but we can avoid panicking because of them. if val.Kind() != reflect.Ptr || !val.IsNil() { // value if _, err := w.WriteString("value:"); err != nil { return err } if !w.compact { if err := w.WriteByte(' '); err != nil { return err } } if err := tm.writeAny(w, val, props.mvalprop); err != nil { return err } if err := w.WriteByte('\n'); err != nil { return err } } // close struct w.unindent() if err := w.WriteByte('>'); err != nil { return err } if err := w.WriteByte('\n'); err != nil { return err } } continue } if props.proto3 && fv.Kind() == reflect.Slice && fv.Len() == 0 { // empty bytes field continue } if fv.Kind() != reflect.Ptr && fv.Kind() != reflect.Slice { // proto3 non-repeated scalar field; skip if zero value if isProto3Zero(fv) { continue } } if fv.Kind() == reflect.Interface { // Check if it is a oneof. if st.Field(i).Tag.Get("protobuf_oneof") != "" { // fv is nil, or holds a pointer to generated struct. // That generated struct has exactly one field, // which has a protobuf struct tag. if fv.IsNil() { continue } inner := fv.Elem().Elem() // interface -> *T -> T tag := inner.Type().Field(0).Tag.Get("protobuf") props = new(Properties) // Overwrite the outer props var, but not its pointee. props.Parse(tag) // Write the value in the oneof, not the oneof itself. fv = inner.Field(0) // Special case to cope with malformed messages gracefully: // If the value in the oneof is a nil pointer, don't panic // in writeAny. if fv.Kind() == reflect.Ptr && fv.IsNil() { // Use errors.New so writeAny won't render quotes. msg := errors.New("/* nil */") fv = reflect.ValueOf(&msg).Elem() } } } if err := writeName(w, props); err != nil { return err } if !w.compact { if err := w.WriteByte(' '); err != nil { return err } } if b, ok := fv.Interface().(raw); ok { if err := writeRaw(w, b.Bytes()); err != nil { return err } continue } // Enums have a String method, so writeAny will work fine. if err := tm.writeAny(w, fv, props); err != nil { return err } if err := w.WriteByte('\n'); err != nil { return err } } // Extensions (the XXX_extensions field). pv := sv.Addr() if _, ok := extendable(pv.Interface()); ok { if err := tm.writeExtensions(w, pv); err != nil { return err } } return nil } // writeRaw writes an uninterpreted raw message. func writeRaw(w *textWriter, b []byte) error { if err := w.WriteByte('<'); err != nil { return err } if !w.compact { if err := w.WriteByte('\n'); err != nil { return err } } w.indent() if err := writeUnknownStruct(w, b); err != nil { return err } w.unindent() if err := w.WriteByte('>'); err != nil { return err } return nil } // writeAny writes an arbitrary field. func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) // Floats have special cases. if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 { x := v.Float() var b []byte switch { case math.IsInf(x, 1): b = posInf case math.IsInf(x, -1): b = negInf case math.IsNaN(x): b = nan } if b != nil { _, err := w.Write(b) return err } // Other values are handled below. } // We don't attempt to serialise every possible value type; only those // that can occur in protocol buffers. switch v.Kind() { case reflect.Slice: // Should only be a []byte; repeated fields are handled in writeStruct. if err := writeString(w, string(v.Bytes())); err != nil { return err } case reflect.String: if err := writeString(w, v.String()); err != nil { return err } case reflect.Struct: // Required/optional group/message. var bra, ket byte = '<', '>' if props != nil && props.Wire == "group" { bra, ket = '{', '}' } if err := w.WriteByte(bra); err != nil { return err } if !w.compact { if err := w.WriteByte('\n'); err != nil { return err } } w.indent() if etm, ok := v.Interface().(encoding.TextMarshaler); ok { text, err := etm.MarshalText() if err != nil { return err } if _, err = w.Write(text); err != nil { return err } } else if err := tm.writeStruct(w, v); err != nil { return err } w.unindent() if err := w.WriteByte(ket); err != nil { return err } default: _, err := fmt.Fprint(w, v.Interface()) return err } return nil } // equivalent to C's isprint. func isprint(c byte) bool { return c >= 0x20 && c < 0x7f } // writeString writes a string in the protocol buffer text format. // It is similar to strconv.Quote except we don't use Go escape sequences, // we treat the string as a byte sequence, and we use octal escapes. // These differences are to maintain interoperability with the other // languages' implementations of the text format. func writeString(w *textWriter, s string) error { // use WriteByte here to get any needed indent if err := w.WriteByte('"'); err != nil { return err } // Loop over the bytes, not the runes. for i := 0; i < len(s); i++ { var err error // Divergence from C++: we don't escape apostrophes. // There's no need to escape them, and the C++ parser // copes with a naked apostrophe. switch c := s[i]; c { case '\n': _, err = w.w.Write(backslashN) case '\r': _, err = w.w.Write(backslashR) case '\t': _, err = w.w.Write(backslashT) case '"': _, err = w.w.Write(backslashDQ) case '\\': _, err = w.w.Write(backslashBS) default: if isprint(c) { err = w.w.WriteByte(c) } else { _, err = fmt.Fprintf(w.w, "\\%03o", c) } } if err != nil { return err } } return w.WriteByte('"') } func writeUnknownStruct(w *textWriter, data []byte) (err error) { if !w.compact { if _, err := fmt.Fprintf(w, "/* %d unknown bytes */\n", len(data)); err != nil { return err } } b := NewBuffer(data) for b.index < len(b.buf) { x, err := b.DecodeVarint() if err != nil { _, err := fmt.Fprintf(w, "/* %v */\n", err) return err } wire, tag := x&7, x>>3 if wire == WireEndGroup { w.unindent() if _, err := w.Write(endBraceNewline); err != nil { return err } continue } if _, err := fmt.Fprint(w, tag); err != nil { return err } if wire != WireStartGroup { if err := w.WriteByte(':'); err != nil { return err } } if !w.compact || wire == WireStartGroup { if err := w.WriteByte(' '); err != nil { return err } } switch wire { case WireBytes: buf, e := b.DecodeRawBytes(false) if e == nil { _, err = fmt.Fprintf(w, "%q", buf) } else { _, err = fmt.Fprintf(w, "/* %v */", e) } case WireFixed32: x, err = b.DecodeFixed32() err = writeUnknownInt(w, x, err) case WireFixed64: x, err = b.DecodeFixed64() err = writeUnknownInt(w, x, err) case WireStartGroup: err = w.WriteByte('{') w.indent() case WireVarint: x, err = b.DecodeVarint() err = writeUnknownInt(w, x, err) default: _, err = fmt.Fprintf(w, "/* unknown wire type %d */", wire) } if err != nil { return err } if err = w.WriteByte('\n'); err != nil { return err } } return nil } func writeUnknownInt(w *textWriter, x uint64, err error) error { if err == nil { _, err = fmt.Fprint(w, x) } else { _, err = fmt.Fprintf(w, "/* %v */", err) } return err } type int32Slice []int32 func (s int32Slice) Len() int { return len(s) } func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // writeExtensions writes all the extensions in pv. // pv is assumed to be a pointer to a protocol message struct that is extendable. func (tm *TextMarshaler) writeExtensions(w *textWriter, pv reflect.Value) error { emap := extensionMaps[pv.Type().Elem()] ep, _ := extendable(pv.Interface()) // Order the extensions by ID. // This isn't strictly necessary, but it will give us // canonical output, which will also make testing easier. m, mu := ep.extensionsRead() if m == nil { return nil } mu.Lock() ids := make([]int32, 0, len(m)) for id := range m { ids = append(ids, id) } sort.Sort(int32Slice(ids)) mu.Unlock() for _, extNum := range ids { ext := m[extNum] var desc *ExtensionDesc if emap != nil { desc = emap[extNum] } if desc == nil { // Unknown extension. if err := writeUnknownStruct(w, ext.enc); err != nil { return err } continue } pb, err := GetExtension(ep, desc) if err != nil { return fmt.Errorf("failed getting extension: %v", err) } // Repeated extensions will appear as a slice. if !desc.repeated() { if err := tm.writeExtension(w, desc.Name, pb); err != nil { return err } } else { v := reflect.ValueOf(pb) for i := 0; i < v.Len(); i++ { if err := tm.writeExtension(w, desc.Name, v.Index(i).Interface()); err != nil { return err } } } } return nil } func (tm *TextMarshaler) writeExtension(w *textWriter, name string, pb interface{}) error { if _, err := fmt.Fprintf(w, "[%s]:", name); err != nil { return err } if !w.compact { if err := w.WriteByte(' '); err != nil { return err } } if err := tm.writeAny(w, reflect.ValueOf(pb), nil); err != nil { return err } if err := w.WriteByte('\n'); err != nil { return err } return nil } func (w *textWriter) writeIndent() { if !w.complete { return } remain := w.ind * 2 for remain > 0 { n := remain if n > len(spaces) { n = len(spaces) } w.w.Write(spaces[:n]) remain -= n } w.complete = false } // TextMarshaler is a configurable text format marshaler. type TextMarshaler struct { Compact bool // use compact text format (one line). ExpandAny bool // expand google.protobuf.Any messages of known types } // Marshal writes a given protocol buffer in text format. // The only errors returned are from w. func (tm *TextMarshaler) Marshal(w io.Writer, pb Message) error { val := reflect.ValueOf(pb) if pb == nil || val.IsNil() { w.Write([]byte("")) return nil } var bw *bufio.Writer ww, ok := w.(writer) if !ok { bw = bufio.NewWriter(w) ww = bw } aw := &textWriter{ w: ww, complete: true, compact: tm.Compact, } if etm, ok := pb.(encoding.TextMarshaler); ok { text, err := etm.MarshalText() if err != nil { return err } if _, err = aw.Write(text); err != nil { return err } if bw != nil { return bw.Flush() } return nil } // Dereference the received pointer so we don't have outer < and >. v := reflect.Indirect(val) if err := tm.writeStruct(aw, v); err != nil { return err } if bw != nil { return bw.Flush() } return nil } // Text is the same as Marshal, but returns the string directly. func (tm *TextMarshaler) Text(pb Message) string { var buf bytes.Buffer tm.Marshal(&buf, pb) return buf.String() } var ( defaultTextMarshaler = TextMarshaler{} compactTextMarshaler = TextMarshaler{Compact: true} ) // TODO: consider removing some of the Marshal functions below. // MarshalText writes a given protocol buffer in text format. // The only errors returned are from w. func MarshalText(w io.Writer, pb Message) error { return defaultTextMarshaler.Marshal(w, pb) } // MarshalTextString is the same as MarshalText, but returns the string directly. func MarshalTextString(pb Message) string { return defaultTextMarshaler.Text(pb) } // CompactText writes a given protocol buffer in compact text format (one line). func CompactText(w io.Writer, pb Message) error { return compactTextMarshaler.Marshal(w, pb) } // CompactTextString is the same as CompactText, but returns the string directly. func CompactTextString(pb Message) string { return compactTextMarshaler.Text(pb) } ================================================ FILE: vendor/github.com/golang/protobuf/proto/text_parser.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto // Functions for parsing the Text protocol buffer format. // TODO: message sets. import ( "encoding" "errors" "fmt" "reflect" "strconv" "strings" "unicode/utf8" ) // Error string emitted when deserializing Any and fields are already set const anyRepeatedlyUnpacked = "Any message unpacked multiple times, or %q already set" type ParseError struct { Message string Line int // 1-based line number Offset int // 0-based byte offset from start of input } func (p *ParseError) Error() string { if p.Line == 1 { // show offset only for first line return fmt.Sprintf("line 1.%d: %v", p.Offset, p.Message) } return fmt.Sprintf("line %d: %v", p.Line, p.Message) } type token struct { value string err *ParseError line int // line number offset int // byte number from start of input, not start of line unquoted string // the unquoted version of value, if it was a quoted string } func (t *token) String() string { if t.err == nil { return fmt.Sprintf("%q (line=%d, offset=%d)", t.value, t.line, t.offset) } return fmt.Sprintf("parse error: %v", t.err) } type textParser struct { s string // remaining input done bool // whether the parsing is finished (success or error) backed bool // whether back() was called offset, line int cur token } func newTextParser(s string) *textParser { p := new(textParser) p.s = s p.line = 1 p.cur.line = 1 return p } func (p *textParser) errorf(format string, a ...interface{}) *ParseError { pe := &ParseError{fmt.Sprintf(format, a...), p.cur.line, p.cur.offset} p.cur.err = pe p.done = true return pe } // Numbers and identifiers are matched by [-+._A-Za-z0-9] func isIdentOrNumberChar(c byte) bool { switch { case 'A' <= c && c <= 'Z', 'a' <= c && c <= 'z': return true case '0' <= c && c <= '9': return true } switch c { case '-', '+', '.', '_': return true } return false } func isWhitespace(c byte) bool { switch c { case ' ', '\t', '\n', '\r': return true } return false } func isQuote(c byte) bool { switch c { case '"', '\'': return true } return false } func (p *textParser) skipWhitespace() { i := 0 for i < len(p.s) && (isWhitespace(p.s[i]) || p.s[i] == '#') { if p.s[i] == '#' { // comment; skip to end of line or input for i < len(p.s) && p.s[i] != '\n' { i++ } if i == len(p.s) { break } } if p.s[i] == '\n' { p.line++ } i++ } p.offset += i p.s = p.s[i:len(p.s)] if len(p.s) == 0 { p.done = true } } func (p *textParser) advance() { // Skip whitespace p.skipWhitespace() if p.done { return } // Start of non-whitespace p.cur.err = nil p.cur.offset, p.cur.line = p.offset, p.line p.cur.unquoted = "" switch p.s[0] { case '<', '>', '{', '}', ':', '[', ']', ';', ',', '/': // Single symbol p.cur.value, p.s = p.s[0:1], p.s[1:len(p.s)] case '"', '\'': // Quoted string i := 1 for i < len(p.s) && p.s[i] != p.s[0] && p.s[i] != '\n' { if p.s[i] == '\\' && i+1 < len(p.s) { // skip escaped char i++ } i++ } if i >= len(p.s) || p.s[i] != p.s[0] { p.errorf("unmatched quote") return } unq, err := unquoteC(p.s[1:i], rune(p.s[0])) if err != nil { p.errorf("invalid quoted string %s: %v", p.s[0:i+1], err) return } p.cur.value, p.s = p.s[0:i+1], p.s[i+1:len(p.s)] p.cur.unquoted = unq default: i := 0 for i < len(p.s) && isIdentOrNumberChar(p.s[i]) { i++ } if i == 0 { p.errorf("unexpected byte %#x", p.s[0]) return } p.cur.value, p.s = p.s[0:i], p.s[i:len(p.s)] } p.offset += len(p.cur.value) } var ( errBadUTF8 = errors.New("proto: bad UTF-8") errBadHex = errors.New("proto: bad hexadecimal") ) func unquoteC(s string, quote rune) (string, error) { // This is based on C++'s tokenizer.cc. // Despite its name, this is *not* parsing C syntax. // For instance, "\0" is an invalid quoted string. // Avoid allocation in trivial cases. simple := true for _, r := range s { if r == '\\' || r == quote { simple = false break } } if simple { return s, nil } buf := make([]byte, 0, 3*len(s)/2) for len(s) > 0 { r, n := utf8.DecodeRuneInString(s) if r == utf8.RuneError && n == 1 { return "", errBadUTF8 } s = s[n:] if r != '\\' { if r < utf8.RuneSelf { buf = append(buf, byte(r)) } else { buf = append(buf, string(r)...) } continue } ch, tail, err := unescape(s) if err != nil { return "", err } buf = append(buf, ch...) s = tail } return string(buf), nil } func unescape(s string) (ch string, tail string, err error) { r, n := utf8.DecodeRuneInString(s) if r == utf8.RuneError && n == 1 { return "", "", errBadUTF8 } s = s[n:] switch r { case 'a': return "\a", s, nil case 'b': return "\b", s, nil case 'f': return "\f", s, nil case 'n': return "\n", s, nil case 'r': return "\r", s, nil case 't': return "\t", s, nil case 'v': return "\v", s, nil case '?': return "?", s, nil // trigraph workaround case '\'', '"', '\\': return string(r), s, nil case '0', '1', '2', '3', '4', '5', '6', '7', 'x', 'X': if len(s) < 2 { return "", "", fmt.Errorf(`\%c requires 2 following digits`, r) } base := 8 ss := s[:2] s = s[2:] if r == 'x' || r == 'X' { base = 16 } else { ss = string(r) + ss } i, err := strconv.ParseUint(ss, base, 8) if err != nil { return "", "", err } return string([]byte{byte(i)}), s, nil case 'u', 'U': n := 4 if r == 'U' { n = 8 } if len(s) < n { return "", "", fmt.Errorf(`\%c requires %d digits`, r, n) } bs := make([]byte, n/2) for i := 0; i < n; i += 2 { a, ok1 := unhex(s[i]) b, ok2 := unhex(s[i+1]) if !ok1 || !ok2 { return "", "", errBadHex } bs[i/2] = a<<4 | b } s = s[n:] return string(bs), s, nil } return "", "", fmt.Errorf(`unknown escape \%c`, r) } // Adapted from src/pkg/strconv/quote.go. func unhex(b byte) (v byte, ok bool) { switch { case '0' <= b && b <= '9': return b - '0', true case 'a' <= b && b <= 'f': return b - 'a' + 10, true case 'A' <= b && b <= 'F': return b - 'A' + 10, true } return 0, false } // Back off the parser by one token. Can only be done between calls to next(). // It makes the next advance() a no-op. func (p *textParser) back() { p.backed = true } // Advances the parser and returns the new current token. func (p *textParser) next() *token { if p.backed || p.done { p.backed = false return &p.cur } p.advance() if p.done { p.cur.value = "" } else if len(p.cur.value) > 0 && isQuote(p.cur.value[0]) { // Look for multiple quoted strings separated by whitespace, // and concatenate them. cat := p.cur for { p.skipWhitespace() if p.done || !isQuote(p.s[0]) { break } p.advance() if p.cur.err != nil { return &p.cur } cat.value += " " + p.cur.value cat.unquoted += p.cur.unquoted } p.done = false // parser may have seen EOF, but we want to return cat p.cur = cat } return &p.cur } func (p *textParser) consumeToken(s string) error { tok := p.next() if tok.err != nil { return tok.err } if tok.value != s { p.back() return p.errorf("expected %q, found %q", s, tok.value) } return nil } // Return a RequiredNotSetError indicating which required field was not set. func (p *textParser) missingRequiredFieldError(sv reflect.Value) *RequiredNotSetError { st := sv.Type() sprops := GetProperties(st) for i := 0; i < st.NumField(); i++ { if !isNil(sv.Field(i)) { continue } props := sprops.Prop[i] if props.Required { return &RequiredNotSetError{fmt.Sprintf("%v.%v", st, props.OrigName)} } } return &RequiredNotSetError{fmt.Sprintf("%v.", st)} // should not happen } // Returns the index in the struct for the named field, as well as the parsed tag properties. func structFieldByName(sprops *StructProperties, name string) (int, *Properties, bool) { i, ok := sprops.decoderOrigNames[name] if ok { return i, sprops.Prop[i], true } return -1, nil, false } // Consume a ':' from the input stream (if the next token is a colon), // returning an error if a colon is needed but not present. func (p *textParser) checkForColon(props *Properties, typ reflect.Type) *ParseError { tok := p.next() if tok.err != nil { return tok.err } if tok.value != ":" { // Colon is optional when the field is a group or message. needColon := true switch props.Wire { case "group": needColon = false case "bytes": // A "bytes" field is either a message, a string, or a repeated field; // those three become *T, *string and []T respectively, so we can check for // this field being a pointer to a non-string. if typ.Kind() == reflect.Ptr { // *T or *string if typ.Elem().Kind() == reflect.String { break } } else if typ.Kind() == reflect.Slice { // []T or []*T if typ.Elem().Kind() != reflect.Ptr { break } } else if typ.Kind() == reflect.String { // The proto3 exception is for a string field, // which requires a colon. break } needColon = false } if needColon { return p.errorf("expected ':', found %q", tok.value) } p.back() } return nil } func (p *textParser) readStruct(sv reflect.Value, terminator string) error { st := sv.Type() sprops := GetProperties(st) reqCount := sprops.reqCount var reqFieldErr error fieldSet := make(map[string]bool) // A struct is a sequence of "name: value", terminated by one of // '>' or '}', or the end of the input. A name may also be // "[extension]" or "[type/url]". // // The whole struct can also be an expanded Any message, like: // [type/url] < ... struct contents ... > for { tok := p.next() if tok.err != nil { return tok.err } if tok.value == terminator { break } if tok.value == "[" { // Looks like an extension or an Any. // // TODO: Check whether we need to handle // namespace rooted names (e.g. ".something.Foo"). extName, err := p.consumeExtName() if err != nil { return err } if s := strings.LastIndex(extName, "/"); s >= 0 { // If it contains a slash, it's an Any type URL. messageName := extName[s+1:] mt := MessageType(messageName) if mt == nil { return p.errorf("unrecognized message %q in google.protobuf.Any", messageName) } tok = p.next() if tok.err != nil { return tok.err } // consume an optional colon if tok.value == ":" { tok = p.next() if tok.err != nil { return tok.err } } var terminator string switch tok.value { case "<": terminator = ">" case "{": terminator = "}" default: return p.errorf("expected '{' or '<', found %q", tok.value) } v := reflect.New(mt.Elem()) if pe := p.readStruct(v.Elem(), terminator); pe != nil { return pe } b, err := Marshal(v.Interface().(Message)) if err != nil { return p.errorf("failed to marshal message of type %q: %v", messageName, err) } if fieldSet["type_url"] { return p.errorf(anyRepeatedlyUnpacked, "type_url") } if fieldSet["value"] { return p.errorf(anyRepeatedlyUnpacked, "value") } sv.FieldByName("TypeUrl").SetString(extName) sv.FieldByName("Value").SetBytes(b) fieldSet["type_url"] = true fieldSet["value"] = true continue } var desc *ExtensionDesc // This could be faster, but it's functional. // TODO: Do something smarter than a linear scan. for _, d := range RegisteredExtensions(reflect.New(st).Interface().(Message)) { if d.Name == extName { desc = d break } } if desc == nil { return p.errorf("unrecognized extension %q", extName) } props := &Properties{} props.Parse(desc.Tag) typ := reflect.TypeOf(desc.ExtensionType) if err := p.checkForColon(props, typ); err != nil { return err } rep := desc.repeated() // Read the extension structure, and set it in // the value we're constructing. var ext reflect.Value if !rep { ext = reflect.New(typ).Elem() } else { ext = reflect.New(typ.Elem()).Elem() } if err := p.readAny(ext, props); err != nil { if _, ok := err.(*RequiredNotSetError); !ok { return err } reqFieldErr = err } ep := sv.Addr().Interface().(Message) if !rep { SetExtension(ep, desc, ext.Interface()) } else { old, err := GetExtension(ep, desc) var sl reflect.Value if err == nil { sl = reflect.ValueOf(old) // existing slice } else { sl = reflect.MakeSlice(typ, 0, 1) } sl = reflect.Append(sl, ext) SetExtension(ep, desc, sl.Interface()) } if err := p.consumeOptionalSeparator(); err != nil { return err } continue } // This is a normal, non-extension field. name := tok.value var dst reflect.Value fi, props, ok := structFieldByName(sprops, name) if ok { dst = sv.Field(fi) } else if oop, ok := sprops.OneofTypes[name]; ok { // It is a oneof. props = oop.Prop nv := reflect.New(oop.Type.Elem()) dst = nv.Elem().Field(0) field := sv.Field(oop.Field) if !field.IsNil() { return p.errorf("field '%s' would overwrite already parsed oneof '%s'", name, sv.Type().Field(oop.Field).Name) } field.Set(nv) } if !dst.IsValid() { return p.errorf("unknown field name %q in %v", name, st) } if dst.Kind() == reflect.Map { // Consume any colon. if err := p.checkForColon(props, dst.Type()); err != nil { return err } // Construct the map if it doesn't already exist. if dst.IsNil() { dst.Set(reflect.MakeMap(dst.Type())) } key := reflect.New(dst.Type().Key()).Elem() val := reflect.New(dst.Type().Elem()).Elem() // The map entry should be this sequence of tokens: // < key : KEY value : VALUE > // However, implementations may omit key or value, and technically // we should support them in any order. See b/28924776 for a time // this went wrong. tok := p.next() var terminator string switch tok.value { case "<": terminator = ">" case "{": terminator = "}" default: return p.errorf("expected '{' or '<', found %q", tok.value) } for { tok := p.next() if tok.err != nil { return tok.err } if tok.value == terminator { break } switch tok.value { case "key": if err := p.consumeToken(":"); err != nil { return err } if err := p.readAny(key, props.mkeyprop); err != nil { return err } if err := p.consumeOptionalSeparator(); err != nil { return err } case "value": if err := p.checkForColon(props.mvalprop, dst.Type().Elem()); err != nil { return err } if err := p.readAny(val, props.mvalprop); err != nil { return err } if err := p.consumeOptionalSeparator(); err != nil { return err } default: p.back() return p.errorf(`expected "key", "value", or %q, found %q`, terminator, tok.value) } } dst.SetMapIndex(key, val) continue } // Check that it's not already set if it's not a repeated field. if !props.Repeated && fieldSet[name] { return p.errorf("non-repeated field %q was repeated", name) } if err := p.checkForColon(props, dst.Type()); err != nil { return err } // Parse into the field. fieldSet[name] = true if err := p.readAny(dst, props); err != nil { if _, ok := err.(*RequiredNotSetError); !ok { return err } reqFieldErr = err } if props.Required { reqCount-- } if err := p.consumeOptionalSeparator(); err != nil { return err } } if reqCount > 0 { return p.missingRequiredFieldError(sv) } return reqFieldErr } // consumeExtName consumes extension name or expanded Any type URL and the // following ']'. It returns the name or URL consumed. func (p *textParser) consumeExtName() (string, error) { tok := p.next() if tok.err != nil { return "", tok.err } // If extension name or type url is quoted, it's a single token. if len(tok.value) > 2 && isQuote(tok.value[0]) && tok.value[len(tok.value)-1] == tok.value[0] { name, err := unquoteC(tok.value[1:len(tok.value)-1], rune(tok.value[0])) if err != nil { return "", err } return name, p.consumeToken("]") } // Consume everything up to "]" var parts []string for tok.value != "]" { parts = append(parts, tok.value) tok = p.next() if tok.err != nil { return "", p.errorf("unrecognized type_url or extension name: %s", tok.err) } } return strings.Join(parts, ""), nil } // consumeOptionalSeparator consumes an optional semicolon or comma. // It is used in readStruct to provide backward compatibility. func (p *textParser) consumeOptionalSeparator() error { tok := p.next() if tok.err != nil { return tok.err } if tok.value != ";" && tok.value != "," { p.back() } return nil } func (p *textParser) readAny(v reflect.Value, props *Properties) error { tok := p.next() if tok.err != nil { return tok.err } if tok.value == "" { return p.errorf("unexpected EOF") } switch fv := v; fv.Kind() { case reflect.Slice: at := v.Type() if at.Elem().Kind() == reflect.Uint8 { // Special case for []byte if tok.value[0] != '"' && tok.value[0] != '\'' { // Deliberately written out here, as the error after // this switch statement would write "invalid []byte: ...", // which is not as user-friendly. return p.errorf("invalid string: %v", tok.value) } bytes := []byte(tok.unquoted) fv.Set(reflect.ValueOf(bytes)) return nil } // Repeated field. if tok.value == "[" { // Repeated field with list notation, like [1,2,3]. for { fv.Set(reflect.Append(fv, reflect.New(at.Elem()).Elem())) err := p.readAny(fv.Index(fv.Len()-1), props) if err != nil { return err } tok := p.next() if tok.err != nil { return tok.err } if tok.value == "]" { break } if tok.value != "," { return p.errorf("Expected ']' or ',' found %q", tok.value) } } return nil } // One value of the repeated field. p.back() fv.Set(reflect.Append(fv, reflect.New(at.Elem()).Elem())) return p.readAny(fv.Index(fv.Len()-1), props) case reflect.Bool: // true/1/t/True or false/f/0/False. switch tok.value { case "true", "1", "t", "True": fv.SetBool(true) return nil case "false", "0", "f", "False": fv.SetBool(false) return nil } case reflect.Float32, reflect.Float64: v := tok.value // Ignore 'f' for compatibility with output generated by C++, but don't // remove 'f' when the value is "-inf" or "inf". if strings.HasSuffix(v, "f") && tok.value != "-inf" && tok.value != "inf" { v = v[:len(v)-1] } if f, err := strconv.ParseFloat(v, fv.Type().Bits()); err == nil { fv.SetFloat(f) return nil } case reflect.Int32: if x, err := strconv.ParseInt(tok.value, 0, 32); err == nil { fv.SetInt(x) return nil } if len(props.Enum) == 0 { break } m, ok := enumValueMaps[props.Enum] if !ok { break } x, ok := m[tok.value] if !ok { break } fv.SetInt(int64(x)) return nil case reflect.Int64: if x, err := strconv.ParseInt(tok.value, 0, 64); err == nil { fv.SetInt(x) return nil } case reflect.Ptr: // A basic field (indirected through pointer), or a repeated message/group p.back() fv.Set(reflect.New(fv.Type().Elem())) return p.readAny(fv.Elem(), props) case reflect.String: if tok.value[0] == '"' || tok.value[0] == '\'' { fv.SetString(tok.unquoted) return nil } case reflect.Struct: var terminator string switch tok.value { case "{": terminator = "}" case "<": terminator = ">" default: return p.errorf("expected '{' or '<', found %q", tok.value) } // TODO: Handle nested messages which implement encoding.TextUnmarshaler. return p.readStruct(fv, terminator) case reflect.Uint32: if x, err := strconv.ParseUint(tok.value, 0, 32); err == nil { fv.SetUint(x) return nil } case reflect.Uint64: if x, err := strconv.ParseUint(tok.value, 0, 64); err == nil { fv.SetUint(x) return nil } } return p.errorf("invalid %v: %v", v.Type(), tok.value) } // UnmarshalText reads a protocol buffer in Text format. UnmarshalText resets pb // before starting to unmarshal, so any existing data in pb is always removed. // If a required field is not set and no other error occurs, // UnmarshalText returns *RequiredNotSetError. func UnmarshalText(s string, pb Message) error { if um, ok := pb.(encoding.TextUnmarshaler); ok { err := um.UnmarshalText([]byte(s)) return err } pb.Reset() v := reflect.ValueOf(pb) if pe := newTextParser(s).readStruct(v.Elem(), ""); pe != nil { return pe } return nil } ================================================ FILE: vendor/github.com/golang/protobuf/proto/text_parser_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "math" "reflect" "testing" . "github.com/golang/protobuf/proto" proto3pb "github.com/golang/protobuf/proto/proto3_proto" . "github.com/golang/protobuf/proto/testdata" ) type UnmarshalTextTest struct { in string err string // if "", no error expected out *MyMessage } func buildExtStructTest(text string) UnmarshalTextTest { msg := &MyMessage{ Count: Int32(42), } SetExtension(msg, E_Ext_More, &Ext{ Data: String("Hello, world!"), }) return UnmarshalTextTest{in: text, out: msg} } func buildExtDataTest(text string) UnmarshalTextTest { msg := &MyMessage{ Count: Int32(42), } SetExtension(msg, E_Ext_Text, String("Hello, world!")) SetExtension(msg, E_Ext_Number, Int32(1729)) return UnmarshalTextTest{in: text, out: msg} } func buildExtRepStringTest(text string) UnmarshalTextTest { msg := &MyMessage{ Count: Int32(42), } if err := SetExtension(msg, E_Greeting, []string{"bula", "hola"}); err != nil { panic(err) } return UnmarshalTextTest{in: text, out: msg} } var unMarshalTextTests = []UnmarshalTextTest{ // Basic { in: " count:42\n name:\"Dave\" ", out: &MyMessage{ Count: Int32(42), Name: String("Dave"), }, }, // Empty quoted string { in: `count:42 name:""`, out: &MyMessage{ Count: Int32(42), Name: String(""), }, }, // Quoted string concatenation with double quotes { in: `count:42 name: "My name is "` + "\n" + `"elsewhere"`, out: &MyMessage{ Count: Int32(42), Name: String("My name is elsewhere"), }, }, // Quoted string concatenation with single quotes { in: "count:42 name: 'My name is '\n'elsewhere'", out: &MyMessage{ Count: Int32(42), Name: String("My name is elsewhere"), }, }, // Quoted string concatenations with mixed quotes { in: "count:42 name: 'My name is '\n\"elsewhere\"", out: &MyMessage{ Count: Int32(42), Name: String("My name is elsewhere"), }, }, { in: "count:42 name: \"My name is \"\n'elsewhere'", out: &MyMessage{ Count: Int32(42), Name: String("My name is elsewhere"), }, }, // Quoted string with escaped apostrophe { in: `count:42 name: "HOLIDAY - New Year\'s Day"`, out: &MyMessage{ Count: Int32(42), Name: String("HOLIDAY - New Year's Day"), }, }, // Quoted string with single quote { in: `count:42 name: 'Roger "The Ramster" Ramjet'`, out: &MyMessage{ Count: Int32(42), Name: String(`Roger "The Ramster" Ramjet`), }, }, // Quoted string with all the accepted special characters from the C++ test { in: `count:42 name: ` + "\"\\\"A string with \\' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"", out: &MyMessage{ Count: Int32(42), Name: String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces"), }, }, // Quoted string with quoted backslash { in: `count:42 name: "\\'xyz"`, out: &MyMessage{ Count: Int32(42), Name: String(`\'xyz`), }, }, // Quoted string with UTF-8 bytes. { in: "count:42 name: '\303\277\302\201\xAB'", out: &MyMessage{ Count: Int32(42), Name: String("\303\277\302\201\xAB"), }, }, // Bad quoted string { in: `inner: < host: "\0" >` + "\n", err: `line 1.15: invalid quoted string "\0": \0 requires 2 following digits`, }, // Number too large for int64 { in: "count: 1 others { key: 123456789012345678901 }", err: "line 1.23: invalid int64: 123456789012345678901", }, // Number too large for int32 { in: "count: 1234567890123", err: "line 1.7: invalid int32: 1234567890123", }, // Number in hexadecimal { in: "count: 0x2beef", out: &MyMessage{ Count: Int32(0x2beef), }, }, // Number in octal { in: "count: 024601", out: &MyMessage{ Count: Int32(024601), }, }, // Floating point number with "f" suffix { in: "count: 4 others:< weight: 17.0f >", out: &MyMessage{ Count: Int32(4), Others: []*OtherMessage{ { Weight: Float32(17), }, }, }, }, // Floating point positive infinity { in: "count: 4 bigfloat: inf", out: &MyMessage{ Count: Int32(4), Bigfloat: Float64(math.Inf(1)), }, }, // Floating point negative infinity { in: "count: 4 bigfloat: -inf", out: &MyMessage{ Count: Int32(4), Bigfloat: Float64(math.Inf(-1)), }, }, // Number too large for float32 { in: "others:< weight: 12345678901234567890123456789012345678901234567890 >", err: "line 1.17: invalid float32: 12345678901234567890123456789012345678901234567890", }, // Number posing as a quoted string { in: `inner: < host: 12 >` + "\n", err: `line 1.15: invalid string: 12`, }, // Quoted string posing as int32 { in: `count: "12"`, err: `line 1.7: invalid int32: "12"`, }, // Quoted string posing a float32 { in: `others:< weight: "17.4" >`, err: `line 1.17: invalid float32: "17.4"`, }, // Enum { in: `count:42 bikeshed: BLUE`, out: &MyMessage{ Count: Int32(42), Bikeshed: MyMessage_BLUE.Enum(), }, }, // Repeated field { in: `count:42 pet: "horsey" pet:"bunny"`, out: &MyMessage{ Count: Int32(42), Pet: []string{"horsey", "bunny"}, }, }, // Repeated field with list notation { in: `count:42 pet: ["horsey", "bunny"]`, out: &MyMessage{ Count: Int32(42), Pet: []string{"horsey", "bunny"}, }, }, // Repeated message with/without colon and <>/{} { in: `count:42 others:{} others{} others:<> others:{}`, out: &MyMessage{ Count: Int32(42), Others: []*OtherMessage{ {}, {}, {}, {}, }, }, }, // Missing colon for inner message { in: `count:42 inner < host: "cauchy.syd" >`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("cauchy.syd"), }, }, }, // Missing colon for string field { in: `name "Dave"`, err: `line 1.5: expected ':', found "\"Dave\""`, }, // Missing colon for int32 field { in: `count 42`, err: `line 1.6: expected ':', found "42"`, }, // Missing required field { in: `name: "Pawel"`, err: `proto: required field "testdata.MyMessage.count" not set`, out: &MyMessage{ Name: String("Pawel"), }, }, // Missing required field in a required submessage { in: `count: 42 we_must_go_deeper < leo_finally_won_an_oscar <> >`, err: `proto: required field "testdata.InnerMessage.host" not set`, out: &MyMessage{ Count: Int32(42), WeMustGoDeeper: &RequiredInnerMessage{LeoFinallyWonAnOscar: &InnerMessage{}}, }, }, // Repeated non-repeated field { in: `name: "Rob" name: "Russ"`, err: `line 1.12: non-repeated field "name" was repeated`, }, // Group { in: `count: 17 SomeGroup { group_field: 12 }`, out: &MyMessage{ Count: Int32(17), Somegroup: &MyMessage_SomeGroup{ GroupField: Int32(12), }, }, }, // Semicolon between fields { in: `count:3;name:"Calvin"`, out: &MyMessage{ Count: Int32(3), Name: String("Calvin"), }, }, // Comma between fields { in: `count:4,name:"Ezekiel"`, out: &MyMessage{ Count: Int32(4), Name: String("Ezekiel"), }, }, // Boolean false { in: `count:42 inner { host: "example.com" connected: false }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(false), }, }, }, // Boolean true { in: `count:42 inner { host: "example.com" connected: true }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(true), }, }, }, // Boolean 0 { in: `count:42 inner { host: "example.com" connected: 0 }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(false), }, }, }, // Boolean 1 { in: `count:42 inner { host: "example.com" connected: 1 }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(true), }, }, }, // Boolean f { in: `count:42 inner { host: "example.com" connected: f }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(false), }, }, }, // Boolean t { in: `count:42 inner { host: "example.com" connected: t }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(true), }, }, }, // Boolean False { in: `count:42 inner { host: "example.com" connected: False }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(false), }, }, }, // Boolean True { in: `count:42 inner { host: "example.com" connected: True }`, out: &MyMessage{ Count: Int32(42), Inner: &InnerMessage{ Host: String("example.com"), Connected: Bool(true), }, }, }, // Extension buildExtStructTest(`count: 42 [testdata.Ext.more]:`), buildExtStructTest(`count: 42 [testdata.Ext.more] {data:"Hello, world!"}`), buildExtDataTest(`count: 42 [testdata.Ext.text]:"Hello, world!" [testdata.Ext.number]:1729`), buildExtRepStringTest(`count: 42 [testdata.greeting]:"bula" [testdata.greeting]:"hola"`), // Big all-in-one { in: "count:42 # Meaning\n" + `name:"Dave" ` + `quote:"\"I didn't want to go.\"" ` + `pet:"bunny" ` + `pet:"kitty" ` + `pet:"horsey" ` + `inner:<` + ` host:"footrest.syd" ` + ` port:7001 ` + ` connected:true ` + `> ` + `others:<` + ` key:3735928559 ` + ` value:"\x01A\a\f" ` + `> ` + `others:<` + " weight:58.9 # Atomic weight of Co\n" + ` inner:<` + ` host:"lesha.mtv" ` + ` port:8002 ` + ` >` + `>`, out: &MyMessage{ Count: Int32(42), Name: String("Dave"), Quote: String(`"I didn't want to go."`), Pet: []string{"bunny", "kitty", "horsey"}, Inner: &InnerMessage{ Host: String("footrest.syd"), Port: Int32(7001), Connected: Bool(true), }, Others: []*OtherMessage{ { Key: Int64(3735928559), Value: []byte{0x1, 'A', '\a', '\f'}, }, { Weight: Float32(58.9), Inner: &InnerMessage{ Host: String("lesha.mtv"), Port: Int32(8002), }, }, }, }, }, } func TestUnmarshalText(t *testing.T) { for i, test := range unMarshalTextTests { pb := new(MyMessage) err := UnmarshalText(test.in, pb) if test.err == "" { // We don't expect failure. if err != nil { t.Errorf("Test %d: Unexpected error: %v", i, err) } else if !reflect.DeepEqual(pb, test.out) { t.Errorf("Test %d: Incorrect populated \nHave: %v\nWant: %v", i, pb, test.out) } } else { // We do expect failure. if err == nil { t.Errorf("Test %d: Didn't get expected error: %v", i, test.err) } else if err.Error() != test.err { t.Errorf("Test %d: Incorrect error.\nHave: %v\nWant: %v", i, err.Error(), test.err) } else if _, ok := err.(*RequiredNotSetError); ok && test.out != nil && !reflect.DeepEqual(pb, test.out) { t.Errorf("Test %d: Incorrect populated \nHave: %v\nWant: %v", i, pb, test.out) } } } } func TestUnmarshalTextCustomMessage(t *testing.T) { msg := &textMessage{} if err := UnmarshalText("custom", msg); err != nil { t.Errorf("Unexpected error from custom unmarshal: %v", err) } if UnmarshalText("not custom", msg) == nil { t.Errorf("Didn't get expected error from custom unmarshal") } } // Regression test; this caused a panic. func TestRepeatedEnum(t *testing.T) { pb := new(RepeatedEnum) if err := UnmarshalText("color: RED", pb); err != nil { t.Fatal(err) } exp := &RepeatedEnum{ Color: []RepeatedEnum_Color{RepeatedEnum_RED}, } if !Equal(pb, exp) { t.Errorf("Incorrect populated \nHave: %v\nWant: %v", pb, exp) } } func TestProto3TextParsing(t *testing.T) { m := new(proto3pb.Message) const in = `name: "Wallace" true_scotsman: true` want := &proto3pb.Message{ Name: "Wallace", TrueScotsman: true, } if err := UnmarshalText(in, m); err != nil { t.Fatal(err) } if !Equal(m, want) { t.Errorf("\n got %v\nwant %v", m, want) } } func TestMapParsing(t *testing.T) { m := new(MessageWithMap) const in = `name_mapping: name_mapping:` + `msg_mapping:,>` + // separating commas are okay `msg_mapping>` + // no colon after "value" `msg_mapping:>` + // omitted key `msg_mapping:` + // omitted value `byte_mapping:` + `byte_mapping:<>` // omitted key and value want := &MessageWithMap{ NameMapping: map[int32]string{ 1: "Beatles", 1234: "Feist", }, MsgMapping: map[int64]*FloatingPoint{ -4: {F: Float64(2.0)}, -2: {F: Float64(4.0)}, 0: {F: Float64(5.0)}, 1: nil, }, ByteMapping: map[bool][]byte{ false: nil, true: []byte("so be it"), }, } if err := UnmarshalText(in, m); err != nil { t.Fatal(err) } if !Equal(m, want) { t.Errorf("\n got %v\nwant %v", m, want) } } func TestOneofParsing(t *testing.T) { const in = `name:"Shrek"` m := new(Communique) want := &Communique{Union: &Communique_Name{"Shrek"}} if err := UnmarshalText(in, m); err != nil { t.Fatal(err) } if !Equal(m, want) { t.Errorf("\n got %v\nwant %v", m, want) } const inOverwrite = `name:"Shrek" number:42` m = new(Communique) testErr := "line 1.13: field 'number' would overwrite already parsed oneof 'Union'" if err := UnmarshalText(inOverwrite, m); err == nil { t.Errorf("TestOneofParsing: Didn't get expected error: %v", testErr) } else if err.Error() != testErr { t.Errorf("TestOneofParsing: Incorrect error.\nHave: %v\nWant: %v", err.Error(), testErr) } } var benchInput string func init() { benchInput = "count: 4\n" for i := 0; i < 1000; i++ { benchInput += "pet: \"fido\"\n" } // Check it is valid input. pb := new(MyMessage) err := UnmarshalText(benchInput, pb) if err != nil { panic("Bad benchmark input: " + err.Error()) } } func BenchmarkUnmarshalText(b *testing.B) { pb := new(MyMessage) for i := 0; i < b.N; i++ { UnmarshalText(benchInput, pb) } b.SetBytes(int64(len(benchInput))) } ================================================ FILE: vendor/github.com/golang/protobuf/proto/text_test.go ================================================ // Go support for Protocol Buffers - Google's data interchange format // // Copyright 2010 The Go Authors. All rights reserved. // https://github.com/golang/protobuf // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package proto_test import ( "bytes" "errors" "io/ioutil" "math" "strings" "testing" "github.com/golang/protobuf/proto" proto3pb "github.com/golang/protobuf/proto/proto3_proto" pb "github.com/golang/protobuf/proto/testdata" ) // textMessage implements the methods that allow it to marshal and unmarshal // itself as text. type textMessage struct { } func (*textMessage) MarshalText() ([]byte, error) { return []byte("custom"), nil } func (*textMessage) UnmarshalText(bytes []byte) error { if string(bytes) != "custom" { return errors.New("expected 'custom'") } return nil } func (*textMessage) Reset() {} func (*textMessage) String() string { return "" } func (*textMessage) ProtoMessage() {} func newTestMessage() *pb.MyMessage { msg := &pb.MyMessage{ Count: proto.Int32(42), Name: proto.String("Dave"), Quote: proto.String(`"I didn't want to go."`), Pet: []string{"bunny", "kitty", "horsey"}, Inner: &pb.InnerMessage{ Host: proto.String("footrest.syd"), Port: proto.Int32(7001), Connected: proto.Bool(true), }, Others: []*pb.OtherMessage{ { Key: proto.Int64(0xdeadbeef), Value: []byte{1, 65, 7, 12}, }, { Weight: proto.Float32(6.022), Inner: &pb.InnerMessage{ Host: proto.String("lesha.mtv"), Port: proto.Int32(8002), }, }, }, Bikeshed: pb.MyMessage_BLUE.Enum(), Somegroup: &pb.MyMessage_SomeGroup{ GroupField: proto.Int32(8), }, // One normally wouldn't do this. // This is an undeclared tag 13, as a varint (wire type 0) with value 4. XXX_unrecognized: []byte{13<<3 | 0, 4}, } ext := &pb.Ext{ Data: proto.String("Big gobs for big rats"), } if err := proto.SetExtension(msg, pb.E_Ext_More, ext); err != nil { panic(err) } greetings := []string{"adg", "easy", "cow"} if err := proto.SetExtension(msg, pb.E_Greeting, greetings); err != nil { panic(err) } // Add an unknown extension. We marshal a pb.Ext, and fake the ID. b, err := proto.Marshal(&pb.Ext{Data: proto.String("3G skiing")}) if err != nil { panic(err) } b = append(proto.EncodeVarint(201<<3|proto.WireBytes), b...) proto.SetRawExtension(msg, 201, b) // Extensions can be plain fields, too, so let's test that. b = append(proto.EncodeVarint(202<<3|proto.WireVarint), 19) proto.SetRawExtension(msg, 202, b) return msg } const text = `count: 42 name: "Dave" quote: "\"I didn't want to go.\"" pet: "bunny" pet: "kitty" pet: "horsey" inner: < host: "footrest.syd" port: 7001 connected: true > others: < key: 3735928559 value: "\001A\007\014" > others: < weight: 6.022 inner: < host: "lesha.mtv" port: 8002 > > bikeshed: BLUE SomeGroup { group_field: 8 } /* 2 unknown bytes */ 13: 4 [testdata.Ext.more]: < data: "Big gobs for big rats" > [testdata.greeting]: "adg" [testdata.greeting]: "easy" [testdata.greeting]: "cow" /* 13 unknown bytes */ 201: "\t3G skiing" /* 3 unknown bytes */ 202: 19 ` func TestMarshalText(t *testing.T) { buf := new(bytes.Buffer) if err := proto.MarshalText(buf, newTestMessage()); err != nil { t.Fatalf("proto.MarshalText: %v", err) } s := buf.String() if s != text { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, text) } } func TestMarshalTextCustomMessage(t *testing.T) { buf := new(bytes.Buffer) if err := proto.MarshalText(buf, &textMessage{}); err != nil { t.Fatalf("proto.MarshalText: %v", err) } s := buf.String() if s != "custom" { t.Errorf("Got %q, expected %q", s, "custom") } } func TestMarshalTextNil(t *testing.T) { want := "" tests := []proto.Message{nil, (*pb.MyMessage)(nil)} for i, test := range tests { buf := new(bytes.Buffer) if err := proto.MarshalText(buf, test); err != nil { t.Fatal(err) } if got := buf.String(); got != want { t.Errorf("%d: got %q want %q", i, got, want) } } } func TestMarshalTextUnknownEnum(t *testing.T) { // The Color enum only specifies values 0-2. m := &pb.MyMessage{Bikeshed: pb.MyMessage_Color(3).Enum()} got := m.String() const want = `bikeshed:3 ` if got != want { t.Errorf("\n got %q\nwant %q", got, want) } } func TestTextOneof(t *testing.T) { tests := []struct { m proto.Message want string }{ // zero message {&pb.Communique{}, ``}, // scalar field {&pb.Communique{Union: &pb.Communique_Number{4}}, `number:4`}, // message field {&pb.Communique{Union: &pb.Communique_Msg{ &pb.Strings{StringField: proto.String("why hello!")}, }}, `msg:`}, // bad oneof (should not panic) {&pb.Communique{Union: &pb.Communique_Msg{nil}}, `msg:/* nil */`}, } for _, test := range tests { got := strings.TrimSpace(test.m.String()) if got != test.want { t.Errorf("\n got %s\nwant %s", got, test.want) } } } func BenchmarkMarshalTextBuffered(b *testing.B) { buf := new(bytes.Buffer) m := newTestMessage() for i := 0; i < b.N; i++ { buf.Reset() proto.MarshalText(buf, m) } } func BenchmarkMarshalTextUnbuffered(b *testing.B) { w := ioutil.Discard m := newTestMessage() for i := 0; i < b.N; i++ { proto.MarshalText(w, m) } } func compact(src string) string { // s/[ \n]+/ /g; s/ $//; dst := make([]byte, len(src)) space, comment := false, false j := 0 for i := 0; i < len(src); i++ { if strings.HasPrefix(src[i:], "/*") { comment = true i++ continue } if comment && strings.HasPrefix(src[i:], "*/") { comment = false i++ continue } if comment { continue } c := src[i] if c == ' ' || c == '\n' { space = true continue } if j > 0 && (dst[j-1] == ':' || dst[j-1] == '<' || dst[j-1] == '{') { space = false } if c == '{' { space = false } if space { dst[j] = ' ' j++ space = false } dst[j] = c j++ } if space { dst[j] = ' ' j++ } return string(dst[0:j]) } var compactText = compact(text) func TestCompactText(t *testing.T) { s := proto.CompactTextString(newTestMessage()) if s != compactText { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v\n===\n", s, compactText) } } func TestStringEscaping(t *testing.T) { testCases := []struct { in *pb.Strings out string }{ { // Test data from C++ test (TextFormatTest.StringEscape). // Single divergence: we don't escape apostrophes. &pb.Strings{StringField: proto.String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces")}, "string_field: \"\\\"A string with ' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"\n", }, { // Test data from the same C++ test. &pb.Strings{StringField: proto.String("\350\260\267\346\255\214")}, "string_field: \"\\350\\260\\267\\346\\255\\214\"\n", }, { // Some UTF-8. &pb.Strings{StringField: proto.String("\x00\x01\xff\x81")}, `string_field: "\000\001\377\201"` + "\n", }, } for i, tc := range testCases { var buf bytes.Buffer if err := proto.MarshalText(&buf, tc.in); err != nil { t.Errorf("proto.MarsalText: %v", err) continue } s := buf.String() if s != tc.out { t.Errorf("#%d: Got:\n%s\nExpected:\n%s\n", i, s, tc.out) continue } // Check round-trip. pb := new(pb.Strings) if err := proto.UnmarshalText(s, pb); err != nil { t.Errorf("#%d: UnmarshalText: %v", i, err) continue } if !proto.Equal(pb, tc.in) { t.Errorf("#%d: Round-trip failed:\nstart: %v\n end: %v", i, tc.in, pb) } } } // A limitedWriter accepts some output before it fails. // This is a proxy for something like a nearly-full or imminently-failing disk, // or a network connection that is about to die. type limitedWriter struct { b bytes.Buffer limit int } var outOfSpace = errors.New("proto: insufficient space") func (w *limitedWriter) Write(p []byte) (n int, err error) { var avail = w.limit - w.b.Len() if avail <= 0 { return 0, outOfSpace } if len(p) <= avail { return w.b.Write(p) } n, _ = w.b.Write(p[:avail]) return n, outOfSpace } func TestMarshalTextFailing(t *testing.T) { // Try lots of different sizes to exercise more error code-paths. for lim := 0; lim < len(text); lim++ { buf := new(limitedWriter) buf.limit = lim err := proto.MarshalText(buf, newTestMessage()) // We expect a certain error, but also some partial results in the buffer. if err != outOfSpace { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", err, outOfSpace) } s := buf.b.String() x := text[:buf.limit] if s != x { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, x) } } } func TestFloats(t *testing.T) { tests := []struct { f float64 want string }{ {0, "0"}, {4.7, "4.7"}, {math.Inf(1), "inf"}, {math.Inf(-1), "-inf"}, {math.NaN(), "nan"}, } for _, test := range tests { msg := &pb.FloatingPoint{F: &test.f} got := strings.TrimSpace(msg.String()) want := `f:` + test.want if got != want { t.Errorf("f=%f: got %q, want %q", test.f, got, want) } } } func TestRepeatedNilText(t *testing.T) { m := &pb.MessageList{ Message: []*pb.MessageList_Message{ nil, &pb.MessageList_Message{ Name: proto.String("Horse"), }, nil, }, } want := `Message Message { name: "Horse" } Message ` if s := proto.MarshalTextString(m); s != want { t.Errorf(" got: %s\nwant: %s", s, want) } } func TestProto3Text(t *testing.T) { tests := []struct { m proto.Message want string }{ // zero message {&proto3pb.Message{}, ``}, // zero message except for an empty byte slice {&proto3pb.Message{Data: []byte{}}, ``}, // trivial case {&proto3pb.Message{Name: "Rob", HeightInCm: 175}, `name:"Rob" height_in_cm:175`}, // empty map {&pb.MessageWithMap{}, ``}, // non-empty map; map format is the same as a repeated struct, // and they are sorted by key (numerically for numeric keys). { &pb.MessageWithMap{NameMapping: map[int32]string{ -1: "Negatory", 7: "Lucky", 1234: "Feist", 6345789: "Otis", }}, `name_mapping: ` + `name_mapping: ` + `name_mapping: ` + `name_mapping:`, }, // map with nil value; not well-defined, but we shouldn't crash { &pb.MessageWithMap{MsgMapping: map[int64]*pb.FloatingPoint{7: nil}}, `msg_mapping:`, }, } for _, test := range tests { got := strings.TrimSpace(test.m.String()) if got != test.want { t.Errorf("\n got %s\nwant %s", got, test.want) } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/.dockerignore ================================================ .git build ================================================ FILE: vendor/github.com/influxdata/kapacitor/.gitattributes ================================================ CHANGELOG.md merge=union ================================================ FILE: vendor/github.com/influxdata/kapacitor/.gitignore ================================================ .*.swp dist/* /build/* /*.conf kapacitor_linux* kapacitord_linux* /*.tick *~ *# kapacitor*.rpm kapacitor*.deb kapacitor*.tar kapacitor*.zip *.pyc *.test /test-logs *.prof # Ignore any built binaries /kapacitor /kapacitord /tickfmt /tickdoc ================================================ FILE: vendor/github.com/influxdata/kapacitor/BLOB_STORE_DESIGN.md ================================================ # Blob Store The blob store is a mechanism to store arbitrary data in Kapacitor. The data stored is immutable and opaque to Kapacitor. Data is stored as blobs where each blob has a unique ID. A tagging system is used to refer various blobs within the store. A blob may be tagged with a given name. A blob may be retrieved by its ID or a tag name. When retrieving a blob via a tag name, the most recently associated blob is returned for that tag. Tags may be updated, meaning they can be modified to point at a different blob. The history of a tag to blob associations are preserved. There are no specific limits on the size of a blob, and blobs can be streamed in and out of the store. ## Uses The following details the various uses of the Kapacitor blob store. ### Snapshots Kapacitor will periodically snapshot the state of a running task. (Currently only implemented for UDFs). When a task is started its previous snapshot or a named snapshot is restored. Kapacitor tasks construct a [DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph) of the data pipeline. Each step in this DAG is called a node. Snapshots are associated with a single node within a single task. All nodes are assigned IDs based on the DAG structure. When the DAG changes the previous snapshots are considered invalid an are no longer used to restore task state. ### UDFs UDFs can explicitly save and request blobs from the store via the protobuf socket connection with Kapacitor. A common use case is to load and store trained model data. However you use the blob store within your UDF is up to you. ## Design The blob store will use content addressable IDs(i.e. shasum of the content) and be exposed via the HTTP API of Kapacitor. Blobs can be created, named and deleted. Creating a blob will accept only the content of the blob data and return the ID of the blob. Naming a blob associates a specified name to the content of the blob. A naming history is recorded, allowing the users to determine the "version" history for a given name. Deleting a blob removes it from the store. ================================================ FILE: vendor/github.com/influxdata/kapacitor/CHANGELOG.md ================================================ # Changelog ## Unreleased ### Features - [#1413](https://github.com/influxdata/kapacitor/issues/1413): Add subscriptions modes to InfluxDB subscriptions. - [#1436](https://github.com/influxdata/kapacitor/issues/1436): Add linear fill support for QueryNode. - [#1345](https://github.com/influxdata/kapacitor/issues/1345): Add MQTT Alert Handler - [#1390](https://github.com/influxdata/kapacitor/issues/1390): Add built in functions to convert timestamps to integers - [#1425](https://github.com/influxdata/kapacitor/pull/1425): BREAKING: Change over internal API to use message passing semantics. The breaking change is that the Combine and Flatten nodes previously, but erroneously, operated across batch boundaries; this has been fixed. - [#1497](https://github.com/influxdata/kapacitor/pull/1497): Add support for Docker Swarm autoscaling services. - [#1485](https://github.com/influxdata/kapacitor/issues/1485): Add bools field types to UDFs. ### Bugfixes - [#1400](https://github.com/influxdata/kapacitor/issues/1400): Allow for `.yml` file extensions in `define-topic-handler` - [#1402](https://github.com/influxdata/kapacitor/pull/1402): Fix http server error logging. - [#1500](https://github.com/influxdata/kapacitor/pull/1500): Fix bugs with stopping running UDF agent. - [#1470](https://github.com/influxdata/kapacitor/pull/1470): Fix error messages for missing fields which are arguments to functions are not clear - [#1516](https://github.com/influxdata/kapacitor/pull/1516): Fix bad PagerDuty test the required server info. ## v1.3.3 [2017-08-11] ### Bugfixes - [#1520](https://github.com/influxdata/kapacitor/pull/1520): Expose pprof without authentication if enabled ## v1.3.2 [2017-08-08] ### Bugfixes - [#1512](https://github.com/influxdata/kapacitor/pull/1512): Use details field from alert node in PagerDuty. ## v1.3.1 [2017-06-02] ### Bugfixes - [#1415](https://github.com/influxdata/kapacitor/pull/1415): Proxy from environment for HTTP request to slack - [#1414](https://github.com/influxdata/kapacitor/pull/1414): Fix derivative node preserving fields from previous point in stream tasks. ## v1.3.0 [2017-05-22] ### Release Notes The v1.3.0 release has two major features. 1. Addition of scraping and discovering for Prometheus style data collection. 2. Updates to the Alert Topic system Here is a quick example of how to configure Kapacitor to scrape discovered targets. First configure a discoverer, here we use the file-discovery discoverer. Next configure a scraper to use that discoverer. >NOTE: The scraping and discovering features are released under technical preview, meaning that the configuration or API around the feature may change in a future release. ``` # Configure file discoverer [[file-discovery]] enabled = true id = "discover_files" refresh-interval = "10s" ##### This will look for prometheus json files ##### File format is here https://prometheus.io/docs/operating/configuration/#%3Cfile_sd_config%3E files = ["/tmp/prom/*.json"] # Configure scraper [[scraper]] enabled = true name = "node_exporter" discoverer-id = "discover_files" discoverer-service = "file-discovery" db = "prometheus" rp = "autogen" type = "prometheus" scheme = "http" metrics-path = "/metrics" scrape-interval = "2s" scrape-timeout = "10s" ``` Add the above snippet to your kapacitor.conf file. Create the below snippet as the file `/tmp/prom/localhost.json`: ``` [{ "targets": ["localhost:9100"] }] ``` Start the Prometheus node_exporter locally. Now startup Kapacitor and it will discover the `localhost:9100` node_exporter target and begin scrapping it for metrics. For more details on the scraping and discovery systems see the full documentation [here](https://docs.influxdata.com/kapacitor/v1.3/scraping). The second major feature with this release, are changes to the alert topic system. The previous release introduce this new system as a technical preview, with this release the alerting service has been simplified. Alert handlers now only ever have a single action and belong to a single topic. The handler definition has been simplified as a result. Here are some example alert handlers using the new structure: ```yaml id: my_handler kind: pagerDuty options: serviceKey: XXX ``` ```yaml id: aggregate_by_1m kind: aggregate options: interval: 1m topic: aggregated ``` ```yaml id: publish_to_system kind: publish options: topics: [ system ] ``` To define a handler now you must specify which topic the handler belongs to. For example to define the above aggregate handler on the system topic use this command: ```sh kapacitor define-handler system aggregate_by_1m.yaml ``` For more details on the alerting system see the full documentation [here](https://docs.influxdata.com/kapacitor/v1.3/alerts). # Bugfixes - [#1396](https://github.com/influxdata/kapacitor/pull/1396): Fix broken ENV var config overrides for the kubernetes section. - [#1397](https://github.com/influxdata/kapacitor/pull/1397): Update default configuration file to include sections for each discoverer service. ## v1.3.0-rc4 [2017-05-19] # Bugfixes - [#1379](https://github.com/influxdata/kapacitor/issues/1379): Copy batch points slice before modification, fixes potential panics and data corruption. - [#1394](https://github.com/influxdata/kapacitor/pull/1394): Use the Prometheus metric name as the measurement name by default for scrape data. - [#1392](https://github.com/influxdata/kapacitor/pull/1392): Fix possible deadlock for scraper configuration updating. ## v1.3.0-rc3 [2017-05-18] ### Bugfixes - [#1369](https://github.com/influxdata/kapacitor/issues/1369): Fix panic with concurrent writes to same points in state tracking nodes. - [#1387](https://github.com/influxdata/kapacitor/pull/1387): static-discovery configuration simplified - [#1378](https://github.com/influxdata/kapacitor/issues/1378): Fix panic in InfluxQL node with missing field. ## v1.3.0-rc2 [2017-05-11] ### Bugfixes - [#1370](https://github.com/influxdata/kapacitor/issues/1370): Fix missing working_cardinality stats on stateDuration and stateCount nodes. ## v1.3.0-rc1 [2017-05-08] ### Features - [#1299](https://github.com/influxdata/kapacitor/pull/1299): Allowing sensu handler to be specified - [#1284](https://github.com/influxdata/kapacitor/pull/1284): Add type signatures to Kapacitor functions. - [#1203](https://github.com/influxdata/kapacitor/issues/1203): Add `isPresent` operator for verifying whether a value is present (part of [#1284](https://github.com/influxdata/kapacitor/pull/1284)). - [#1354](https://github.com/influxdata/kapacitor/pull/1354): Add Kubernetes scraping support. - [#1359](https://github.com/influxdata/kapacitor/pull/1359): Add groupBy exclude and Add dropOriginalFieldName to flatten. - [#1360](https://github.com/influxdata/kapacitor/pull/1360): Add KapacitorLoopback node to be able to send data from a task back into Kapacitor. ### Bugfixes - [#1329](https://github.com/influxdata/kapacitor/issues/1329): BREAKING: A bug was fixed around missing fields in the derivative node. The behavior of the node changes slightly in order to provide a consistent fix to the bug. The breaking change is that now, the time of the points returned are from the right hand or current point time, instead of the left hand or previous point time. - [#1353](https://github.com/influxdata/kapacitor/issues/1353): Fix panic in scraping TargetManager. - [#1238](https://github.com/influxdata/kapacitor/pull/1238): Use ProxyFromEnvironment for all outgoing HTTP traffic. ## v1.3.0-beta2 [2017-05-01] ### Features - [#117](https://github.com/influxdata/kapacitor/issues/117): Add headers to alert POST requests. ### Bugfixes - [#1294](https://github.com/influxdata/kapacitor/issues/1294): Fix bug where batch queries would be missing all fields after the first nil field. - [#1343](https://github.com/influxdata/kapacitor/issues/1343): BREAKING: The UDF agent Go API has changed, the changes now make it so that the agent package is self contained. ## v1.3.0-beta1 [2017-04-29] ### Features - [#1322](https://github.com/influxdata/kapacitor/pull/1322): TLS configuration in Slack service for Mattermost compatibility - [#1330](https://github.com/influxdata/kapacitor/issues/1330): Generic HTTP Post node - [#1159](https://github.com/influxdata/kapacitor/pulls/1159): Go version 1.7.4 -> 1.7.5 - [#1175](https://github.com/influxdata/kapacitor/pull/1175): BREAKING: Add generic error counters to every node type. Renamed `query_errors` to `errors` in batch node. Renamed `eval_errors` to `errors` in eval node. - [#922](https://github.com/influxdata/kapacitor/issues/922): Expose server specific information in alert templates. - [#1162](https://github.com/influxdata/kapacitor/pulls/1162): Add Pushover integration. - [#1221](https://github.com/influxdata/kapacitor/pull/1221): Add `working_cardinality` stat to each node type that tracks the number of groups per node. - [#1211](https://github.com/influxdata/kapacitor/issues/1211): Add StateDuration node. - [#1209](https://github.com/influxdata/kapacitor/issues/1209): BREAKING: Refactor the Alerting service. The change is completely breaking for the technical preview alerting service, a.k.a. the new alert topic handler features. The change boils down to simplifying how you define and interact with topics. Alert handlers now only ever have a single action and belong to a single topic. An automatic migration from old to new handler definitions will be performed during startup. See the updated API docs. - [#1286](https://github.com/influxdata/kapacitor/issues/1286): Default HipChat URL should be blank - [#507](https://github.com/influxdata/kapacitor/issues/507): Add API endpoint for performing Kapacitor database backups. - [#1132](https://github.com/influxdata/kapacitor/issues/1132): Adding source for sensu alert as parameter - [#1346](https://github.com/influxdata/kapacitor/pull/1346): Add discovery and scraping services. ### Bugfixes - [#1133](https://github.com/influxdata/kapacitor/issues/1133): Fix case-sensitivity for Telegram `parseMode` value. - [#1147](https://github.com/influxdata/kapacitor/issues/1147): Fix pprof debug endpoint - [#1164](https://github.com/influxdata/kapacitor/pull/1164): Fix hang in config API to update a config section. Now if the service update process takes too long the request will timeout and return an error. Previously the request would block forever. - [#1165](https://github.com/influxdata/kapacitor/issues/1165): Make the alerta auth token prefix configurable and default it to Bearer. - [#1184](https://github.com/influxdata/kapacitor/pull/1184): Fix logrotate file to correctly rotate error log. - [#1200](https://github.com/influxdata/kapacitor/pull/1200): Fix bug with alert duration being incorrect after restoring alert state. - [#1199](https://github.com/influxdata/kapacitor/pull/1199): BREAKING: Fix inconsistency with JSON data from alerts. The alert handlers Alerta, Log, OpsGenie, PagerDuty, Post and VictorOps allow extra opaque data to be attached to alert notifications. That opaque data was inconsistent and this change fixes that. Depending on how that data was consumed this could result in a breaking change, since the original behavior was inconsistent we decided it would be best to fix the issue now and make it consistent for all future builds. Specifically in the JSON result data the old key `Series` is always `series`, and the old key `Err` is now always `error` instead of for only some of the outputs. - [#1181](https://github.com/influxdata/kapacitor/pull/1181): Fix bug parsing dbrp values with quotes. - [#1228](https://github.com/influxdata/kapacitor/pull/1228): Fix panic on loading replay files without a file extension. - [#1192](https://github.com/influxdata/kapacitor/issues/1192): Fix bug in Default Node not updating batch tags and groupID. Also empty string on a tag value is now a sufficient condition for the default conditions to be applied. See [#1233](https://github.com/influxdata/kapacitor/pull/1233) for more information. - [#1068](https://github.com/influxdata/kapacitor/issues/1068): Fix dot view syntax to use xlabels and not create invalid quotes. - [#1295](https://github.com/influxdata/kapacitor/issues/1295): Fix curruption of recordings list after deleting all recordings. - [#1237](https://github.com/influxdata/kapacitor/issues/1237): Fix missing "vars" key when listing tasks. - [#1271](https://github.com/influxdata/kapacitor/issues/1271): Fix bug where aggregates would not be able to change type. - [#1261](https://github.com/influxdata/kapacitor/issues/1261): Fix panic when the process cannot stat the data dir. ## v1.2.1 [2017-04-13] ### Bugfixes - [#1323](https://github.com/influxdata/kapacitor/pull/1323): Fix issue where credentials to InfluxDB could not be updated dynamically. ## v1.2.0 [2017-01-23] ### Release Notes A new system for working with alerts has been introduced. This alerting system allows you to configure topics for alert events and then configure handlers for various topics. This way alert generation is decoupled from alert handling. Existing TICKscripts will continue to work without modification. To use this new alerting system remove any explicit alert handlers from your TICKscript and specify a topic. Then configure the handlers for the topic. ``` stream |from() .measurement('cpu') .groupBy('host') |alert() // Specify the topic for the alert .topic('cpu') .info(lambda: "value" > 60) .warn(lambda: "value" > 70) .crit(lambda: "value" > 80) // No handlers are configured in the script, they are instead defined on the topic via the API. ``` The API exposes endpoints to query the state of each alert and endpoints for configuring alert handlers. See the [API docs](https://docs.influxdata.com/kapacitor/latest/api/api/) for more details. The kapacitor CLI has been updated with commands for defining alert handlers. This release introduces a new feature where you can window based off the number of points instead of their time. For example: ``` stream |from() .measurement('my-measurement') // Emit window for every 10 points with 100 points per window. |window() .periodCount(100) .everyCount(10) |mean('value') |alert() .crit(lambda: "mean" > 100) .slack() .channel('#alerts') ``` With this change alert nodes will have an anonymous topic created for them. This topic is managed like all other topics preserving state etc. across restarts. As a result existing alert nodes will now remember the state of alerts after restarts and disiabling/enabling a task. >NOTE: The new alerting features are being released under technical preview. This means breaking changes may be made in later releases until the feature is considered complete. See the [API docs on technical preview](https://docs.influxdata.com/kapacitor/v1.2/api/api/#technical-preview) for specifics of how this effects the API. ### Features - [#1110](https://github.com/influxdata/kapacitor/pull/1110): Add new query property for aligning group by intervals to start times. - [#1095](https://github.com/influxdata/kapacitor/pull/1095): Add new alert API, with support for configuring handlers and topics. - [#1052](https://github.com/influxdata/kapacitor/issues/1052): Move alerta api token to header and add option to skip TLS verification. - [#929](https://github.com/influxdata/kapacitor/pull/929): Add SNMP trap service for alerting. - [#913](https://github.com/influxdata/kapacitor/issues/913): Add fillPeriod option to Window node, so that the first emit waits till the period has elapsed before emitting. - [#898](https://github.com/influxdata/kapacitor/issues/898): Now when the Window node every value is zero, the window will be emitted immediately for each new point. - [#744](https://github.com/influxdata/kapacitor/issues/744): Preserve alert state across restarts and disable/enable actions. - [#327](https://github.com/influxdata/kapacitor/issues/327): You can now window based on count in addition to time. - [#251](https://github.com/influxdata/kapacitor/issues/251): Enable markdown in slack attachments. ### Bugfixes - [#1100](https://github.com/influxdata/kapacitor/issues/1100): Fix issue with the Union node buffering more points than necessary. - [#1087](https://github.com/influxdata/kapacitor/issues/1087): Fix panic during close of failed startup when connecting to InfluxDB. - [#1045](https://github.com/influxdata/kapacitor/issues/1045): Fix panic during replays. - [#1043](https://github.com/influxdata/kapacitor/issues/1043): logrotate.d ignores kapacitor configuration due to bad file mode. - [#872](https://github.com/influxdata/kapacitor/issues/872): Fix panic during failed aggregate results. ## v1.1.1 [2016-12-02] ### Release Notes No changes to Kapacitor, only upgrading to go 1.7.4 for security patches. ## v1.1.0 [2016-10-07] ### Release Notes New K8sAutoscale node that allows you to auotmatically scale Kubernetes deployments driven by any metrics Kapacitor consumes. For example, to scale a deployment `myapp` based off requests per second: ``` // The target requests per second per host var target = 100.0 stream |from() .measurement('requests') .where(lambda: "deployment" == 'myapp') // Compute the moving average of the last 5 minutes |movingAverage('requests', 5*60) .as('mean_requests_per_second') |k8sAutoscale() .resourceName('app') .kind('deployments') .min(4) .max(100) // Compute the desired number of replicas based on target. .replicas(lambda: int(ceil("mean_requests_per_second" / target))) ``` New API endpoints have been added to be able to configure InfluxDB clusters and alert handlers dynamically without needing to restart the Kapacitor daemon. Along with the ability to dynamically configure a service, API endpoints have been added to test the configurable services. See the [API docs](https://docs.influxdata.com/kapacitor/latest/api/api/) for more details. >NOTE: The `connect_errors` stat from the query node was removed since the client changed, all errors are now counted in the `query_errors` stat. ### Features - [#931](https://github.com/influxdata/kapacitor/issues/931): Add a Kubernetes autoscaler node. You can now autoscale your Kubernetes deployments via Kapacitor. - [#928](https://github.com/influxdata/kapacitor/issues/928): Add new API endpoint for dynamically overriding sections of the configuration. - [#980](https://github.com/influxdata/kapacitor/pull/980): Upgrade to using go 1.7 - [#957](https://github.com/influxdata/kapacitor/issues/957): Add API endpoints for testing service integrations. - [#958](https://github.com/influxdata/kapacitor/issues/958): Add support for Slack icon emojis and custom usernames. - [#991](https://github.com/influxdata/kapacitor/pull/991): Bring Kapacitor up to parity with available InfluxQL functions in 1.1 ### Bugfixes - [#984](https://github.com/influxdata/kapacitor/issues/984): Fix bug where keeping a list of fields that where not referenced in the eval expressions would cause an error. - [#955](https://github.com/influxdata/kapacitor/issues/955): Fix the number of subscriptions statistic. - [#999](https://github.com/influxdata/kapacitor/issues/999): Fix inconsistency with InfluxDB by adding config option to set a default retention policy. - [#1018](https://github.com/influxdata/kapacitor/pull/1018): Sort and dynamically adjust column width in CLI output. Fixes #785 - [#1019](https://github.com/influxdata/kapacitor/pull/1019): Adds missing strLength function. ## v1.0.2 [2016-10-06] ### Release Notes ### Features ### Bugfixes - [#951](https://github.com/influxdata/kapacitor/pull/951): Fix bug where errors to save cluster/server ID files were ignored. - [#954](https://github.com/influxdata/kapacitor/pull/954): Create data_dir on startup if it does not exist. ## v1.0.1 [2016-09-26] ### Release Notes ### Features - [#873](https://github.com/influxdata/kapacitor/pull/873): Add TCP alert handler - [#869](https://github.com/influxdata/kapacitor/issues/869): Add ability to set alert message as a field - [#854](https://github.com/influxdata/kapacitor/issues/854): Add `.create` property to InfluxDBOut node, which when set will create the database and retention policy on task start. - [#909](https://github.com/influxdata/kapacitor/pull/909): Allow duration / duration in TICKscript. - [#777](https://github.com/influxdata/kapacitor/issues/777): Add support for string manipulation functions. - [#886](https://github.com/influxdata/kapacitor/issues/886): Add ability to set specific HTTP port and hostname per configured InfluxDB cluster. ### Bugfixes - [#889](https://github.com/influxdata/kapacitor/issues/889): Some typo in the default config file - [#914](https://github.com/influxdata/kapacitor/pull/914): Change |log() output to be in JSON format so its self documenting structure. - [#915](https://github.com/influxdata/kapacitor/pull/915): Fix issue with TMax and the Holt-Winters method. - [#927](https://github.com/influxdata/kapacitor/pull/927): Fix bug with TMax and group by time. ## v1.0.0 [2016-09-02] ### Release Notes Final release of v1.0.0. ## v1.0.0-rc3 [2016-09-01] ### Release Notes ### Features ### Bugfixes - [#842](https://github.com/influxdata/kapacitor/issues/842): Fix side-effecting modification in batch WhereNode. ## v1.0.0-rc2 [2016-08-29] ### Release Notes ### Features - [#827](https://github.com/influxdata/kapacitor/issues/827): Bring Kapacitor up to parity with available InfluxQL functions in 1.0 ### Bugfixes - [#763](https://github.com/influxdata/kapacitor/issues/763): Fix NaNs begin returned from the `sigma` stateful function. - [#468](https://github.com/influxdata/kapacitor/issues/468): Fix tickfmt munging escaped slashes in regexes. ## v1.0.0-rc1 [2016-08-22] ### Release Notes #### Alert reset expressions Kapacitor now supports alert reset expressions. This way when an alert enters a state, it can only be lowered in severity if its reset expression evaluates to true. Example: ```go stream |from() .measurement('cpu') .where(lambda: "host" == 'serverA') .groupBy('host') |alert() .info(lambda: "value" > 60) .infoReset(lambda: "value" < 50) .warn(lambda: "value" > 70) .warnReset(lambda: "value" < 60) .crit(lambda: "value" > 80) .critReset(lambda: "value" < 70) ``` For example given the following values: 61 73 64 85 62 56 47 The corresponding alert states are: INFO WARNING WARNING CRITICAL INFO INFO OK ### Features - [#740](https://github.com/influxdata/kapacitor/pull/740): Support reset expressions to prevent an alert from being lowered in severity. Thanks @minhdanh! - [#670](https://github.com/influxdata/kapacitor/issues/670): Add ability to supress OK recovery alert events. - [#804](https://github.com/influxdata/kapacitor/pull/804): Add API endpoint for refreshing subscriptions. Also fixes issue where subs were not relinked if the sub was deleted. UDP listen ports are closed when a database is dropped. ### Bugfixes - [#783](https://github.com/influxdata/kapacitor/pull/783): Fix panic when revoking tokens not already defined. - [#784](https://github.com/influxdata/kapacitor/pull/784): Fix several issues with comment formatting in TICKscript. - [#786](https://github.com/influxdata/kapacitor/issues/786): Deleting tags now updates the group by dimensions if needed. - [#772](https://github.com/influxdata/kapacitor/issues/772): Delete task snapshot data when a task is deleted. - [#797](https://github.com/influxdata/kapacitor/issues/797): Fix panic from race condition in task master. - [#811](https://github.com/influxdata/kapacitor/pull/811): Fix bug where subscriptions + tokens would not work with more than one InfluxDB cluster. - [#812](https://github.com/influxdata/kapacitor/issues/812): Upgrade to use protobuf version 3.0.0 ## v1.0.0-beta4 [2016-07-27] ### Release Notes #### Group By Fields Kapacitor now supports grouping by fields. First convert a field into a tag using the EvalNode. Then group by the new tag. Example: ```go stream |from() .measurement('alerts') // Convert field 'level' to tag. |eval(lambda: string("level")) .as('level') .tags('level') // Group by new tag 'level'. |groupBy('alert', 'level') |... ``` Note the field `level` is now removed from the point since `.keep` was not used. See the [docs](https://docs.influxdata.com/kapacitor/v1.0/nodes/eval_node/#tags) for more details on how `.tags` works. #### Delete Fields or Tags In companion with being able to create new tags, you can now delete tags or fields. Example: ```go stream |from() .measurement('alerts') |delete() // Remove the field `extra` and tag `uuid` from all points. .field('extra') .tag('uuid') |... ``` ### Features - [#702](https://github.com/influxdata/kapacitor/pull/702): Add plumbing for authentication backends. - [#624](https://github.com/influxdata/kapacitor/issue/624): BREAKING: Add ability to GroupBy fields. First use EvalNode to create a tag from a field and then group by the new tag. Also allows for grouping by measurement. The breaking change is that the group ID format has changed to allow for the measurement name. - [#759](https://github.com/influxdata/kapacitor/pull/759): Add mechanism for token based subscription auth. - [#745](https://github.com/influxdata/kapacitor/pull/745): Add if function for tick script, for example: `if("value" > 6, 1, 2)`. ### Bugfixes - [#710](https://github.com/influxdata/kapacitor/pull/710): Fix infinite loop when parsing unterminated regex in TICKscript. - [#711](https://github.com/influxdata/kapacitor/issues/711): Fix where database name with quotes breaks subscription startup logic. - [#719](https://github.com/influxdata/kapacitor/pull/719): Fix panic on replay. - [#723](https://github.com/influxdata/kapacitor/pull/723): BREAKING: Search for valid configuration on startup in ~/.kapacitor and /etc/kapacitor/. This is so that the -config CLI flag is not required if the configuration is found in a standard location. The configuration file being used is always logged to STDERR. - [#298](https://github.com/influxdata/kapacitor/issues/298): BREAKING: Change alert level evaluation so each level is independent and not required to be a subset of the previous level. The breaking change is that expression evaluation order changed. As a result stateful expressions that relied on that order are broken. - [#749](https://github.com/influxdata/kapacitor/issues/749): Fix issue with tasks with empty DAG. - [#718](https://github.com/influxdata/kapacitor/issues/718): Fix broken extra expressions for deadman's switch. - [#752](https://github.com/influxdata/kapacitor/issues/752): Fix various bugs relating to the `fill` operation on a JoinNode. Fill with batches and fill when using the `on` property were broken. Also changes the DefaultNode set defaults for nil fields. ## v1.0.0-beta3 [2016-07-09] ### Release Notes ### Features - [#662](https://github.com/influxdata/kapacitor/pull/662): Add `-skipVerify` flag to `kapacitor` CLI tool to skip SSL verification. - [#680](https://github.com/influxdata/kapacitor/pull/680): Add Telegram Alerting option, thanks @burdandrei! - [#46](https://github.com/influxdata/kapacitor/issues/46): Can now create combinations of points within the same stream. This is kind of like join but instead joining a stream with itself. - [#669](https://github.com/influxdata/kapacitor/pull/669): Add size function for humanize byte size. thanks @jsvisa! - [#697](https://github.com/influxdata/kapacitor/pull/697): Can now flatten a set of points into a single points creating dynamcially named fields. - [#698](https://github.com/influxdata/kapacitor/pull/698): Join delimiter can be specified. - [#695](https://github.com/influxdata/kapacitor/pull/695): Bash completion filters by enabled disabled status. Thanks @bbczeuz! - [#706](https://github.com/influxdata/kapacitor/pull/706): Package UDF agents - [#707](https://github.com/influxdata/kapacitor/pull/707): Add size field to BeginBatch struct of UDF protocol. Provides hint as to size of incoming batch. ### Bugfixes - [#656](https://github.com/influxdata/kapacitor/pull/656): Fix issues where an expression could not be passed as a function parameter in TICKscript. - [#627](https://github.com/influxdata/kapacitor/issues/627): Fix where InfluxQL functions that returned a batch could drop tags. - [#674](https://github.com/influxdata/kapacitor/issues/674): Fix panic with Join On and batches. - [#665](https://github.com/influxdata/kapacitor/issues/665): BREAKING: Fix file mode not being correct for Alert.Log files. Breaking change is that integers numbers prefixed with a 0 in TICKscript are interpreted as octal numbers. - [#667](https://github.com/influxdata/kapacitor/issues/667): Align deadman timestamps to interval. ## v1.0.0-beta2 [2016-06-17] ### Release Notes ### Features - [#636](https://github.com/influxdata/kapacitor/pull/636): Change HTTP logs to be in Common Log format. - [#652](https://github.com/influxdata/kapacitor/pull/652): Add optional replay ID to the task API so that you can get information about a task inside a running replay. ### Bugfixes - [#621](https://github.com/influxdata/kapacitor/pull/621): Fix obscure error about single vs double quotes. - [#623](https://github.com/influxdata/kapacitor/pull/623): Fix issues with recording metadata missing data url. - [#631](https://github.com/influxdata/kapacitor/issues/631): Fix issues with using iterative lambda expressions in an EvalNode. - [#628](https://github.com/influxdata/kapacitor/issues/628): BREAKING: Change `kapacitord config` to not search default location for configuration files but rather require the `-config` option. Since the `kapacitord run` command behaves this way they should be consistent. Fix issue with `kapacitord config > kapacitor.conf` when the output file was a default location for the config. - [#626](https://github.com/influxdata/kapacitor/issues/626): Fix issues when changing the ID of an enabled task. - [#624](https://github.com/influxdata/kapacitor/pull/624): Fix issues where you could get a read error on a closed UDF socket. - [#651](https://github.com/influxdata/kapacitor/pull/651): Fix issues where an error during a batch replay would hang because the task wouldn't stop. - [#650](https://github.com/influxdata/kapacitor/pull/650): BREAKING: The default retention policy name was changed to `autogen` in InfluxDB. This changes Kapacitor to use `autogen` for the default retention policy for the stats. You may need to update your task DBRPs to use `autogen` instead of `default`. ## v1.0.0-beta1 [2016-06-06] ### Release Notes #### Template Tasks The ability to create and use template tasks has been added. you can define a template for a task and reuse that template across multiple tasks. A simple example: ```go // Which measurement to consume var measurement string // Optional where filter var where_filter = lambda: TRUE // Optional list of group by dimensions var groups = [*] // Which field to process var field string // Warning criteria, has access to 'mean' field var warn lambda // Critical criteria, has access to 'mean' field var crit lambda // How much data to window var window = 5m // The slack channel for alerts var slack_channel = '#alerts' stream |from() .measurement(measurement) .where(where_filter) .groupBy(groups) |window() .period(window) .every(window) |mean(field) |alert() .warn(warn) .crit(crit) .slack() .channel(slack_channel) ``` Then you can define the template like so: ``` kapacitor define-template generic_mean_alert -tick path/to/above/script.tick -type stream ``` Next define a task that uses the template: ``` kapacitor define cpu_alert -template generic_mean_alert -vars cpu_vars.json -dbrp telegraf.default ``` Where `cpu_vars.json` would like like this: ```json { "measurement": {"type" : "string", "value" : "cpu" }, "where_filter": {"type": "lambda", "value": "\"cpu\" == 'cpu-total'"}, "groups": {"type": "list", "value": [{"type":"string", "value":"host"},{"type":"string", "value":"dc"}]}, "field": {"type" : "string", "value" : "usage_idle" }, "warn": {"type" : "lambda", "value" : " \"mean\" < 30.0" }, "crit": {"type" : "lambda", "value" : " \"mean\" < 10.0" }, "window": {"type" : "duration", "value" : "1m" }, "slack_channel": {"type" : "string", "value" : "#alerts_testing" } } ``` #### Live Replays With this release you can now replay data directly against a task from InfluxDB without having to first create a recording. Replay the queries defined in the batch task `cpu_alert` for the past 10 hours. ```sh kapacitor replay-live batch -task cpu_alert -past 10h ``` Or for a stream task with use a query directly: ```sh kapacitor replay-live query -task cpu_alert -query 'SELECT usage_idle FROM telegraf."default".cpu WHERE time > now() - 10h' ``` #### HTTP based subscriptions Now InfluxDB and Kapacitor support HTTP/S based subscriptions. This means that Kapacitor need only listen on a single port for the HTTP service, greatly simplifying configuration and setup. In order to start using HTTP subscriptions change the `subscription-protocol` option for your configured InfluxDB clusters. For example: ``` [[influxdb]] enabled = true urls = ["http://localhost:8086",] subscription-protocol = "http" # or to use https #subscription-protocol = "https" ``` On startup Kapacitor will detect the change and recreate the subscriptions in InfluxDB to use the HTTP protocol. >NOTE: While HTTP itself is a TCP transport such that packet loss shouldn't be an issue, if Kapacitor starts to slow down for whatever reason, InfluxDB will drop the subscription writes to Kapacitor. In order to know if subscription writes are being dropped you should monitor the measurement `_internal.monitor.subscriber` for the field `writeFailures`. #### Holt-Winters Forecasting This release contains an new Holt Winters InfluxQL function. With this forecasting method one can now define an alert based off forecasted future values. For example, the following TICKscript will take the last 30 days of disk usage stats and using holt-winters forecast the next 7 days. If the forecasted value crosses a threshold an alert is triggered. The result is now Kapacitor will alert you 7 days in advance of a disk filling up. This assumes a slow growth but by changing the vars in the script you could check for shorter growth intervals. ```go // The interval on which to aggregate the disk usage var growth_interval = 1d // The number of `growth_interval`s to forecast into the future var forecast_count = 7 // The amount of historical data to use for the fit var history = 30d // The critical threshold on used_percent var threshold = 90.0 batch |query(''' SELECT max(used_percent) as used_percent FROM "telegraf"."default"."disk" ''') .period(history) .every(growth_interval) .align() .groupBy(time(growth_interval), *) |holtWinters('used_percent', forecast_count, 0, growth_interval) .as('used_percent') |max('used_percent') .as('used_percent') |alert() // Trigger alert if the forecasted disk usage is greater than threshold .crit(lambda: "used_percent" > threshold) ``` ### Features - [#283](https://github.com/influxdata/kapacitor/issues/283): Add live replays. - [#500](https://github.com/influxdata/kapacitor/issues/500): Support Float,Integer,String and Boolean types. - [#82](https://github.com/influxdata/kapacitor/issues/82): Multiple services for PagerDuty alert. thanks @savagegus! - [#558](https://github.com/influxdata/kapacitor/pull/558): Preserve fields as well as tags on selector InfluxQL functions. - [#259](https://github.com/influxdata/kapacitor/issues/259): Template Tasks have been added. - [#562](https://github.com/influxdata/kapacitor/pull/562): HTTP based subscriptions. - [#595](https://github.com/influxdata/kapacitor/pull/595): Support counting and summing empty batches to 0. - [#596](https://github.com/influxdata/kapacitor/pull/596): Support new group by time offset i.e. time(30s, 5s) - [#416](https://github.com/influxdata/kapacitor/issues/416): Track ingress counts by database, retention policy, and measurement. Expose stats via cli. - [#586](https://github.com/influxdata/kapacitor/pull/586): Add spread stateful function. thanks @upccup! - [#600](https://github.com/influxdata/kapacitor/pull/600): Add close http response after handler laert post, thanks @jsvisa! - [#606](https://github.com/influxdata/kapacitor/pull/606): Add Holt-Winters forecasting method. - [#605](https://github.com/influxdata/kapacitor/pull/605): BREAKING: StatsNode for batch edge now count the number of points in a batch instead of count batches as a whole. This is only breaking if you have a deadman switch configured on a batch edge. - [#611](https://github.com/influxdata/kapacitor/pull/611): Adds bash completion to the kapacitor CLI tool. ### Bugfixes - [#540](https://github.com/influxdata/kapacitor/issues/540): Fixes bug with log level API endpoint. - [#521](https://github.com/influxdata/kapacitor/issues/521): EvalNode now honors groups. - [#561](https://github.com/influxdata/kapacitor/issues/561): Fixes bug when lambda expressions would return error about types with nested binary expressions. - [#555](https://github.com/influxdata/kapacitor/issues/555): Fixes bug where "time" functions didn't work in lambda expressions. - [#570](https://github.com/influxdata/kapacitor/issues/570): Removes panic in SMTP service on failed close connection. - [#587](https://github.com/influxdata/kapacitor/issues/587): Allow number literals without leading zeros. - [#584](https://github.com/influxdata/kapacitor/issues/584): Do not block during startup to send usage stats. - [#553](https://github.com/influxdata/kapacitor/issues/553): Periodically check if new InfluxDB DBRPs have been created. - [#602](https://github.com/influxdata/kapacitor/issues/602): Fix missing To property on email alert handler. - [#581](https://github.com/influxdata/kapacitor/issues/581): Record/Replay batch tasks get cluster info from task not API. - [#613](https://github.com/influxdata/kapacitor/issues/613): BREAKING: Allow the ID of templates and tasks to be updated via the PATCH method. The breaking change is that now PATCH request return a 200 with the template or task definition, where before they returned 204. ## v0.13.1 [2016-05-13] ### Release Notes >**Breaking changes may require special upgrade steps from versions <= 0.12, please read the 0.13.0 release notes** Along with the API changes of 0.13.0, validation logic was added to task IDs, but this was not well documented. This minor release remedies that. All IDs (tasks, recordings, replays) must match this regex `^[-\._\p{L}0-9]+$`, which is essentially numbers, unicode letters, '-', '.' and '_'. If you have existing tasks which do not match this pattern they should continue to function normally. ### Features ### Bugfixes - [#545](https://github.com/influxdata/kapacitor/issues/545): Fixes inconsistency with API docs for creating a task. - [#544](https://github.com/influxdata/kapacitor/issues/544): Fixes issues with existings tasks and invalid names. - [#543](https://github.com/influxdata/kapacitor/issues/543): Fixes default values not being set correctly in API calls. ## v0.13.0 [2016-05-11] ### Release Notes >**Breaking changes may require special upgrade steps please read below.** #### Upgrade Steps Changes to how and where task data is store have been made. In order to safely upgrade to version 0.13 you need to follow these steps: 1. Upgrade InfluxDB to version 0.13 first. 2. Update all TICKscripts to use the new `|` and `@` operators. Once Kapacitor no longer issues any `DEPRECATION` warnings you are ready to begin the upgrade. The upgrade will work without this step but tasks using the old syntax cannot be enabled, until modified to use the new syntax. 3. Upgrade the Kapacitor binary/package. 4. Configure new database location. By default the location `/var/lib/kapacitor/kapacitor.db` is chosen for package installs or `./kapacitor.db` for manual installs. Do **not** remove the configuration for the location of the old task.db database file since it is still needed to do the migration. ``` [storage] boltdb = "/var/lib/kapacitor/kapacitor.db" ``` 5. Restart Kapacitor. At this point Kapacitor will migrate all existing data to the new database file. If any errors occur Kapacitor will log them and fail to startup. This way if Kapacitor starts up you can be sure the migration was a success and can continue normal operation. The old database is opened in read only mode so that existing data cannot be corrupted. Its recommended to start Kapacitor in debug logging mode for the migration so you can follow the details of the migration process. At this point you may remove the configuration for the old `task` `dir` and restart Kapacitor to ensure everything is working. Kapacitor will attempt the migration on every startup while the old configuration and db file exist, but will skip any data that was already migrated. #### API Changes With this release the API has been updated to what we believe will be the stable version for a 1.0 release. Small changes may still be made but the significant work to create a RESTful HTTP API is complete. Many breaking changes introduced, see the [client/API.md](http://github.com/influxdata/kapacitor/blob/master/client/API.md) doc for details on how the API works now. #### CLI Changes Along with the API changes, breaking changes where also made to the `kapacitor` CLI command. Here is a break down of the CLI changes: * Every thing has an ID now: tasks, recordings, even replays. The `name` used before to define a task is now its `ID`. As such instead of using `-name` and `-id` to refer to tasks and recordings, the flags have been changed to `-task` and `-recording` accordingly. * Replays can be listed and deleted like tasks and recordings. * Replays default to `fast` clock mode. * The record and replay commands now have a `-no-wait` option to start but not wait for the recording/replay to complete. * Listing recordings and replays displays the status of the respective action. * Record and Replay command now have an optional flag `-replay-id`/`-recording-id` to specify the ID of the replay or recording. If not set then a random ID will be chosen like the previous behavior. #### Notable features UDF can now be managed externally to Kapacitor via Unix sockets. A process or container can be launched independent of Kapacitor exposing a socket. On startup Kapacitor will connect to the socket and begin communication. Example UDF config for a socket based UDF. ``` [udf] [udf.functions] [udf.functions.myCustomUDF] socket = "/path/to/socket" timeout = "10s" ``` Alert data can now be consumed directly from within TICKscripts. For example, let's say we want to store all data that triggered an alert in InfluxDB with a tag `level` containing the level string value (i.e CRITICAL). ```javascript ... |alert() .warn(...) .crit(...) .levelTag('level') // and/or use a field //.levelField('level') // Also tag the data with the alert ID .idTag('id') // and/or use a field //.idField('id') |influxDBOut() .database('alerts') ... ``` ### Features - [#360](https://github.com/influxdata/kapacitor/pull/360): Forking tasks by measurement in order to improve performance - [#386](https://github.com/influxdata/kapacitor/issues/386): Adds official Go HTTP client package. - [#399](https://github.com/influxdata/kapacitor/issues/399): Allow disabling of subscriptions. - [#417](https://github.com/influxdata/kapacitor/issues/417): UDFs can be connected over a Unix socket. This enables UDFs from across Docker containers. - [#451](https://github.com/influxdata/kapacitor/issues/451): StreamNode supports `|groupBy` and `|where` methods. - [#93](https://github.com/influxdata/kapacitor/issues/93): AlertNode now outputs data to child nodes. The output data can have either a tag or field indicating the alert level. - [#281](https://github.com/influxdata/kapacitor/issues/281): AlertNode now has an `.all()` property that specifies that all points in a batch must match the criteria in order to trigger an alert. - [#384](https://github.com/influxdata/kapacitor/issues/384): Add `elapsed` function to compute the time difference between subsequent points. - [#230](https://github.com/influxdata/kapacitor/issues/230): Alert.StateChangesOnly now accepts optional duration arg. An alert will be triggered for every interval even if the state has not changed. - [#426](https://github.com/influxdata/kapacitor/issues/426): Add `skip-format` query parameter to the `GET /task` endpoint so that returned TICKscript content is left unmodified from the user input. - [#388](https://github.com/influxdata/kapacitor/issues/388): The duration of an alert is now tracked and exposed as part of the alert data as well as can be set as a field via `.durationField('duration')`. - [#486](https://github.com/influxdata/kapacitor/pull/486): Default config file location. - [#461](https://github.com/influxdata/kapacitor/pull/461): Make Alerta `event` property configurable. - [#491](https://github.com/influxdata/kapacitor/pull/491): BREAKING: Rewriting stateful expression in order to improve performance, the only breaking change is: short circuit evaluation for booleans - for example: ``lambda: "bool_value" && (count() > 100)`` if "bool_value" is false, we won't evaluate "count". - [#504](https://github.com/influxdata/kapacitor/pull/504): BREAKING: Many changes to the API and underlying storage system. This release requires a special upgrade process. - [#511](https://github.com/influxdata/kapacitor/pull/511): Adds DefaultNode for providing default values for missing fields or tags. - [#285](https://github.com/influxdata/kapacitor/pull/285): Track created,modified and last enabled dates on tasks. - [#533](https://github.com/influxdata/kapacitor/pull/533): Add useful statistics for nodes. ### Bugfixes - [#499](https://github.com/influxdata/kapacitor/issues/499): Fix panic in InfluxQL nodes if field is missing or incorrect type. - [#441](https://github.com/influxdata/kapacitor/issues/441): Fix panic in UDF code. - [#429](https://github.com/influxdata/kapacitor/issues/429): BREAKING: Change TICKscript parser to be left-associative on equal precedence operators. For example previously this statement `(1+2-3*4/5)` was evaluated as `(1+(2-(3*(4/5))))` which is not the typical/expected behavior. Now using left-associative parsing the statement is evaluated as `((1+2)-((3*4)/5))`. - [#456](https://github.com/influxdata/kapacitor/pull/456): Fixes Alerta integration to let server set status, fix `rawData` attribute and set default severity to `indeterminate`. - [#425](https://github.com/influxdata/kapacitor/pull/425): BREAKING: Preserving tags on influxql simple selectors - first, last, max, min, percentile - [#423](https://github.com/influxdata/kapacitor/issues/423): Recording stream queries with group by now correctly saves data in time order not group by order. - [#331](https://github.com/influxdata/kapacitor/issues/331): Fix panic when missing `.as()` for JoinNode. - [#523](https://github.com/influxdata/kapacitor/pull/523): JoinNode will now emit join sets as soon as they are ready. If multiple joinable sets arrive in the same tolerance window than each will be emitted (previously the first points were dropped). - [#537](https://github.com/influxdata/kapacitor/issues/537): Fix panic in alert node when batch is empty. ## v0.12.0 [2016-04-04] ### Release Notes New TICKscript syntax that uses a different operators for chaining methods vs property methods vs UDF methods. * A chaining method is a method that creates a new node in the pipeline. Uses the `|` operator. * A property method is a method that changes a property on a node. Uses the `.` operator. * A UDF method is a method that calls out to a UDF. Uses the `@` operator. For example below the `from`, `mean`, and `alert` methods create new nodes, the `detectAnomalies` method calls a UDF, and the other methods modify the nodes as property methods. ```javascript stream |from() .measurement('cpu') .where(lambda: "cpu" == 'cpu-total') |mean('usage_idle') .as('value') @detectAnomalies() .field('mean') |alert() .crit(lambda: "anomaly_score" > 10) .log('/tmp/cpu.log') ``` With this change a new binary is provided with Kapacitor `tickfmt` which will format a TICKscript file according to a common standard. ### Features - [#299](https://github.com/influxdata/kapacitor/issues/299): Changes TICKscript chaining method operators and adds `tickfmt` binary. - [#389](https://github.com/influxdata/kapacitor/pull/389): Adds benchmarks to Kapacitor for basic use cases. - [#390](https://github.com/influxdata/kapacitor/issues/390): BREAKING: Remove old `.mapReduce` functions. - [#381](https://github.com/influxdata/kapacitor/pull/381): Adding enable/disable/delete/reload tasks by glob. - [#401](https://github.com/influxdata/kapacitor/issues/401): Add `.align()` property to BatchNode so you can align query start and stop times. ### Bugfixes - [#378](https://github.com/influxdata/kapacitor/issues/378): Fix issue where derivative would divide by zero. - [#387](https://github.com/influxdata/kapacitor/issues/387): Add `.quiet()` option to EvalNode so errors can be suppressed if expected. - [#400](https://github.com/influxdata/kapacitor/issues/400): All query/connection errors are counted and reported in BatchNode stats. - [#412](https://github.com/influxdata/kapacitor/pull/412): Fix issues with batch queries dropping points because of nil fields. - [#413](https://github.com/influxdata/kapacitor/pull/413): Allow disambiguation between ".groupBy" and "|groupBy". ## v0.11.0 [2016-03-22] ### Release Notes Kapacitor is now using the functions from the new query engine in InfluxDB core. Along with this change is a change in the TICKscript API so that using the InfluxQL functions is easier. Simply call the desired method directly no need to call `.mapReduce` explicitly. This change now hides the mapReduce aspect and handles it internally. Using `.mapReduce` is officially deprecated in this release and will be remove in the next major release. We feel that this change improves the readability of TICKscripts and exposes less implementation details to the end user. Updating your exising TICKscripts is simple. If previously you had code like this: ```javascript stream.from()... .window()... .mapReduce(influxql.count('value')) ``` then update it to look like this: ```javascript stream.from()... .window()... .count('value') ``` a simple regex could fix all your existing scripts. Kapacitor now exposes more internal metrics for determining the performance of a given task. The internal statistics includes a new measurement named `node` that contains any stats a node provides, tagged by the task, node, task type and kind of node (i.e. window vs union). All nodes provide an averaged execution time for the node. These stats are also available in the DOT output of the Kapacitor show command. Significant performance improvements have also been added. In some cases Kapacitor throughput has improved by 4X. Kapacitor can now connect to different InfluxDB clusters. Multiple InfluxDB config sections can be defined and one will be marked as default. To upgrade convert an `influxdb` config. From this: ``` [influxdb] enabled = true ... ``` to this: ``` [[influxdb]] enabled = true default = true name = "localhost" ... ``` Various improvements to joining features have been implemented. With #144 you can now join streams with differing group by dimensions. If you previously configured Email, Slack or HipChat globally now you must also set the `state-changes-only` option to true as well if you want to preserve the original behavior. For example: ``` [slack] enable = true global = true state-changes-only = true ``` ### Features - [#236](https://github.com/influxdata/kapacitor/issues/236): Implement batched group by - [#231](https://github.com/influxdata/kapacitor/pull/231): Add ShiftNode so values can be shifted in time for joining/comparisons. - [#190](https://github.com/influxdata/kapacitor/issues/190): BREAKING: Deadman's switch now triggers off emitted counts and is grouped by to original grouping of the data. The breaking change is that the 'collected' stat is no longer output for `.stats` and has been replaced by `emitted`. - [#145](https://github.com/influxdata/kapacitor/issues/145): The InfluxDB Out Node now writes data to InfluxDB in buffers. - [#215](https://github.com/influxdata/kapacitor/issues/215): Add performance metrics to nodes for average execution times and node throughput values. - [#144](https://github.com/influxdata/kapacitor/issues/144): Can now join streams with differing dimensions using the join.On property. - [#249](https://github.com/influxdata/kapacitor/issues/249): Can now use InfluxQL functions directly instead of via the MapReduce method. Example `stream.from().count()`. - [#233](https://github.com/influxdata/kapacitor/issues/233): BREAKING: Now you can use multiple InfluxDB clusters. The config changes to make this possible are breaking. See notes above for changes. - [#302](https://github.com/influxdata/kapacitor/issues/302): Can now use .Time in alert message. - [#239](https://github.com/influxdata/kapacitor/issues/239): Support more detailed TLS config when connecting to an InfluxDB host. - [#323](https://github.com/influxdata/kapacitor/pull/323): Stats for task execution are provided via JSON HTTP request instead of just DOT string. thanks @yosiat - [#358](https://github.com/influxdata/kapacitor/issues/358): Improved logging. Adds LogNode so any data in a pipeline can be logged. - [#366](https://github.com/influxdata/kapacitor/issues/366): HttpOutNode now allows chaining methods. ### Bugfixes - [#199](https://github.com/influxdata/kapacitor/issues/199): BREAKING: Various fixes for the Alerta integration. The `event` property has been removed from the Alerta node and is now set as the value of the alert ID. - [#232](https://github.com/influxdata/kapacitor/issues/232): Better error message for alert integrations. Better error message for VictorOps 404 response. - [#231](https://github.com/influxdata/kapacitor/issues/231): Fix window logic when there were gaps in the data stream longer than window every value. - [#213](https://github.com/influxdata/kapacitor/issues/231): Add SourceStreamNode so that yuou must always first call `.from` on the `stream` object before filtering it, so as to not create confusing to understand TICKscripts. - [#255](https://github.com/influxdata/kapacitor/issues/255): Add OPTIONS handler for task delete method so it can be preflighted. - [#258](https://github.com/influxdata/kapacitor/issues/258): Fix UDP internal metrics, change subscriptions to use clusterID. - [#240](https://github.com/influxdata/kapacitor/issues/240): BREAKING: Fix issues with Sensu integration. The breaking change is that the config no longer takes a `url` but rather a `host` option since the communication is raw TCP rather HTTP. - [#270](https://github.com/influxdata/kapacitor/issues/270): The HTTP server will now gracefully stop. - [#300](https://github.com/influxdata/kapacitor/issues/300): Add OPTIONS method to /recording endpoint for deletes. - [#304](https://github.com/influxdata/kapacitor/issues/304): Fix panic if recording query but do not have an InfluxDB instance configured - [#289](https://github.com/influxdata/kapacitor/issues/289): Add better error handling to batch node. - [#142](https://github.com/influxdata/kapacitor/issues/142): Fixes bug when defining multiple influxdb hosts. - [#266](https://github.com/influxdata/kapacitor/issues/266): Fixes error log for HipChat that is not an error. - [#333](https://github.com/influxdata/kapacitor/issues/333): Fixes hang when replaying with .stats node. Fixes issues with batch and stats. - [#340](https://github.com/influxdata/kapacitor/issues/340): BREAKING: Decouples global setting for alert handlers from the state changes only setting. - [#348](https://github.com/influxdata/kapacitor/issues/348): config.go: refactor to simplify structure and fix support for array elements - [#362](https://github.com/influxdata/kapacitor/issues/362): Fix bug with join tolerance and batches. ## v0.10.1 [2016-02-08] ### Release Notes This is a bug fix release that fixes many issues releated to the recent 0.10.0 release. The few additional features are focused on usability improvements from recent feedback. Improved UDFs, lots of bug fixes and improvements on the API. There was a breaking change for UDFs protobuf messages, see #176. There was a breaking change to the `define` command, see [#173](https://github.com/influxdata/kapacitor/issues/173) below. ### Features - [#176](https://github.com/influxdata/kapacitor/issues/176): BREAKING: Improved UDFs and groups. Now it is easy to deal with groups from the UDF process. There is a breaking change in the BeginBatch protobuf message for this change. - [#196](https://github.com/influxdata/kapacitor/issues/196): Adds a 'details' property to the alert node so that the email body can be defined. See also [#75](https://github.com/influxdata/kapacitor/issues/75). - [#132](https://github.com/influxdata/kapacitor/issues/132): Make is so multiple calls to `where` simply `AND` expressions together instead of replacing or creating extra nodes in the pipeline. - [#173](https://github.com/influxdata/kapacitor/issues/173): BREAKING: Added a `-no-reload` flag to the define command in the CLI. Now if the task is enabled define will automatically reload it unless `-no-reload` is passed. - [#194](https://github.com/influxdata/kapacitor/pull/194): Adds Talk integration for alerts. Thanks @wutaizeng! - [#320](https://github.com/influxdata/kapacitor/pull/320): Upgrade to go 1.6 ### Bugfixes - [#177](https://github.com/influxdata/kapacitor/issues/177): Fix panic for show command on batch tasks. - [#185](https://github.com/influxdata/kapacitor/issues/185): Fix panic in define command with invalid dbrp value. - [#195](https://github.com/influxdata/kapacitor/issues/195): Fix panic in where node. - [#208](https://github.com/influxdata/kapacitor/issues/208): Add default stats dbrp to default subscription excludes. - [#203](https://github.com/influxdata/kapacitor/issues/203): Fix hang when deleteing invalid batch task. - [#182](https://github.com/influxdata/kapacitor/issues/182): Fix missing/incorrect Content-Type headers for various HTTP endpoints. - [#187](https://github.com/influxdata/kapacitor/issues/187): Retry connecting to InfluxDB on startup for up to 5 minutes by default. ## v0.10.0 [2016-01-26] ### Release Notes This release marks the next major release of Kapacitor. With this release you can now run your own custom code for processing data within Kapacitor. See [udf/agent/README.md](https://github.com/influxdata/kapacitor/blob/master/udf/agent/README.md) for more details. With the addition of UDFs it is now possible to run custom anomaly detection alogrithms suited to your needs. There are simple examples of how to use UDFs in [udf/agent/examples](https://github.com/influxdata/kapacitor/tree/master/udf/agent/examples/). The version has jumped significantly so that it is inline with other projects in the TICK stack. This way you can easily tell which versions of Telegraf, InfluxDB, Chronograf and Kapacitor work together. See note on a breaking change in the HTTP API below. #163 ### Features - [#137](https://github.com/influxdata/kapacitor/issues/137): Add deadman's switch. Can be setup via TICKscript and globally via configuration. - [#72](https://github.com/influxdata/kapacitor/issues/72): Add support for User Defined Functions (UDFs). - [#139](https://github.com/influxdata/kapacitor/issues/139): Alerta.io support thanks! @md14454 - [#85](https://github.com/influxdata/kapacitor/issues/85): Sensu support using JIT clients. Thanks @sstarcher! - [#141](https://github.com/influxdata/kapacitor/issues/141): Time of day expressions for silencing alerts. ### Bugfixes - [#153](https://github.com/influxdata/kapacitor/issues/153): Fix panic if referencing non existant field in MapReduce function. - [#138](https://github.com/influxdata/kapacitor/issues/138): Change over to influxdata github org. - [#164](https://github.com/influxdata/kapacitor/issues/164): Update imports etc from InfluxDB as per the new meta store/client changes. - [#163](https://github.com/influxdata/kapacitor/issues/163): BREAKING CHANGE: Removed the 'api/v1' pathing from the HTTP API so that Kapacitor is path compatible with InfluxDB. While this is a breaking change the kapacitor cli has been updated accordingly and you will not experience any distruptions unless you were calling the HTTP API directly. - [#147](https://github.com/influxdata/kapacitor/issues/147): Compress .tar archives from builds. ## v0.2.4 [2016-01-07] ### Release Notes ### Features - [#118](https://github.com/influxdata/kapacitor/issues/118): Can now define multiple handlers of the same type on an AlertNode. - [#119](https://github.com/influxdata/kapacitor/issues/119): HipChat support thanks! @ericiles *2 - [#113](https://github.com/influxdata/kapacitor/issues/113): OpsGenie support thanks! @ericiles - [#107](https://github.com/influxdata/kapacitor/issues/107): Enable TICKscript variables to be defined and then referenced from lambda expressions. Also fixes various bugs around using regexes. ### Bugfixes - [#124](https://github.com/influxdata/kapacitor/issues/124): Fix panic where there is an error starting a task. - [#122](https://github.com/influxdata/kapacitor/issues/122): Fixes panic when using WhereNode. - [#128](https://github.com/influxdata/kapacitor/issues/128): Fix not sending emails when using recipient list from config. ## v0.2.3 [2015-12-22] ### Release Notes Bugfix #106 made a breaking change to the internal HTTP API. This was to facilitate integration testing and overall better design. Now POSTing a recording request will start the recording and immediately return. If you want to wait till it is complete do a GET for the recording info and it will block until its complete. The kapacitor cli has been updated accordingly. ### Features - [#96](https://github.com/influxdata/kapacitor/issues/96): Use KAPACITOR_URL env var for setting the kapacitord url in the client. - [#109](https://github.com/influxdata/kapacitor/pull/109): Add throughput counts to DOT format in `kapacitor show` command, if task is executing. ### Bugfixes - [#102](https://github.com/influxdata/kapacitor/issues/102): Fix race when start/stoping timeTicker in batch.go - [#106](https://github.com/influxdata/kapacitor/pull/106): Fix hang when replaying stream recording. ## v0.2.2 [2015-12-16] ### Release Notes Some bug fixes including one that cause Kapacitor to deadlock. ### Features - [#83](https://github.com/influxdata/kapacitor/pull/83): Use enterprise usage client, remove deprecated enterprise register and reporting features. ### Bugfixes - [#86](https://github.com/influxdata/kapacitor/issues/86): Fix dealock form errors in tasks. Also fixes issue where task failures did not get logged. - [#95](https://github.com/influxdata/kapacitor/pull/95): Fix race in bolt usage when starting enabled tasks at startup. ## v0.2.0 [2015-12-8] ### Release Notes Major public release. ================================================ FILE: vendor/github.com/influxdata/kapacitor/CONTRIBUTING.md ================================================ Contributing to Kapacitor ========================= Bug reports --------------- Before you file an issue, please search existing issues in case it has already been filed, or perhaps even fixed. If you file an issue, please include the following. * Full details of your operating system (or distribution) e.g. 64-bit Ubuntu 14.04. * The version of Kapacitor you are running * Whether you installed it using a pre-built package, or built it from source. * A small test case, if applicable, that demonstrates the issues. Remember the golden rule of bug reports: **The easier you make it for us to reproduce the problem, the faster it will get fixed.** If you have never written a bug report before, or if you want to brush up on your bug reporting skills, we recommend reading [Simon Tatham's essay "How to Report Bugs Effectively."](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html) Please note that issues are *not the place to file general questions* such as "how do I use InfluxDB with Kapacitor?" Questions of this nature should be sent to the [Google Group](https://groups.google.com/forum/#!forum/influxdb), not filed as issues. Issues like this will be closed. Feature requests --------------- We really like to receive feature requests, as it helps us prioritize our work. Please be clear about your requirements, as incomplete feature requests may simply be closed if we don't understand what you would like to see added to Kapacitor. Contributing to the source code --------------- Kapacitor follows standard Go project structure. This means that all your go development are done in `$GOPATH/src`. GOPATH can be any directory under which InfluxDB and all its dependencies will be cloned. For more details on recommended go project's structure, see [How to Write Go Code](http://golang.org/doc/code.html) and [Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/), or you can just follow the steps below. Submitting a pull request ------------ To submit a pull request you should fork the Kapacitor repository, and make your change on a feature branch of your fork. Then generate a pull request from your branch against *master* of the Kapacitor repository. Include in your pull request details of your change -- the why *and* the how -- as well as the testing your performed. Also, be sure to run the test suite with your change in place. Changes that cause tests to fail cannot be merged. There will usually be some back and forth as we finalize the change, but once that completes it may be merged. To assist in review for the PR, please add the following to your pull request comment: ```md - [ ] CHANGELOG.md updated - [ ] Rebased/mergable - [ ] Tests pass - [ ] Sign [CLA](http://influxdb.com/community/cla.html) (if not already signed) ``` Use of third-party packages --------------------------- A third-party package is defined as one that is not part of the standard Go distribution. Generally speaking we prefer to minimize our use of third-party packages, and avoid them unless absolutely necessarily. We'll often write a little bit of code rather than pull in a third-party package. So to maximise the chance your change will be accepted by us, use only the standard libraries, or the third-party packages we have decided to use. For rationale, check out the post [The Case Against Third Party Libraries](http://blog.gopheracademy.com/advent-2014/case-against-3pl/). Signing the CLA --------------- If you are going to be contributing back to Kapacitor please take a second to sign our CLA, which can be found [on our website](http://influxdb.com/community/cla.html). Installing Go ------------- Kapacitor typically requires the lastest version of Go. To install go see https://golang.org/dl/ Getting the source ------ Setup the project structure and fetch the repo like so: mkdir $HOME/go export GOPATH=$HOME/go go get github.com/influxdata/kapacitor You can add the line `export GOPATH=$HOME/go` to your bash/zsh file to be set for every shell instead of having to manually run it everytime. Cloning a fork ------------- If you wish to work with fork of Kapacitor, your own fork for example, you must still follow the directory structure above. But instead of cloning the main repo, instead clone your fork. Follow the steps below to work with a fork: export GOPATH=$HOME/go mkdir -p $GOPATH/src/github.com/influxdata cd $GOPATH/src/github.com/influxdata git clone git@github.com:/kapacitor Retaining the directory structure `$GOPATH/src/github.com/influxdata` is necessary so that Go imports work correctly. Pre-commit checks ------------- We have a pre-commit hook to make sure code is formatted properly and vetted before you commit any changes. We strongly recommend using the pre-commit hook to guard against accidentally committing unformatted code. To use the pre-commit hook, run the following: cd $GOPATH/src/github.com/influxdata/kapacitor cp .hooks/pre-commit .git/hooks/ In case the commit is rejected because it's not formatted you can run the following to format the code: ``` go fmt ./... go vet ./... ``` For more information on `go vet`, [read the GoDoc](https://godoc.org/golang.org/x/tools/cmd/vet). Build and Test -------------- Make sure you have Go installed and the project structure as shown above. To then build the project, execute the following commands: ```bash cd $GOPATH/src/github.com/influxdata/kapacitor go build ./cmd/kapacitor go build ./cmd/kapacitord ``` Kapacitor builds two binares is named `kapacitor`, and `kapacitord`. To run the tests, execute the following command: ```bash go test $(go list ./... | grep -v /vendor/) ``` Dependencies ------------ Kapacitor vendors all dependencies. Kapacitor uses the golang [dep](https://github.com/golang/dep) tool. Install the dep tool: ``` go get -u github.com/golang/dep ``` See the dep help for usage and documentation. Kapacitor commits vendored deps into the repo, as a result always run `dep prune` after any `dep ensure` operation. This helps keep the amount of code committed to a minimum. Generating Code --------------- Kapacitor uses generated code. The generated code is committed to the repository so normally it is not necessary to regenerate it. But if you modify one of the templates for code generation you must re-run the generate commands. Go provides a consistent command for generating all necessary code: ```bash go generate ./... ``` For the generate command to succeed you will need a few dependencies installed on your system. These dependencies are already vendored in the code and and can be installed from there. * tmpl -- A utility used to generate code from templates. Install via `go install ./vendor/github.com/benbjohnson/tmpl` * protoc + protoc-gen-go -- A protobuf compiler plus the protoc-gen-go extension. You need version 3.0.0 of protoc. To install the go plugin run `go install ./vendor/github.com/golang/protobuf/protoc-gen-go` The Build Script ---------------- The above commands have all be encapsulated for you in a `build.py` script. The script has flags for testing code, building binaries and complete distribution packages. To build kapacitor use: ```bash ./build.py ``` To run the tests use: ```bash ./build.py --test ``` If you want to generate code run: ```bash ./build.py --generate ``` If you want to build packages run: ```bash ./build.py --packages ``` There are many more options available see ```bash ./build.py --help ``` The Build Script + Docker ------------------------- Kapacitor requires a few extra dependencies to perform certain build actions. Specifically to build packages or to regenerate any of the generated code you will need a few extra tools. A `build.sh` script is provided that will run `build.py` in a docker container with all the needed dependencies installed with correct versions. All you need is to have docker installed and then use the `./build.sh` command as if it were the `./build.py` command. Profiling --------- When troubleshooting problems with CPU or memory the Go toolchain can be helpful. You can start InfluxDB with CPU or memory profiling turned on. For example: ```sh # start kapacitord with profiling ./kapacitord -cpuprofile kapacitord.prof # run task, replays whatever you're testing # Quit out of kapacitord and kapacitord.prof will then be written. # open up pprof to examine the profiling data. go tool pprof ./kapacitord kapacitord.prof # once inside run "web", opens up browser with the CPU graph # can also run "web " to zoom in. Or "list " to see specific lines ``` Note that when you pass the binary to `go tool pprof` *you must specify the path to the binary*. Continuous Integration testing ------------------------------ Kapacitor uses CircleCI for continuous integration testing. Useful links ------------ - [Useful techniques in Go](http://arslan.io/ten-useful-techniques-in-go) - [Go in production](http://peter.bourgon.org/go-in-production/) - [Principles of designing Go APIs with channels](https://inconshreveable.com/07-08-2014/principles-of-designing-go-apis-with-channels/) - [Common mistakes in Golang](http://soryy.com/blog/2014/common-mistakes-with-go-lang/). Especially this section `Loops, Closures, and Local Variables` ================================================ FILE: vendor/github.com/influxdata/kapacitor/DESIGN.md ================================================ # Kapacitor Internal Design This document is meant to layout both the high level design of Kapacitor as well and discuss the details of the implementation. It should be accessible to someone wanting to contribute to Kapacitor. ## Topic * Key Concepts * Data Flow -- How data flows through Kapacitor * TICKscript -- How TICKscript is implemented (not written yet) * kapacitord/kapacitor -- How the daemon ties all the pieces together. (not written yet) ## Key Concepts Kapacitor is a framework for processing time series data. It follows a [flow based programing](https://en.wikipedia.org/wiki/Flow-based_programming) model. Data flows from node to node and each node is a *black box* process that can manipulate the data in any way. The data model used to transport data from node to node matches the schema used by InfluxDB, namely measurements, tags and fields. Nodes can be arranged in a directed acyclic graph [(DAG)](https://en.wikipedia.org/wiki/Directed_acyclic_graph). ### Tasks Users define tasks for Kapacitor to run. A task defines a DAG of nodes that process the data. This task is defined via a DSL named TICKscript. To learn more about how to use and interact with Kapacitor see the [docs](https://docs.influxdata.com/kapacitor/). A task defines a potentially infinite amount of work to be done. The amount work is determined by the data that is received by the task. Once the source data stream is *closed* the task is complete. It is normal for a task to never complete but rather run indefinitely. As a result, tasks can be in one of three states, disabled, enabled not executing, and enabled executing. An enabled task is not executing if it encountered an error or its data source was closed. ## Data Flow Data flows from node to node and each node is a black box that can process the data the however it sees fit. In order for a system like this to work the transport method and data model needs to be well defined. ### Models The data model for transporting data has two types: * Stream -- Data points are passed as single entities. * Batch -- Data points are passed in groups of data. A batch consists of a type that describes the common attributes of all data points within the batch and a list of all the individual data points. A data point consists of a timestamp, a map of fields, and a map of tags. When data points are transfered as a stream not within the context of a batch they also contain information on their scope, i.e database, retention policy and measurement. This data model is schemaless in that the names of fields and tags are arbitrary and opaque to Kapacitor. Lastly both batches and streamed data points contain information about the *group* they belong two if the data set has been grouped by any number of dimensions. More on that later. ### Time Time is measured based on the timestamps of the data flowing through a node. If data flow stops so does time. If a node performs a transformation dependent on time then it is always consistent based on a given data set. ### Edges Kapacitor models data transfer along *edges*. An edge connects exactly two nodes and data flows from the *parent* node to the *child* node. There are two actions performed on an edge: * Collect -- The parent presents a data point for the edge to consume. * Emit -- The child retrieves a data point from the edge. From the perspective of an *edge* data is collected from a parent and then emitted to a child. From the perspective of a *node*, data is pulled off from any number of parent edges and collected into any number of child edges. Nodes, not edges, control the flow of the data. Edges simply provide the transport mechanism. Meaning that if a child node stops pulling data from its parent edge, data flow stops. Edges are typed, meaning that a given edge only transports a certain type of data, i.e. streams or batches. Nodes are said to *want* parent edges of a certain type and to *provide* child edges of a certain type. A node can want and provide the same or different type of edges. For example the `WindowNode` wants a stream edge while providing a batch edge. Modeling data flow through edges allows for the transport mechanism to be abstracted. If the data is being transfered within the same process then it can be sent via in-memory structures; if the data is being transfered to another Kapacitor host it can be serialized and transfered accordingly. The current implementation of an edge uses Go channels and can be found in `edge.go`. There are three channel per edge instance but only ever one channel is non nil based on the type of the edge. Direct access to the channel is not provided but rather wrapper methods for collecting and emitting the data. This allows for the edge to keep counts on throughput and be aborted at any point. The channels are currently unbuffered, this will probably need to change eventually, but for now the simplicity is useful. Passing batch data can be accomplished in one of two ways. First, pass the data as a single object containing the complete batch and all points. Second, pass marker objects that indicate the beginning and end of batches and stream individual points between the markers. The marker objects can also contain the common data to the batch. Currently the first option is used. This has the advantage that fewer objects are passing through the channels. It also works better with the current map-reduce functions in core sense they expect all the data in a single object. It has the disadvantage that the whole batch has to be held in memory. In some cases the entire batch does need to live in memory but not in all. For example a node that is counting points per batch need only maintain a counter in memory and not the entire batch. ### Source Mapping Kapacitor can receive data from many different sources, including querying InfluxDB. The type TaskMaster in`task_master.go` is responsible for managing which tasks are receiving which data. For stream tasks this is done by having on global edge. All sources (graphite, collectd, http, etc) write their data to the TaskMaster, who writes the data to the global *stream* edge. When a stream task is started it gets a *fork* of the global stream filtered down by the databases and retention policies its allowed to access. Then the task can further process the data stream. In the case of the batch tasks, the TaskMaster manages starting the schedules for querying InfluxDB. The results of the queries are passed to the root nodes of the task directly. ### Windowing Windowing data is an important piece to creating pipelines. Windowing is concerned with how you can slice a data stream into multiple windows and is orthogonal to how batches are transfered. Kapacitor handles windowing explicitly, by allowing the user to define a WindowNode that has two parameters. First, the `period` is the length of the window in time. Second, the `every` property defines how often an window should be emitted into the stream. This allows for creating windows that overlap, have no overlap, or have gaps between the windows. As a result the concept of a window does not exist inherently in the data stream, but rather windowing is the method of converting a stream of data into a batch of data. Example TICKscript: ```javascript stream .window() .period(10s) .every(5s) ``` The above script slices the incoming stream into overlapping windows. Each window contains the last 10s of data and a new window is emitted every 5s. ### Challenges Challenges with the current implementation: * For stream tasks: If a single node stop processing data all nodes will eventually stop including nodes from other tasks. This is because of the global stream to aggregate all incoming sources and the fact the edges just block instead of dropping data. This could be mitigated further by creating independent streams for each database retention policy pair, but this only provides isolation and not a solution. We need a contract in place for what to do when a given node stops processing data. * Nodes are responsible for not creating deadlock in the way they read and write data from their parent and child edges. For example the `JoinNode` has multiple parents and has to guarantee that the goroutines that are reading from the parents never block on each other. Otherwise a deadlock can be created since a parent may be blocked writing to the JoinNode while the JoinNode is blocked reading from a different parent. Since both parents could have a common ancestor the blocked parent will eventually block the ancestor which in turn will block the other parent. * Fragile, so far the smallest of changes to the way the system work almost always results in a deadlock, because of the order of processing data. * If data flow stops so does time. In many use cases this is exactly what you want, but in some cases you would still like the data in transit to be flushed out. As for monitoring the throughput of tasks this is possible out-of-band of the task so even if the task stop processing data you can still trigger an event in a different task. ================================================ FILE: vendor/github.com/influxdata/kapacitor/Dockerfile_build_ubuntu32 ================================================ FROM 32bit/ubuntu:16.04 # This dockerfile capabable of the the minumum required # to run the tests and nothing else. MAINTAINER support@influxdb.com RUN apt-get -qq update && apt-get -qq install -y \ wget \ unzip \ git \ mercurial \ build-essential \ autoconf \ automake \ libtool \ python-setuptools \ zip \ curl # Install protobuf3 protoc binary ENV PROTO_VERSION 3.0.0 RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protoc-${PROTO_VERSION}-linux-x86_32.zip\ && unzip -j protoc-${PROTO_VERSION}-linux-x86_32.zip bin/protoc -d /bin \ rm protoc-${PROTO_VERSION}-linux-x86_64.zip # Install protobuf3 python library RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protobuf-python-${PROTO_VERSION}.tar.gz \ && tar -xf protobuf-python-${PROTO_VERSION}.tar.gz \ && cd /protobuf-${PROTO_VERSION}/python \ && python setup.py install \ && rm -rf /protobuf-${PROTO_VERSION} protobuf-python-${PROTO_VERSION}.tar.gz # Install go ENV GOPATH /root/go ENV GO_VERSION 1.7.5 ENV GO_ARCH 386 RUN wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \ tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \ rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ENV PATH /usr/local/go/bin:$PATH ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor ENV PATH $GOPATH/bin:$PATH RUN mkdir -p $PROJECT_DIR WORKDIR $PROJECT_DIR VOLUME $PROJECT_DIR ENTRYPOINT [ "/root/go/src/github.com/influxdata/kapacitor/build.py" ] ================================================ FILE: vendor/github.com/influxdata/kapacitor/Dockerfile_build_ubuntu64 ================================================ FROM ubuntu:latest # This dockerfile is capabable of performing all # build/test/package/deploy actions needed for Kapacitor. MAINTAINER support@influxdb.com RUN apt-get -qq update && apt-get -qq install -y \ python-software-properties \ software-properties-common \ wget \ unzip \ git \ mercurial \ make \ ruby \ ruby-dev \ rpm \ zip \ python \ python-boto \ build-essential \ autoconf \ automake \ libtool \ python-setuptools \ curl RUN gem install fpm # Install protobuf3 protoc binary ENV PROTO_VERSION 3.0.0 RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protoc-${PROTO_VERSION}-linux-x86_64.zip \ && unzip -j protoc-${PROTO_VERSION}-linux-x86_64.zip bin/protoc -d /bin \ rm protoc-${PROTO_VERSION}-linux-x86_64.zip # Install protobuf3 python library RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protobuf-python-${PROTO_VERSION}.tar.gz \ && tar -xf protobuf-python-${PROTO_VERSION}.tar.gz \ && cd /protobuf-${PROTO_VERSION}/python \ && python setup.py install \ && rm -rf /protobuf-${PROTO_VERSION} protobuf-python-${PROTO_VERSION}.tar.gz # Install go ENV GOPATH /root/go ENV GO_VERSION 1.7.5 ENV GO_ARCH amd64 RUN wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \ tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \ rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ENV PATH /usr/local/go/bin:$PATH ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor ENV PATH $GOPATH/bin:$PATH RUN mkdir -p $PROJECT_DIR WORKDIR $PROJECT_DIR VOLUME $PROJECT_DIR VOLUME /root/go/src # Configure local git RUN git config --global user.email "support@influxdb.com" RUN git config --global user.Name "Docker Builder" ENTRYPOINT [ "/root/go/src/github.com/influxdata/kapacitor/build.py" ] ================================================ FILE: vendor/github.com/influxdata/kapacitor/Gopkg.toml ================================================ required = ["github.com/benbjohnson/tmpl","github.com/golang/protobuf/protoc-gen-go"] [[constraint]] branch = "master" name = "github.com/davecgh/go-spew" [[constraint]] branch = "master" name = "github.com/evanphx/json-patch" [[constraint]] branch = "master" name = "github.com/ghodss/yaml" [[constraint]] branch = "master" name = "github.com/google/uuid" [[constraint]] name = "github.com/influxdata/influxdb" version = "~1.1.0" [[constraint]] branch = "master" name = "github.com/mitchellh/mapstructure" [[constraint]] branch = "logger-targetmanager-wait" name = "github.com/prometheus/prometheus" source = "github.com/goller/prometheus" [[constraint]] branch = "master" name = "github.com/shurcooL/markdownfmt" [[constraint]] name = "github.com/eclipse/paho.mqtt.golang" version = "~1.0.0" # Force the Azure projects to be a specific older version that Prometheus needs [[override]] name = "github.com/Azure/azure-sdk-for-go" revision = "bd73d950fa4440dae889bd9917bff7cef539f86e" [[override]] name = "github.com/Azure/go-autorest" revision = "a2fdd780c9a50455cecd249b00bdc3eb73a78e31" ================================================ FILE: vendor/github.com/influxdata/kapacitor/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2015 InfluxDB Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/influxdata/kapacitor/LICENSE_OF_DEPENDENCIES.md ================================================ Dependencies ============ * github.com/BurntSushi/toml [WTFPL](https://github.com/BurntSushi/toml/blob/master/COPYING) * github.com/boltdb/bolt [MIT](https://github.com/boltdb/bolt/blob/master/LICENSE) * github.com/cenkalti/backoff [MIT](https://github.com/cenkalti/backoff/blob/master/LICENSE) * github.com/dgrijalva/jwt-go [MIT](https://github.com/dgrijalva/jwt-go/blob/master/LICENSE) * github.com/dustin/go-humanize [MIT](https://github.com/dustin/go-humanize/blob/master/LICENSE) * github.com/golang/protobuf [BSD](https://github.com/golang/protobuf/blob/master/LICENSE) * github.com/google/uuid [BSD](https://github.com/google/uuid/blob/master/LICENSE) * github.com/gorhill/cronexpr [APLv2](https://github.com/gorhill/cronexpr/blob/master/APLv2) * github.com/k-sone/snmpgo [MIT](https://github.com/k-sone/snmpgo/blob/master/LICENSE) * github.com/kimor79/gollectd [BSD](https://github.com/kimor79/gollectd/blob/master/LICENSE) * github.com/mattn/go-runewidth [MIT](https://github.com/mattn/go-runewidth/blob/master/README.mkd) * github.com/mitchellh/copystructure[MIT](https://github.com/mitchellh/copystructure/blob/master/LICENSE) * github.com/mitchellh/reflectwalk [MIT](https://github.com/mitchellh/reflectwalk/blob/master/LICENSE) * github.com/pkg/errors [BSD](https://github.com/pkg/errors/blob/master/LICENSE) * github.com/russross/blackfriday [BSD](https://github.com/russross/blackfriday/blob/master/LICENSE.txt) * github.com/serenize/snaker [MIT](https://github.com/serenize/snaker/blob/master/LICENSE.txt) * github.com/shurcooL/go [MIT](https://github.com/shurcooL/go/blob/master/README.md) * github.com/shurcooL/markdownfmt [MIT](https://github.com/shurcooL/markdownfmt/blob/master/README.md) * github.com/shurcooL/sanitized\_anchor\_name [MIT](https://github.com/shurcooL/sanitized_anchor_name/blob/master/LICENSE) * github.com/stretchr/testify [MIT](https://github.com/stretchr/testify/blob/master/LICENSE) * gopkg.in/gomail.v2 [MIT](https://github.com/go-gomail/gomail/blob/v2/LICENSE) ================================================ FILE: vendor/github.com/influxdata/kapacitor/README.md ================================================ # Kapacitor [![Circle CI](https://circleci.com/gh/influxdata/kapacitor/tree/master.svg?style=svg&circle-token=78c97422cf89526309e502a290c230e8a463229f)](https://circleci.com/gh/influxdata/kapacitor/tree/master) [![Docker pulls](https://img.shields.io/docker/pulls/library/kapacitor.svg)](https://hub.docker.com/_/kapacitor/) Open source framework for processing, monitoring, and alerting on time series data # Installation Kapacitor has two binaries: * kapacitor – a CLI program for calling the Kapacitor API. * kapacitord – the Kapacitor server daemon. You can either download the binaries directly from the [downloads](https://influxdata.com/downloads/#kapacitor) page or go get them: ```sh go get github.com/influxdata/kapacitor/cmd/kapacitor go get github.com/influxdata/kapacitor/cmd/kapacitord ``` # Configuration An example configuration file can be found [here](https://github.com/influxdata/kapacitor/blob/master/etc/kapacitor/kapacitor.conf) Kapacitor can also provide an example config for you using this command: ```sh kapacitord config ``` # Getting Started This README gives you a high level overview of what Kapacitor is and what its like to use it. As well as some details of how it works. To get started using Kapacitor see [this guide](https://docs.influxdata.com/kapacitor/latest/introduction/getting_started/). After you finish the getting started exercise you can check out the [TICKscripts](https://github.com/influxdata/kapacitor/tree/master/examples/telegraf) for different Telegraf plugins. # Basic Example Kapacitor use a DSL named [TICKscript](https://docs.influxdata.com/kapacitor/latest/tick/) to define tasks. A simple TICKscript that alerts on high cpu usage looks like this: ```javascript stream |from() .measurement('cpu_usage_idle') .groupBy('host') |window() .period(1m) .every(1m) |mean('value') |eval(lambda: 100.0 - "mean") .as('used') |alert() .message('{{ .Level}}: {{ .Name }}/{{ index .Tags "host" }} has high cpu usage: {{ index .Fields "used" }}') .warn(lambda: "used" > 70.0) .crit(lambda: "used" > 85.0) // Send alert to hander of choice. // Slack .slack() .channel('#alerts') // VictorOps .victorOps() .routingKey('team_rocket') // PagerDuty .pagerDuty() ``` Place the above script into a file `cpu_alert.tick` then run these commands to start the task: ```sh # Define the task (assumes cpu data is in db 'telegraf') kapacitor define \ cpu_alert \ -type stream \ -dbrp telegraf.default \ -tick ./cpu_alert.tick # Start the task kapacitor enable cpu_alert ``` For more complete examples see the [documentation](https://docs.influxdata.com/kapacitor/latest/examples/). ================================================ FILE: vendor/github.com/influxdata/kapacitor/alert.go ================================================ package kapacitor import ( "bytes" "encoding/json" "fmt" html "html/template" "log" "os" "sync" text "text/template" "time" "github.com/influxdata/kapacitor/alert" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" alertservice "github.com/influxdata/kapacitor/services/alert" "github.com/influxdata/kapacitor/services/hipchat" "github.com/influxdata/kapacitor/services/httppost" "github.com/influxdata/kapacitor/services/mqtt" "github.com/influxdata/kapacitor/services/opsgenie" "github.com/influxdata/kapacitor/services/pagerduty" "github.com/influxdata/kapacitor/services/pushover" "github.com/influxdata/kapacitor/services/sensu" "github.com/influxdata/kapacitor/services/slack" "github.com/influxdata/kapacitor/services/smtp" "github.com/influxdata/kapacitor/services/snmptrap" "github.com/influxdata/kapacitor/services/telegram" "github.com/influxdata/kapacitor/services/victorops" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" "github.com/pkg/errors" ) const ( statsAlertsTriggered = "alerts_triggered" statsOKsTriggered = "oks_triggered" statsInfosTriggered = "infos_triggered" statsWarnsTriggered = "warns_triggered" statsCritsTriggered = "crits_triggered" statsEventsDropped = "events_dropped" ) // The newest state change is weighted 'weightDiff' times more than oldest state change. const weightDiff = 1.5 // Maximum weight applied to newest state change. const maxWeight = 1.2 type AlertNode struct { node a *pipeline.AlertNode topic string anonTopic string handlers []alert.Handler levels []stateful.Expression scopePools []stateful.ScopePool idTmpl *text.Template messageTmpl *text.Template detailsTmpl *html.Template alertsTriggered *expvar.Int oksTriggered *expvar.Int infosTriggered *expvar.Int warnsTriggered *expvar.Int critsTriggered *expvar.Int eventsDropped *expvar.Int bufPool sync.Pool levelResets []stateful.Expression lrScopePools []stateful.ScopePool } // Create a new AlertNode which caches the most recent item and exposes it over the HTTP API. func newAlertNode(et *ExecutingTask, n *pipeline.AlertNode, l *log.Logger) (an *AlertNode, err error) { an = &AlertNode{ node: node{Node: n, et: et, logger: l}, a: n, } an.node.runF = an.runAlert an.topic = n.Topic // Create anonymous topic name an.anonTopic = fmt.Sprintf("%s:%s:%s", et.tm.ID(), et.Task.ID, an.Name()) // Create buffer pool for the templates an.bufPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } // Parse templates an.idTmpl, err = text.New("id").Parse(n.Id) if err != nil { return nil, err } an.messageTmpl, err = text.New("message").Parse(n.Message) if err != nil { return nil, err } an.detailsTmpl, err = html.New("details").Funcs(html.FuncMap{ "json": func(v interface{}) html.JS { tmpBuffer := an.bufPool.Get().(*bytes.Buffer) defer func() { tmpBuffer.Reset() an.bufPool.Put(tmpBuffer) }() _ = json.NewEncoder(tmpBuffer).Encode(v) return html.JS(tmpBuffer.String()) }, }).Parse(n.Details) if err != nil { return nil, err } for _, tcp := range n.TcpHandlers { c := alertservice.TCPHandlerConfig{ Address: tcp.Address, } h := alertservice.NewTCPHandler(c, l) an.handlers = append(an.handlers, h) } for _, email := range n.EmailHandlers { c := smtp.HandlerConfig{ To: email.ToList, } h := et.tm.SMTPService.Handler(c, l) an.handlers = append(an.handlers, h) } if len(n.EmailHandlers) == 0 && (et.tm.SMTPService != nil && et.tm.SMTPService.Global()) { c := smtp.HandlerConfig{} h := et.tm.SMTPService.Handler(c, l) an.handlers = append(an.handlers, h) } // If email has been configured with state changes only set it. if et.tm.SMTPService != nil && et.tm.SMTPService.Global() && et.tm.SMTPService.StateChangesOnly() { n.IsStateChangesOnly = true } for _, e := range n.ExecHandlers { c := alertservice.ExecHandlerConfig{ Prog: e.Command[0], Args: e.Command[1:], Commander: et.tm.Commander, } h := alertservice.NewExecHandler(c, l) an.handlers = append(an.handlers, h) } for _, log := range n.LogHandlers { c := alertservice.DefaultLogHandlerConfig() c.Path = log.FilePath if log.Mode != 0 { c.Mode = os.FileMode(log.Mode) } h, err := alertservice.NewLogHandler(c, l) if err != nil { return nil, errors.Wrap(err, "failed to create log alert handler") } an.handlers = append(an.handlers, h) } for _, vo := range n.VictorOpsHandlers { c := victorops.HandlerConfig{ RoutingKey: vo.RoutingKey, } h := et.tm.VictorOpsService.Handler(c, l) an.handlers = append(an.handlers, h) } if len(n.VictorOpsHandlers) == 0 && (et.tm.VictorOpsService != nil && et.tm.VictorOpsService.Global()) { c := victorops.HandlerConfig{} h := et.tm.VictorOpsService.Handler(c, l) an.handlers = append(an.handlers, h) } for _, pd := range n.PagerDutyHandlers { c := pagerduty.HandlerConfig{ ServiceKey: pd.ServiceKey, } h := et.tm.PagerDutyService.Handler(c, l) an.handlers = append(an.handlers, h) } if len(n.PagerDutyHandlers) == 0 && (et.tm.PagerDutyService != nil && et.tm.PagerDutyService.Global()) { c := pagerduty.HandlerConfig{} h := et.tm.PagerDutyService.Handler(c, l) an.handlers = append(an.handlers, h) } for _, s := range n.SensuHandlers { c := sensu.HandlerConfig{ Source: s.Source, Handlers: s.HandlersList, } h, err := et.tm.SensuService.Handler(c, l) if err != nil { return nil, errors.Wrap(err, "failed to create sensu alert handler") } an.handlers = append(an.handlers, h) } for _, s := range n.SlackHandlers { c := slack.HandlerConfig{ Channel: s.Channel, Username: s.Username, IconEmoji: s.IconEmoji, } h := et.tm.SlackService.Handler(c, l) an.handlers = append(an.handlers, h) } if len(n.SlackHandlers) == 0 && (et.tm.SlackService != nil && et.tm.SlackService.Global()) { h := et.tm.SlackService.Handler(slack.HandlerConfig{}, l) an.handlers = append(an.handlers, h) } // If slack has been configured with state changes only set it. if et.tm.SlackService != nil && et.tm.SlackService.Global() && et.tm.SlackService.StateChangesOnly() { n.IsStateChangesOnly = true } for _, t := range n.TelegramHandlers { c := telegram.HandlerConfig{ ChatId: t.ChatId, ParseMode: t.ParseMode, DisableWebPagePreview: t.IsDisableWebPagePreview, DisableNotification: t.IsDisableNotification, } h := et.tm.TelegramService.Handler(c, l) an.handlers = append(an.handlers, h) } for _, s := range n.SNMPTrapHandlers { dataList := make([]snmptrap.Data, len(s.DataList)) for i, d := range s.DataList { dataList[i] = snmptrap.Data{ Oid: d.Oid, Type: d.Type, Value: d.Value, } } c := snmptrap.HandlerConfig{ TrapOid: s.TrapOid, DataList: dataList, } h, err := et.tm.SNMPTrapService.Handler(c, l) if err != nil { return nil, errors.Wrapf(err, "failed to create SNMP handler") } an.handlers = append(an.handlers, h) } if len(n.TelegramHandlers) == 0 && (et.tm.TelegramService != nil && et.tm.TelegramService.Global()) { c := telegram.HandlerConfig{} h := et.tm.TelegramService.Handler(c, l) an.handlers = append(an.handlers, h) } // If telegram has been configured with state changes only set it. if et.tm.TelegramService != nil && et.tm.TelegramService.Global() && et.tm.TelegramService.StateChangesOnly() { n.IsStateChangesOnly = true } for _, hc := range n.HipChatHandlers { c := hipchat.HandlerConfig{ Room: hc.Room, Token: hc.Token, } h := et.tm.HipChatService.Handler(c, l) an.handlers = append(an.handlers, h) } if len(n.HipChatHandlers) == 0 && (et.tm.HipChatService != nil && et.tm.HipChatService.Global()) { c := hipchat.HandlerConfig{} h := et.tm.HipChatService.Handler(c, l) an.handlers = append(an.handlers, h) } // If HipChat has been configured with state changes only set it. if et.tm.HipChatService != nil && et.tm.HipChatService.Global() && et.tm.HipChatService.StateChangesOnly() { n.IsStateChangesOnly = true } for _, a := range n.AlertaHandlers { c := et.tm.AlertaService.DefaultHandlerConfig() if a.Token != "" { c.Token = a.Token } if a.Resource != "" { c.Resource = a.Resource } if a.Event != "" { c.Event = a.Event } if a.Environment != "" { c.Environment = a.Environment } if a.Group != "" { c.Group = a.Group } if a.Value != "" { c.Value = a.Value } if a.Origin != "" { c.Origin = a.Origin } if len(a.Service) != 0 { c.Service = a.Service } h, err := et.tm.AlertaService.Handler(c, l) if err != nil { return nil, errors.Wrap(err, "failed to create Alerta handler") } an.handlers = append(an.handlers, h) } for _, p := range n.PushoverHandlers { c := pushover.HandlerConfig{} if p.Device != "" { c.Device = p.Device } if p.Title != "" { c.Title = p.Title } if p.URL != "" { c.URL = p.URL } if p.URLTitle != "" { c.URLTitle = p.URLTitle } if p.Sound != "" { c.Sound = p.Sound } h := et.tm.PushoverService.Handler(c, l) an.handlers = append(an.handlers, h) } for _, p := range n.HTTPPostHandlers { c := httppost.HandlerConfig{ URL: p.URL, Endpoint: p.Endpoint, Headers: p.Headers, } h := et.tm.HTTPPostService.Handler(c, l) an.handlers = append(an.handlers, h) } for _, og := range n.OpsGenieHandlers { c := opsgenie.HandlerConfig{ TeamsList: og.TeamsList, RecipientsList: og.RecipientsList, } h := et.tm.OpsGenieService.Handler(c, l) an.handlers = append(an.handlers, h) } if len(n.OpsGenieHandlers) == 0 && (et.tm.OpsGenieService != nil && et.tm.OpsGenieService.Global()) { c := opsgenie.HandlerConfig{} h := et.tm.OpsGenieService.Handler(c, l) an.handlers = append(an.handlers, h) } for range n.TalkHandlers { h := et.tm.TalkService.Handler(l) an.handlers = append(an.handlers, h) } for _, m := range n.MQTTHandlers { c := mqtt.HandlerConfig{ BrokerName: m.BrokerName, Topic: m.Topic, QoS: mqtt.QoSLevel(m.Qos), Retained: m.Retained, } h := et.tm.MQTTService.Handler(c, l) an.handlers = append(an.handlers, h) } // Parse level expressions an.levels = make([]stateful.Expression, alert.Critical+1) an.scopePools = make([]stateful.ScopePool, alert.Critical+1) an.levelResets = make([]stateful.Expression, alert.Critical+1) an.lrScopePools = make([]stateful.ScopePool, alert.Critical+1) if n.Info != nil { statefulExpression, expressionCompileError := stateful.NewExpression(n.Info.Expression) if expressionCompileError != nil { return nil, fmt.Errorf("Failed to compile stateful expression for info: %s", expressionCompileError) } an.levels[alert.Info] = statefulExpression an.scopePools[alert.Info] = stateful.NewScopePool(ast.FindReferenceVariables(n.Info.Expression)) if n.InfoReset != nil { lstatefulExpression, lexpressionCompileError := stateful.NewExpression(n.InfoReset.Expression) if lexpressionCompileError != nil { return nil, fmt.Errorf("Failed to compile stateful expression for infoReset: %s", lexpressionCompileError) } an.levelResets[alert.Info] = lstatefulExpression an.lrScopePools[alert.Info] = stateful.NewScopePool(ast.FindReferenceVariables(n.InfoReset.Expression)) } } if n.Warn != nil { statefulExpression, expressionCompileError := stateful.NewExpression(n.Warn.Expression) if expressionCompileError != nil { return nil, fmt.Errorf("Failed to compile stateful expression for warn: %s", expressionCompileError) } an.levels[alert.Warning] = statefulExpression an.scopePools[alert.Warning] = stateful.NewScopePool(ast.FindReferenceVariables(n.Warn.Expression)) if n.WarnReset != nil { lstatefulExpression, lexpressionCompileError := stateful.NewExpression(n.WarnReset.Expression) if lexpressionCompileError != nil { return nil, fmt.Errorf("Failed to compile stateful expression for warnReset: %s", lexpressionCompileError) } an.levelResets[alert.Warning] = lstatefulExpression an.lrScopePools[alert.Warning] = stateful.NewScopePool(ast.FindReferenceVariables(n.WarnReset.Expression)) } } if n.Crit != nil { statefulExpression, expressionCompileError := stateful.NewExpression(n.Crit.Expression) if expressionCompileError != nil { return nil, fmt.Errorf("Failed to compile stateful expression for crit: %s", expressionCompileError) } an.levels[alert.Critical] = statefulExpression an.scopePools[alert.Critical] = stateful.NewScopePool(ast.FindReferenceVariables(n.Crit.Expression)) if n.CritReset != nil { lstatefulExpression, lexpressionCompileError := stateful.NewExpression(n.CritReset.Expression) if lexpressionCompileError != nil { return nil, fmt.Errorf("Failed to compile stateful expression for critReset: %s", lexpressionCompileError) } an.levelResets[alert.Critical] = lstatefulExpression an.lrScopePools[alert.Critical] = stateful.NewScopePool(ast.FindReferenceVariables(n.CritReset.Expression)) } } // Setup states if n.History < 2 { n.History = 2 } // Configure flapping if n.UseFlapping { if n.FlapLow > 1 || n.FlapHigh > 1 { return nil, errors.New("alert flap thresholds are percentages and should be between 0 and 1") } } return } func (n *AlertNode) runAlert([]byte) error { // Register delete hook if n.hasAnonTopic() { n.et.tm.registerDeleteHookForTask(n.et.Task.ID, deleteAlertHook(n.anonTopic)) // Register Handlers on topic for _, h := range n.handlers { n.et.tm.AlertService.RegisterAnonHandler(n.anonTopic, h) } // Restore anonTopic n.et.tm.AlertService.RestoreTopic(n.anonTopic) } // Setup stats n.alertsTriggered = &expvar.Int{} n.statMap.Set(statsAlertsTriggered, n.alertsTriggered) n.oksTriggered = &expvar.Int{} n.statMap.Set(statsOKsTriggered, n.oksTriggered) n.infosTriggered = &expvar.Int{} n.statMap.Set(statsInfosTriggered, n.infosTriggered) n.warnsTriggered = &expvar.Int{} n.statMap.Set(statsWarnsTriggered, n.warnsTriggered) n.critsTriggered = &expvar.Int{} n.statMap.Set(statsCritsTriggered, n.critsTriggered) n.eventsDropped = &expvar.Int{} n.statMap.Set(statsCritsTriggered, n.critsTriggered) // Setup consumer consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) if err := consumer.Consume(); err != nil { return err } // Close the anonymous topic. n.et.tm.AlertService.CloseTopic(n.anonTopic) // Deregister Handlers on topic for _, h := range n.handlers { n.et.tm.AlertService.DeregisterAnonHandler(n.anonTopic, h) } return nil } func (n *AlertNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { id, err := n.renderID(first.Name(), first.GroupID(), first.Tags()) if err != nil { return nil, err } t := first.Time() state := n.restoreEventState(id, t) return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver( n.timer, state, ), ), nil } func (n *AlertNode) restoreEventState(id string, t time.Time) *alertState { state := n.newAlertState() currentLevel, triggered := n.restoreEvent(id) if currentLevel != alert.OK { // Add initial event state.addEvent(t, currentLevel) // Record triggered time state.triggered(triggered) } return state } func (n *AlertNode) newAlertState() *alertState { return &alertState{ history: make([]alert.Level, n.a.History), n: n, buffer: new(edge.BatchBuffer), } } func (n *AlertNode) restoreEvent(id string) (alert.Level, time.Time) { var topicState, anonTopicState alert.EventState var anonFound, topicFound bool // Check for previous state on anonTopic if n.hasAnonTopic() { if state, ok, err := n.et.tm.AlertService.EventState(n.anonTopic, id); err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to get event state for anonymous topic %s, event %s: %v", n.anonTopic, id, err) } else if ok { anonTopicState = state anonFound = true } } // Check for previous state on topic. if n.hasTopic() { if state, ok, err := n.et.tm.AlertService.EventState(n.topic, id); err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to get event state for topic %s, event %s: %v", n.topic, id, err) } else if ok { topicState = state topicFound = true } } if topicState.Level != anonTopicState.Level { if anonFound && topicFound { // Anon topic takes precedence if err := n.et.tm.AlertService.UpdateEvent(n.topic, anonTopicState); err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to update topic %q event state for event %q", n.topic, id) } } else if topicFound && n.hasAnonTopic() { // Update event state for topic if err := n.et.tm.AlertService.UpdateEvent(n.anonTopic, topicState); err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to update topic %q event state for event %q", n.topic, id) } } // else nothing was found, nothing to do } if anonFound { return anonTopicState.Level, anonTopicState.Time } return topicState.Level, topicState.Time } func deleteAlertHook(anonTopic string) deleteHook { return func(tm *TaskMaster) { tm.AlertService.DeleteTopic(anonTopic) } } func (n *AlertNode) hasAnonTopic() bool { return len(n.handlers) > 0 } func (n *AlertNode) hasTopic() bool { return n.topic != "" } func (n *AlertNode) handleEvent(event alert.Event) { n.alertsTriggered.Add(1) switch event.State.Level { case alert.OK: n.oksTriggered.Add(1) case alert.Info: n.infosTriggered.Add(1) case alert.Warning: n.warnsTriggered.Add(1) case alert.Critical: n.critsTriggered.Add(1) } n.logger.Printf("D! %v alert triggered id:%s msg:%s data:%v", event.State.Level, event.State.ID, event.State.Message, event.Data.Result.Series[0]) // If we have anon handlers, emit event to the anonTopic if n.hasAnonTopic() { event.Topic = n.anonTopic err := n.et.tm.AlertService.Collect(event) if err != nil { n.eventsDropped.Add(1) n.incrementErrorCount() n.logger.Println("E!", err) } } // If we have a user define topic, emit event to the topic. if n.hasTopic() { event.Topic = n.topic err := n.et.tm.AlertService.Collect(event) if err != nil { n.eventsDropped.Add(1) n.incrementErrorCount() n.logger.Println("E!", err) } } } func (n *AlertNode) determineLevel(p edge.FieldsTagsTimeGetter, currentLevel alert.Level) alert.Level { if higherLevel, found := n.findFirstMatchLevel(alert.Critical, currentLevel-1, p); found { return higherLevel } if rse := n.levelResets[currentLevel]; rse != nil { if pass, err := EvalPredicate(rse, n.lrScopePools[currentLevel], p); err != nil { n.incrementErrorCount() n.logger.Printf("E! error evaluating reset expression for current level %v: %s", currentLevel, err) } else if !pass { return currentLevel } } if newLevel, found := n.findFirstMatchLevel(currentLevel, alert.OK, p); found { return newLevel } return alert.OK } func (n *AlertNode) findFirstMatchLevel(start alert.Level, stop alert.Level, p edge.FieldsTagsTimeGetter) (alert.Level, bool) { if stop < alert.OK { stop = alert.OK } for l := start; l > stop; l-- { se := n.levels[l] if se == nil { continue } if pass, err := EvalPredicate(se, n.scopePools[l], p); err != nil { n.incrementErrorCount() n.logger.Printf("E! error evaluating expression for level %v: %s", alert.Level(l), err) continue } else if pass { return alert.Level(l), true } } return alert.OK, false } func (n *AlertNode) event( id, name string, group models.GroupID, tags models.Tags, fields models.Fields, level alert.Level, t time.Time, d time.Duration, result models.Result, ) (alert.Event, error) { msg, details, err := n.renderMessageAndDetails(id, name, t, group, tags, fields, level) if err != nil { return alert.Event{}, err } event := alert.Event{ Topic: n.anonTopic, State: alert.EventState{ ID: id, Message: msg, Details: details, Time: t, Duration: d, Level: level, }, Data: alert.EventData{ Name: name, TaskName: n.et.Task.ID, Group: string(group), Tags: tags, Fields: fields, Result: result, }, } return event, nil } type alertState struct { n *AlertNode buffer *edge.BatchBuffer history []alert.Level idx int flapping bool changed bool // Time when first alert was triggered firstTriggered time.Time // Time when last alert was triggered. // Note: Alerts are not triggered for every event. lastTriggered time.Time expired bool } func (a *alertState) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { return nil, a.buffer.BeginBatch(begin) } func (a *alertState) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { return nil, a.buffer.BatchPoint(bp) } func (a *alertState) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return a.BufferedBatch(a.buffer.BufferedBatchMessage(end)) } func (a *alertState) BufferedBatch(b edge.BufferedBatchMessage) (edge.Message, error) { begin := b.Begin() id, err := a.n.renderID(begin.Name(), begin.GroupID(), begin.Tags()) if err != nil { return nil, err } if len(b.Points()) == 0 { return nil, nil } // Keep track of lowest level for any point lowestLevel := alert.Critical // Keep track of highest level and point highestLevel := alert.OK var highestPoint edge.BatchPointMessage currentLevel := a.currentLevel() for _, bp := range b.Points() { l := a.n.determineLevel(bp, currentLevel) if l < lowestLevel { lowestLevel = l } if l > highestLevel || highestPoint == nil { highestLevel = l highestPoint = bp } } // Default the determined level to lowest. l := lowestLevel // Update determined level to highest if we don't care about all if !a.n.a.AllFlag { l = highestLevel } // Create alert Data t := highestPoint.Time() if a.n.a.AllFlag || l == alert.OK { t = begin.Time() } a.addEvent(t, l) // Trigger alert only if: // l == OK and state.changed (aka recovery) // OR // l != OK and flapping/statechanges checkout if !(a.changed && l == alert.OK || (l != alert.OK && !((a.n.a.UseFlapping && a.flapping) || (a.n.a.IsStateChangesOnly && !a.changed && !a.expired)))) { return nil, nil } a.triggered(t) // Suppress the recovery event. if a.n.a.NoRecoveriesFlag && l == alert.OK { return nil, nil } duration := a.duration() event, err := a.n.event(id, begin.Name(), begin.GroupID(), begin.Tags(), highestPoint.Fields(), l, t, duration, b.ToResult()) if err != nil { return nil, err } a.n.handleEvent(event) // Update tags or fields with event state if a.n.a.LevelTag != "" || a.n.a.LevelField != "" || a.n.a.IdTag != "" || a.n.a.IdField != "" || a.n.a.DurationField != "" || a.n.a.MessageField != "" { b = b.ShallowCopy() points := make([]edge.BatchPointMessage, len(b.Points())) for i, bp := range b.Points() { bp = bp.ShallowCopy() a.augmentTagsWithEventState(bp, event.State) a.augmentFieldsWithEventState(bp, event.State) points[i] = bp } b.SetPoints(points) newBegin := begin.ShallowCopy() a.augmentTagsWithEventState(newBegin, event.State) b.SetBegin(newBegin) } return b, nil } func (a *alertState) Point(p edge.PointMessage) (edge.Message, error) { id, err := a.n.renderID(p.Name(), p.GroupID(), p.Tags()) if err != nil { return nil, err } l := a.n.determineLevel(p, a.currentLevel()) a.addEvent(p.Time(), l) if (a.n.a.UseFlapping && a.flapping) || (a.n.a.IsStateChangesOnly && !a.changed && !a.expired) { return nil, nil } // send alert if we are not OK or we are OK and state changed (i.e recovery) if l != alert.OK || a.changed { a.triggered(p.Time()) // Suppress the recovery event. if a.n.a.NoRecoveriesFlag && l == alert.OK { return nil, nil } // Create an alert event duration := a.duration() event, err := a.n.event( id, p.Name(), p.GroupID(), p.Tags(), p.Fields(), l, p.Time(), duration, p.ToResult(), ) if err != nil { return nil, err } a.n.handleEvent(event) // Prepare an augmented point to return p = p.ShallowCopy() a.augmentTagsWithEventState(p, event.State) a.augmentFieldsWithEventState(p, event.State) return p, nil } return nil, nil } func (a *alertState) augmentTagsWithEventState(p edge.TagSetter, eventState alert.EventState) { if a.n.a.LevelTag != "" || a.n.a.IdTag != "" { tags := p.Tags().Copy() if a.n.a.LevelTag != "" { tags[a.n.a.LevelTag] = eventState.Level.String() } if a.n.a.IdTag != "" { tags[a.n.a.IdTag] = eventState.ID } p.SetTags(tags) } } func (a *alertState) augmentFieldsWithEventState(p edge.FieldSetter, eventState alert.EventState) { if a.n.a.LevelField != "" || a.n.a.IdField != "" || a.n.a.DurationField != "" || a.n.a.MessageField != "" { fields := p.Fields().Copy() if a.n.a.LevelField != "" { fields[a.n.a.LevelField] = eventState.Level.String() } if a.n.a.MessageField != "" { fields[a.n.a.MessageField] = eventState.Message } if a.n.a.IdField != "" { fields[a.n.a.IdField] = eventState.ID } if a.n.a.DurationField != "" { fields[a.n.a.DurationField] = int64(eventState.Duration) } p.SetFields(fields) } } func (a *alertState) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (a *alertState) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } // Return the duration of the current alert state. func (a *alertState) duration() time.Duration { return a.lastTriggered.Sub(a.firstTriggered) } // Record that the alert was triggered at time t. func (a *alertState) triggered(t time.Time) { a.lastTriggered = t // Check if we are being triggered for first time since an alert.OKAlert // If so reset firstTriggered time p := a.idx - 1 if p == -1 { p = len(a.history) - 1 } if a.history[p] == alert.OK { a.firstTriggered = t } } // Record an event in the alert history. func (a *alertState) addEvent(t time.Time, level alert.Level) { // Check for changes a.changed = a.history[a.idx] != level // Add event to history a.idx = (a.idx + 1) % len(a.history) a.history[a.idx] = level a.updateFlapping() a.updateExpired(t) } // Return current level of this state func (a *alertState) currentLevel() alert.Level { return a.history[a.idx] } // Compute the percentage change in the alert history. func (a *alertState) percentChange() float64 { l := len(a.history) changes := 0.0 weight := (maxWeight / weightDiff) step := (maxWeight - weight) / float64(l-1) for i := 0; i < l-1; i++ { // get current index c := (i + a.idx) % l // get previous index p := c - 1 // check for wrap around if p < 0 { p = l - 1 } if a.history[c] != a.history[p] { changes += weight } weight += step } p := changes / float64(l-1) return p } func (a *alertState) updateFlapping() { if !a.n.a.UseFlapping { return } p := a.percentChange() if a.flapping && p < a.n.a.FlapLow { a.flapping = false } else if !a.flapping && p > a.n.a.FlapHigh { a.flapping = true } } func (a *alertState) updateExpired(t time.Time) { a.expired = !a.changed && a.n.a.StateChangesOnlyDuration != 0 && t.Sub(a.lastTriggered) >= a.n.a.StateChangesOnlyDuration } type serverInfo struct { Hostname string ClusterID string ServerID string } // Type containing information available to ID template. type idInfo struct { // Measurement name Name string // Task name TaskName string // Concatenation of all group-by tags of the form [key=value,]+. // If not groupBy is performed equal to literal 'nil' Group string // Map of tags Tags map[string]string ServerInfo serverInfo } type messageInfo struct { idInfo // The ID of the alert. ID string // Fields of alerting data point. Fields map[string]interface{} // Alert Level, one of: INFO, WARNING, CRITICAL. Level string // Time Time time.Time } type detailsInfo struct { messageInfo // The Message of the Alert Message string } func (n *AlertNode) serverInfo() serverInfo { return serverInfo{ Hostname: n.et.tm.ServerInfo.Hostname(), ClusterID: n.et.tm.ServerInfo.ClusterID().String(), ServerID: n.et.tm.ServerInfo.ServerID().String(), } } func (n *AlertNode) renderID(name string, group models.GroupID, tags models.Tags) (string, error) { g := string(group) if group == models.NilGroup { g = "nil" } info := idInfo{ Name: name, TaskName: n.et.Task.ID, Group: g, Tags: tags, ServerInfo: n.serverInfo(), } id := n.bufPool.Get().(*bytes.Buffer) defer func() { id.Reset() n.bufPool.Put(id) }() err := n.idTmpl.Execute(id, info) if err != nil { return "", err } return id.String(), nil } func (n *AlertNode) renderMessageAndDetails(id, name string, t time.Time, group models.GroupID, tags models.Tags, fields models.Fields, level alert.Level) (string, string, error) { g := string(group) if group == models.NilGroup { g = "nil" } minfo := messageInfo{ idInfo: idInfo{ Name: name, TaskName: n.et.Task.ID, Group: g, Tags: tags, ServerInfo: n.serverInfo(), }, ID: id, Fields: fields, Level: level.String(), Time: t, } // Grab a buffer for the message template and the details template tmpBuffer := n.bufPool.Get().(*bytes.Buffer) defer func() { tmpBuffer.Reset() n.bufPool.Put(tmpBuffer) }() tmpBuffer.Reset() err := n.messageTmpl.Execute(tmpBuffer, minfo) if err != nil { return "", "", err } msg := tmpBuffer.String() dinfo := detailsInfo{ messageInfo: minfo, Message: msg, } // Reuse the buffer, for the details template tmpBuffer.Reset() err = n.detailsTmpl.Execute(tmpBuffer, dinfo) if err != nil { return "", "", err } details := tmpBuffer.String() return msg, details, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/autoscale.go ================================================ package kapacitor import ( "fmt" "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" k8s "github.com/influxdata/kapacitor/services/k8s/client" swarm "github.com/influxdata/kapacitor/services/swarm/client" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" "github.com/pkg/errors" ) const ( statsAutoscaleIncreaseEventsCount = "increase_events" statsAutoscaleDecreaseEventsCount = "decrease_events" statsAutoscaleCooldownDropsCount = "cooldown_drops" ) type resourceID interface { ID() string } type autoscaler interface { ResourceIDFromTags(models.Tags) (resourceID, error) Replicas(id resourceID) (int, error) SetReplicas(id resourceID, replicas int) error SetResourceIDOnTags(id resourceID, tags models.Tags) } type resourceState struct { lastIncrease time.Time lastDecrease time.Time current int } type event struct { ID resourceID Old int New int } type AutoscaleNode struct { node a autoscaler replicasExpr stateful.Expression replicasScopePool stateful.ScopePool resourceStates map[string]resourceState increaseCount *expvar.Int decreaseCount *expvar.Int cooldownDropsCount *expvar.Int min int max int increaseCooldown time.Duration decreaseCooldown time.Duration currentField string } // Create a new AutoscaleNode which can trigger autoscale events. func newAutoscaleNode( et *ExecutingTask, l *log.Logger, n pipeline.Node, a autoscaler, min, max int, increaseCooldown, decreaseCooldown time.Duration, currentField string, replicas *ast.LambdaNode, ) (*AutoscaleNode, error) { if min < 1 { return nil, fmt.Errorf("minimum count must be >= 1, got %d", min) } // Initialize the replicas lambda expression scope pool replicasExpr, err := stateful.NewExpression(replicas.Expression) if err != nil { return nil, errors.Wrap(err, "invalid replicas expression") } replicasScopePool := stateful.NewScopePool(ast.FindReferenceVariables(replicas.Expression)) kn := &AutoscaleNode{ node: node{Node: n, et: et, logger: l}, resourceStates: make(map[string]resourceState), min: min, max: max, increaseCooldown: increaseCooldown, decreaseCooldown: decreaseCooldown, currentField: currentField, a: a, replicasExpr: replicasExpr, replicasScopePool: replicasScopePool, } kn.node.runF = kn.runAutoscale return kn, nil } func (n *AutoscaleNode) runAutoscale([]byte) error { n.increaseCount = &expvar.Int{} n.decreaseCount = &expvar.Int{} n.cooldownDropsCount = &expvar.Int{} n.statMap.Set(statsAutoscaleIncreaseEventsCount, n.increaseCount) n.statMap.Set(statsAutoscaleDecreaseEventsCount, n.decreaseCount) n.statMap.Set(statsAutoscaleCooldownDropsCount, n.cooldownDropsCount) consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *AutoscaleNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup()), ), nil } func (n *AutoscaleNode) newGroup() *autoscaleGroup { return &autoscaleGroup{ n: n, expr: n.replicasExpr.CopyReset(), } } type autoscaleGroup struct { n *AutoscaleNode expr stateful.Expression begin edge.BeginBatchMessage } func (g *autoscaleGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { g.begin = begin return nil, nil } func (g *autoscaleGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { np, err := g.n.handlePoint(g.begin.Name(), g.begin.Dimensions(), bp, g.expr) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E!", err) } return np, nil } func (g *autoscaleGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return nil, nil } func (g *autoscaleGroup) Point(p edge.PointMessage) (edge.Message, error) { np, err := g.n.handlePoint(p.Name(), p.Dimensions(), p, g.expr) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E!", err) } return np, nil } func (g *autoscaleGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *autoscaleGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (n *AutoscaleNode) handlePoint(streamName string, dims models.Dimensions, p edge.FieldsTagsTimeGetter, expr stateful.Expression) (edge.PointMessage, error) { id, err := n.a.ResourceIDFromTags(p.Tags()) if err != nil { return nil, err } state, ok := n.resourceStates[id.ID()] if !ok { // If we haven't seen this resource before, get its state replicas, err := n.a.Replicas(id) if err != nil { return nil, errors.Wrapf(err, "could not determine initial scale for %q", id) } state = resourceState{ current: replicas, } n.resourceStates[id.ID()] = state } // Eval the replicas expression newReplicas, err := n.evalExpr(state.current, expr, p) if err != nil { return nil, errors.Wrap(err, "failed to evaluate the replicas expression") } // Create the event e := event{ ID: id, Old: state.current, New: newReplicas, } // Check bounds if n.max > 0 && e.New > n.max { e.New = n.max } if e.New < n.min { e.New = n.min } // Validate something changed if e.New == e.Old { // Nothing to do return nil, nil } // Update local copy of state change := e.New - e.Old state.current = e.New // Check last change cooldown times t := p.Time() var counter *expvar.Int switch { case change > 0: if t.Before(state.lastIncrease.Add(n.increaseCooldown)) { // Still hot, nothing to do n.cooldownDropsCount.Add(1) return nil, nil } state.lastIncrease = t counter = n.increaseCount case change < 0: if t.Before(state.lastDecrease.Add(n.decreaseCooldown)) { // Still hot, nothing to do n.cooldownDropsCount.Add(1) return nil, nil } state.lastDecrease = t counter = n.decreaseCount } // We have a valid event to apply if err := n.applyEvent(e); err != nil { return nil, errors.Wrap(err, "failed to apply scaling event") } // Only save the updated state if we were successful n.resourceStates[id.ID()] = state // Count event counter.Add(1) // Create new tags for the point. // Leave room for the namespace,kind, and resource tags. newTags := make(models.Tags, len(dims.TagNames)+3) // Copy group by tags for _, d := range dims.TagNames { newTags[d] = p.Tags()[d] } n.a.SetResourceIDOnTags(id, newTags) // Create point representing the event return edge.NewPointMessage( streamName, "", "", dims, models.Fields{ "old": int64(e.Old), "new": int64(e.New), }, newTags, t, ), nil } func (n *AutoscaleNode) applyEvent(e event) error { n.logger.Printf("D! setting replicas to %d was %d for %q", e.New, e.Old, e.ID) err := n.a.SetReplicas(e.ID, e.New) return errors.Wrapf(err, "failed to set new replica count for %q", e.ID) } func (n *AutoscaleNode) evalExpr( current int, expr stateful.Expression, p edge.FieldsTagsTimeGetter, ) (int, error) { vars := n.replicasScopePool.Get() defer n.replicasScopePool.Put(vars) // Set the current replicas value on the scope if requested. if n.currentField != "" { vars.Set(n.currentField, current) } // Fill the scope with the rest of the values err := fillScope(vars, n.replicasScopePool.ReferenceVariables(), p) if err != nil { return 0, err } i, err := expr.EvalInt(vars) if err != nil { return 0, err } return int(i), err } //////////////////////////////////// // K8s implementation of Autoscaler type k8sAutoscaler struct { client k8s.Client resourceName string resourceNameTag string namespaceTag string kindTag string nameTag string kind string namespace string } func newK8sAutoscaleNode(et *ExecutingTask, n *pipeline.K8sAutoscaleNode, l *log.Logger) (*AutoscaleNode, error) { client, err := et.tm.K8sService.Client(n.Cluster) if err != nil { return nil, fmt.Errorf("cannot use the k8sAutoscale node, could not create kubernetes client: %v", err) } a := &k8sAutoscaler{ client: client, resourceName: n.ResourceName, resourceNameTag: n.ResourceNameTag, namespaceTag: n.NamespaceTag, kindTag: n.KindTag, nameTag: n.ResourceTag, kind: n.Kind, namespace: n.Namespace, } return newAutoscaleNode( et, l, n, a, int(n.Min), int(n.Max), n.IncreaseCooldown, n.DecreaseCooldown, n.CurrentField, n.Replicas, ) } type k8sResourceID struct { Namespace, Kind, Name string } func (id k8sResourceID) ID() string { return id.Name } func (id k8sResourceID) String() string { return fmt.Sprintf("%s/%s/%s", id.Namespace, id.Kind, id.Name) } func (a *k8sAutoscaler) ResourceIDFromTags(tags models.Tags) (resourceID, error) { // Get the name of the resource var name string switch { case a.resourceName != "": name = a.resourceName case a.resourceNameTag != "": t, ok := tags[a.resourceNameTag] if ok { name = t } default: return nil, errors.New("expected one of ResourceName or ResourceNameTag to be set") } if name == "" { return nil, errors.New("could not determine the name of the resource") } namespace := a.namespace if namespace == "" { namespace = k8s.NamespaceDefault } return k8sResourceID{ Namespace: namespace, Kind: a.kind, Name: name, }, nil } func (a *k8sAutoscaler) getScale(kid k8sResourceID) (*k8s.Scale, error) { scales := a.client.Scales(kid.Namespace) scale, err := scales.Get(kid.Kind, kid.Name) return scale, err } func (a *k8sAutoscaler) Replicas(id resourceID) (int, error) { kid := id.(k8sResourceID) scale, err := a.getScale(kid) if err != nil { return 0, err } return int(scale.Spec.Replicas), nil } func (a *k8sAutoscaler) SetReplicas(id resourceID, replicas int) error { kid := id.(k8sResourceID) scale, err := a.getScale(kid) if err != nil { return err } scale.Spec.Replicas = int32(replicas) scales := a.client.Scales(kid.Namespace) if err := scales.Update(kid.Kind, scale); err != nil { return err } return nil } func (a *k8sAutoscaler) SetResourceIDOnTags(id resourceID, tags models.Tags) { kid := id.(k8sResourceID) // Set namespace,kind,resource tags if a.namespaceTag != "" { tags[a.namespaceTag] = kid.Namespace } if a.kindTag != "" { tags[a.kindTag] = kid.Kind } if a.nameTag != "" { tags[a.nameTag] = kid.Name } } ///////////////////////////////////////////// // Docker Swarm implementation of Autoscaler type swarmAutoscaler struct { client swarm.Client serviceName string serviceNameTag string outputServiceNameTag string } func newSwarmAutoscaleNode(et *ExecutingTask, n *pipeline.SwarmAutoscaleNode, l *log.Logger) (*AutoscaleNode, error) { client, err := et.tm.SwarmService.Client(n.Cluster) if err != nil { return nil, fmt.Errorf("cannot use the swarmAutoscale node, could not create swarm client: %v", err) } outputServiceNameTag := n.OutputServiceNameTag if outputServiceNameTag == "" { outputServiceNameTag = n.ServiceNameTag } a := &swarmAutoscaler{ client: client, serviceName: n.ServiceName, serviceNameTag: n.ServiceNameTag, outputServiceNameTag: outputServiceNameTag, } return newAutoscaleNode( et, l, n, a, int(n.Min), int(n.Max), n.IncreaseCooldown, n.DecreaseCooldown, n.CurrentField, n.Replicas, ) } type swarmResourceID string func (id swarmResourceID) ID() string { return string(id) } func (a *swarmAutoscaler) ResourceIDFromTags(tags models.Tags) (resourceID, error) { // Get the name of the resource var name string switch { case a.serviceName != "": name = a.serviceName case a.serviceNameTag != "": t, ok := tags[a.serviceNameTag] if ok { name = t } default: return nil, errors.New("expected one of ServiceName or ServiceNameTag to be set") } if name == "" { return nil, errors.New("could not determine the name of the resource") } return swarmResourceID(name), nil } func (a *swarmAutoscaler) Replicas(id resourceID) (int, error) { sid := id.ID() service, err := a.client.Service(sid) if err != nil { return 0, errors.Wrapf(err, "failed to get swarm service for %q", id) } return int(*service.Spec.Mode.Replicated.Replicas), nil } func (a *swarmAutoscaler) SetReplicas(id resourceID, replicas int) error { sid := id.ID() service, err := a.client.Service(sid) if err != nil { return errors.Wrapf(err, "failed to get swarm service for %q", id) } *service.Spec.Mode.Replicated.Replicas = uint64(replicas) return a.client.UpdateService(service) } func (a *swarmAutoscaler) SetResourceIDOnTags(id resourceID, tags models.Tags) { if a.outputServiceNameTag != "" { tags[a.outputServiceNameTag] = id.ID() } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/batch.go ================================================ package kapacitor import ( "bytes" "fmt" "log" "sync" "time" "github.com/gorhill/cronexpr" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/influxdb" "github.com/influxdata/kapacitor/pipeline" "github.com/pkg/errors" ) const ( statsBatchesQueried = "batches_queried" statsPointsQueried = "points_queried" ) type BatchNode struct { node s *pipeline.BatchNode idx int } func newBatchNode(et *ExecutingTask, n *pipeline.BatchNode, l *log.Logger) (*BatchNode, error) { sn := &BatchNode{ node: node{Node: n, et: et, logger: l}, s: n, } return sn, nil } func (n *BatchNode) linkChild(c Node) error { // add child if n.Provides() != c.Wants() { return fmt.Errorf("cannot add child mismatched edges: %s -> %s", n.Provides(), c.Wants()) } n.children = append(n.children, c) // add parent c.addParent(n) return nil } func (n *BatchNode) addParentEdge(in edge.StatsEdge) { // Pass edges down to children n.children[n.idx].addParentEdge(in) n.idx++ } func (n *BatchNode) start([]byte) { } func (n *BatchNode) Wait() error { return nil } // Return list of databases and retention policies // the batcher will query. func (n *BatchNode) DBRPs() ([]DBRP, error) { var dbrps []DBRP for _, b := range n.children { d, err := b.(*QueryNode).DBRPs() if err != nil { return nil, err } dbrps = append(dbrps, d...) } return dbrps, nil } func (n *BatchNode) Count() int { return len(n.children) } func (n *BatchNode) Start() { for _, b := range n.children { b.(*QueryNode).Start() } } func (n *BatchNode) Abort() { for _, b := range n.children { b.(*QueryNode).Abort() } } type BatchQueries struct { Queries []*Query Cluster string GroupByMeasurement bool } func (n *BatchNode) Queries(start, stop time.Time) ([]BatchQueries, error) { queries := make([]BatchQueries, len(n.children)) for i, b := range n.children { qn := b.(*QueryNode) qs, err := qn.Queries(start, stop) if err != nil { return nil, err } queries[i] = BatchQueries{ Queries: qs, Cluster: qn.Cluster(), GroupByMeasurement: qn.GroupByMeasurement(), } } return queries, nil } // Do not add the source batch node to the dot output // since its not really an edge. func (n *BatchNode) edot(*bytes.Buffer, bool) {} func (n *BatchNode) collectedCount() (count int64) { for _, child := range n.children { count += child.collectedCount() } return } type QueryNode struct { node b *pipeline.QueryNode query *Query ticker ticker queryMu sync.Mutex queryErr chan error closing chan struct{} aborting chan struct{} batchesQueried *expvar.Int pointsQueried *expvar.Int byName bool } func newQueryNode(et *ExecutingTask, n *pipeline.QueryNode, l *log.Logger) (*QueryNode, error) { bn := &QueryNode{ node: node{Node: n, et: et, logger: l}, b: n, closing: make(chan struct{}), aborting: make(chan struct{}), byName: n.GroupByMeasurementFlag, } bn.node.runF = bn.runBatch bn.node.stopF = bn.stopBatch // Create query q, err := NewQuery(n.QueryStr) if err != nil { return nil, err } bn.query = q // Add in dimensions err = bn.query.Dimensions(n.Dimensions) if err != nil { return nil, err } // Set offset alignment if n.AlignGroupFlag { bn.query.AlignGroup() } // Set fill switch fill := n.Fill.(type) { case string: switch fill { case "null": bn.query.Fill(influxql.NullFill, nil) case "none": bn.query.Fill(influxql.NoFill, nil) case "previous": bn.query.Fill(influxql.PreviousFill, nil) case "linear": bn.query.Fill(influxql.LinearFill, nil) default: return nil, fmt.Errorf("unexpected fill option %s", fill) } case int64, float64: bn.query.Fill(influxql.NumberFill, fill) } // Determine schedule if n.Every != 0 && n.Cron != "" { return nil, errors.New("must not set both 'every' and 'cron' properties") } switch { case n.Every != 0: bn.ticker = newTimeTicker(n.Every, n.AlignFlag) case n.Cron != "": var err error bn.ticker, err = newCronTicker(n.Cron) if err != nil { return nil, err } default: return nil, errors.New("must define one of 'every' or 'cron'") } return bn, nil } func (n *QueryNode) GroupByMeasurement() bool { return n.byName } // Return list of databases and retention policies // the batcher will query. func (n *QueryNode) DBRPs() ([]DBRP, error) { return n.query.DBRPs() } func (n *QueryNode) Start() { n.queryMu.Lock() defer n.queryMu.Unlock() n.queryErr = make(chan error, 1) go func() { n.queryErr <- n.doQuery(n.ins[0]) }() } func (n *QueryNode) Abort() { close(n.aborting) } func (n *QueryNode) Cluster() string { return n.b.Cluster } func (n *QueryNode) Queries(start, stop time.Time) ([]*Query, error) { now := time.Now() if stop.IsZero() { stop = now } // Crons are sensitive to timezones. // Make sure we are using local time. current := start.Local() queries := make([]*Query, 0) for { current = n.ticker.Next(current) if current.IsZero() || current.After(stop) { break } qstop := current.Add(-1 * n.b.Offset) if qstop.After(now) { break } q, err := n.query.Clone() if err != nil { return nil, err } q.SetStartTime(qstop.Add(-1 * n.b.Period)) q.SetStopTime(qstop) queries = append(queries, q) } return queries, nil } // Query InfluxDB and collect batches on batch collector. func (n *QueryNode) doQuery(in edge.Edge) error { defer in.Close() n.batchesQueried = &expvar.Int{} n.pointsQueried = &expvar.Int{} n.statMap.Set(statsBatchesQueried, n.batchesQueried) n.statMap.Set(statsPointsQueried, n.pointsQueried) if n.et.tm.InfluxDBService == nil { return errors.New("InfluxDB not configured, cannot query InfluxDB for batch query") } con, err := n.et.tm.InfluxDBService.NewNamedClient(n.b.Cluster) if err != nil { return errors.Wrap(err, "failed to get InfluxDB client") } tickC := n.ticker.Start() for { select { case <-n.closing: return nil case <-n.aborting: return errors.New("batch doQuery aborted") case now := <-tickC: n.timer.Start() // Update times for query stop := now.Add(-1 * n.b.Offset) n.query.SetStartTime(stop.Add(-1 * n.b.Period)) n.query.SetStopTime(stop) qStr := n.query.String() n.logger.Println("D! starting next batch query:", qStr) // Execute query q := influxdb.Query{ Command: qStr, } resp, err := con.Query(q) if err != nil { n.incrementErrorCount() n.logger.Println("E!", err) n.timer.Stop() break } // Collect batches for _, res := range resp.Results { batches, err := edge.ResultToBufferedBatches(res, n.byName) if err != nil { n.incrementErrorCount() n.logger.Println("E! failed to understand query result:", err) continue } for _, bch := range batches { // Set stop time based off query bounds if bch.Begin().Time().IsZero() || !n.query.IsGroupedByTime() { bch.Begin().SetTime(stop) } n.batchesQueried.Add(1) n.pointsQueried.Add(int64(len(bch.Points()))) n.timer.Pause() if err := in.Collect(bch); err != nil { return err } n.timer.Resume() } } n.timer.Stop() } } } func (n *QueryNode) runBatch([]byte) error { errC := make(chan error, 1) go func() { defer func() { err := recover() if err != nil { errC <- fmt.Errorf("%v", err) } }() for bt, ok := n.ins[0].Emit(); ok; bt, ok = n.ins[0].Emit() { for _, child := range n.outs { err := child.Collect(bt) if err != nil { errC <- err return } } } errC <- nil }() var queryErr error n.queryMu.Lock() if n.queryErr != nil { n.queryMu.Unlock() select { case queryErr = <-n.queryErr: case <-n.aborting: queryErr = errors.New("batch queryErr aborted") } } else { n.queryMu.Unlock() } var err error select { case err = <-errC: case <-n.aborting: err = errors.New("batch run aborted") } if queryErr != nil { return queryErr } return err } func (n *QueryNode) stopBatch() { if n.ticker != nil { n.ticker.Stop() } close(n.closing) } type ticker interface { Start() <-chan time.Time Stop() // Return the next time the ticker will tick after now. Next(now time.Time) time.Time } type timeTicker struct { every time.Duration align bool alignChan chan time.Time stopping chan struct{} ticker *time.Ticker mu sync.Mutex wg sync.WaitGroup } func newTimeTicker(every time.Duration, align bool) *timeTicker { t := &timeTicker{ align: align, every: every, } if align { t.alignChan = make(chan time.Time) t.stopping = make(chan struct{}) } return t } func (t *timeTicker) Start() <-chan time.Time { t.mu.Lock() defer t.mu.Unlock() if t.alignChan != nil { t.wg.Add(1) go func() { defer t.wg.Done() // Sleep until we are roughly aligned now := time.Now() next := now.Truncate(t.every).Add(t.every) after := time.NewTicker(next.Sub(now)) select { case <-after.C: after.Stop() case <-t.stopping: after.Stop() return } t.ticker = time.NewTicker(t.every) // Send first event since we waited for it explicitly t.alignChan <- next for { select { case <-t.stopping: return case now := <-t.ticker.C: now = now.Round(t.every) t.alignChan <- now } } }() return t.alignChan } else { t.ticker = time.NewTicker(t.every) return t.ticker.C } } func (t *timeTicker) Stop() { t.mu.Lock() defer t.mu.Unlock() if t.ticker != nil { t.ticker.Stop() } if t.alignChan != nil { close(t.stopping) } t.wg.Wait() } func (t *timeTicker) Next(now time.Time) time.Time { next := now.Add(t.every) if t.align { next = next.Round(t.every) } return next } type cronTicker struct { expr *cronexpr.Expression ticker chan time.Time closing chan struct{} wg sync.WaitGroup } func newCronTicker(cronExpr string) (*cronTicker, error) { expr, err := cronexpr.Parse(cronExpr) if err != nil { return nil, err } return &cronTicker{ expr: expr, ticker: make(chan time.Time), closing: make(chan struct{}), }, nil } func (c *cronTicker) Start() <-chan time.Time { c.wg.Add(1) go func() { defer c.wg.Done() for { now := time.Now() next := c.expr.Next(now) diff := next.Sub(now) select { case <-time.After(diff): c.ticker <- next case <-c.closing: return } } }() return c.ticker } func (c *cronTicker) Stop() { close(c.closing) c.wg.Wait() } func (c *cronTicker) Next(now time.Time) time.Time { return c.expr.Next(now) } ================================================ FILE: vendor/github.com/influxdata/kapacitor/build.py ================================================ #!/usr/bin/python2.7 -u import sys import os import subprocess import time from datetime import datetime import shutil import tempfile import hashlib import re import logging import argparse ################ #### Kapacitor Variables ################ # Enable Go vendoring os.environ["GO15VENDOREXPERIMENT"] = "1" # PACKAGING VARIABLES PACKAGE_NAME = "kapacitor" INSTALL_ROOT_DIR = "/usr/bin" LOG_DIR = "/var/log/kapacitor" DATA_DIR = "/var/lib/kapacitor" SCRIPT_DIR = "/usr/lib/kapacitor/scripts" INIT_SCRIPT = "scripts/init.sh" SYSTEMD_SCRIPT = "scripts/kapacitor.service" POSTINST_SCRIPT = "scripts/post-install.sh" POSTUNINST_SCRIPT = "scripts/post-uninstall.sh" LOGROTATE_CONFIG = "etc/logrotate.d/kapacitor" BASH_COMPLETION_SH = "usr/share/bash-completion/completions/kapacitor" DEFAULT_CONFIG = "etc/kapacitor/kapacitor.conf" PREINST_SCRIPT = None # Default AWS S3 bucket for uploads DEFAULT_BUCKET = "dl.influxdata.com/kapacitor/artifacts" # META-PACKAGE VARIABLES PACKAGE_LICENSE = "MIT" PACKAGE_URL = "github.com/influxdata/kapacitor" MAINTAINER = "support@influxdb.com" VENDOR = "InfluxData" DESCRIPTION = "Time series data processing engine" # SCRIPT START go_vet_command = "go tool vet -composites=false" prereqs = [ 'git', 'go' ] optional_prereqs = [ 'fpm', 'rpmbuild', 'gpg' ] fpm_common_args = "-f -s dir --log error \ --vendor {} \ --url {} \ --after-install {} \ --after-remove {} \ --license {} \ --maintainer {} \ --config-files {} \ --config-files {} \ --directories {} \ --description \"{}\"".format( VENDOR, PACKAGE_URL, POSTINST_SCRIPT, POSTUNINST_SCRIPT, PACKAGE_LICENSE, MAINTAINER, DEFAULT_CONFIG, LOGROTATE_CONFIG, ' --directories '.join([ LOG_DIR[1:], DATA_DIR[1:], SCRIPT_DIR[1:], os.path.dirname(SCRIPT_DIR[1:]), os.path.dirname(DEFAULT_CONFIG), ]), DESCRIPTION) targets = { 'kapacitor' : './cmd/kapacitor', 'kapacitord' : './cmd/kapacitord', 'tickfmt' : './tick/cmd/tickfmt' } supported_builds = { 'darwin': [ "amd64", "i386" ], 'linux': [ "amd64", "i386", "armhf", "arm64", "armel", "static_i386", "static_amd64" ], 'windows': [ "amd64", "i386" ] } supported_packages = { "darwin": [ "tar"], "linux": [ "deb", "rpm", "tar"], # experimental "windows": [ "zip" ] } ################ #### Kapacitor Functions ################ def print_banner(): logging.info(""" '##:::'##::::'###::::'########:::::'###:::::'######::'####:'########::'#######::'########:: ##::'##::::'## ##::: ##.... ##:::'## ##:::'##... ##:. ##::... ##..::'##.... ##: ##.... ##: ##:'##::::'##:. ##:: ##:::: ##::'##:. ##:: ##:::..::: ##::::: ##:::: ##:::: ##: ##:::: ##: #####::::'##:::. ##: ########::'##:::. ##: ##:::::::: ##::::: ##:::: ##:::: ##: ########:: ##. ##::: #########: ##.....::: #########: ##:::::::: ##::::: ##:::: ##:::: ##: ##.. ##::: ##:. ##:: ##.... ##: ##:::::::: ##.... ##: ##::: ##:: ##::::: ##:::: ##:::: ##: ##::. ##:: ##::. ##: ##:::: ##: ##:::::::: ##:::: ##:. ######::'####:::: ##::::. #######:: ##:::. ##: ..::::..::..:::::..::..:::::::::..:::::..:::......:::....:::::..::::::.......:::..:::::..:: Build Script """) def create_package_fs(build_root): """Create a filesystem structure to mimic the package filesystem. """ logging.debug("Creating a filesystem hierarchy from directory: {}".format(build_root)) # Using [1:] for the path names due to them being absolute # (will overwrite previous paths, per 'os.path.join' documentation) os.makedirs(os.path.join(build_root, INSTALL_ROOT_DIR[1:])) os.makedirs(os.path.join(build_root, LOG_DIR[1:])) os.makedirs(os.path.join(build_root, DATA_DIR[1:])) os.makedirs(os.path.join(build_root, SCRIPT_DIR[1:])) os.makedirs(os.path.join(build_root, os.path.dirname(DEFAULT_CONFIG))) os.makedirs(os.path.join(build_root, os.path.dirname(LOGROTATE_CONFIG))) os.makedirs(os.path.join(build_root, os.path.dirname(BASH_COMPLETION_SH))) def package_scripts(build_root, config_only=False): """Copy the necessary scripts and configuration files to the package filesystem. """ if config_only: logging.info("Copying configuration to build directory.") conf_name = os.path.basename(DEFAULT_CONFIG) shutil.copyfile(DEFAULT_CONFIG, os.path.join(build_root, conf_name)) os.chmod(os.path.join(build_root, conf_name), 0o644) else: logging.info("Copying scripts and configuration to build directory") shutil.copy(INIT_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1])) shutil.copy(SYSTEMD_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1])) shutil.copy(LOGROTATE_CONFIG, os.path.join(build_root, LOGROTATE_CONFIG)) shutil.copy(BASH_COMPLETION_SH, os.path.join(build_root, BASH_COMPLETION_SH)) shutil.copy(DEFAULT_CONFIG, os.path.join(build_root, DEFAULT_CONFIG)) os.chmod(os.path.join(build_root, LOGROTATE_CONFIG), 0o644) def run_generate(): """Run 'go generate' to rebuild any static assets. """ logging.info("Running generate...") run("go install ./vendor/github.com/golang/protobuf/protoc-gen-go") run("go install ./vendor/github.com/benbjohnson/tmpl") generate_cmd = ["go", "generate"] generate_cmd.extend(go_list()) p = subprocess.Popen(generate_cmd) code = p.wait() if code == 0: logging.info("Generate succeeded.") return True else: logging.error("Generate failed.") return False def go_get(): """ Retrieve build dependencies or restore pinned dependencies. """ # Nothing to do, all dependencies are vendored. return True def check_nochanges(): """ Check that there are no changes """ changes = run("git status --porcelain").strip() if len(changes) > 0: logging.error("There are un-committed changes in your local branch, --no-uncommited was given, cannot continue") logging.debug("Changes:\n{}".format(changes)) return False return True def run_tests(race, parallel, timeout, no_vet): """Run the Go test suite on binary output. """ logging.info("Starting tests...") if race: logging.info("Race is enabled.") if parallel is not None: logging.info("Using parallel: {}".format(parallel)) if timeout is not None: logging.info("Using timeout: {}".format(timeout)) out = run("go fmt {}".format(' '.join(go_list()))) if len(out) > 0: logging.error("Code not formatted. Please use 'go fmt ./...' to fix formatting errors.") logging.error("{}".format(out)) return False if not no_vet: vet_cmd = go_vet_command + " {}".format(" ".join(go_list(relative=True))) out = run(vet_cmd) if len(out) > 0: logging.error("Go vet failed. Please run '{}' and fix any errors.".format(vet_cmd)) logging.error("{}".format(out)) return False else: logging.info("Skipping 'go vet' call...") test_command = "go test -v" if race: test_command += " -race" if parallel is not None: test_command += " -parallel {}".format(parallel) if timeout is not None: test_command += " -timeout {}".format(timeout) test_command += " {}".format(' '.join(go_list())) logging.info("Running tests...") output = run(test_command, printOutput=logging.getLogger().getEffectiveLevel() == logging.DEBUG) return True def package_udfs(version, dist_dir): """ Create packages for UDF agents """ logging.info("Packaging UDF agents") packages = package_python_udf(version, dist_dir) return packages def package_python_udf(version, dist_dir): """ Bundle python sources for UDF agent """ logging.debug("Packaging python UDF agent") # Update python package version version_file = './udf/agent/py/kapacitor/udf/__init__.py' with open(version_file, 'w') as f: f.write('VERSION = "{}"\n'.format(version)) # Create tar of python sources fname = "python-kapacitor_udf-{}.tar.gz".format(version) outfile = os.path.join(dist_dir, fname) tar_cmd = ['tar', '-cz', '-C', './udf/agent/py', '--transform', 's/^./kapacitor_udf-{}/'.format(version), '-f'] tar_cmd.append(outfile) exclude_list = ['*.pyc', '*.pyo', '__pycache__'] for e in exclude_list: tar_cmd.append('--exclude='+e) tar_cmd.append('./') p = subprocess.Popen(tar_cmd) code = p.wait() if code != 0: logging.error("Python UDF tar failed.") sys.exit(1) # Revert version file version_file = './udf/agent/py/kapacitor/udf/__init__.py' with open(version_file, 'w') as f: f.write('VERSION = ""\n') return [outfile] ################ #### All Kapacitor-specific content above this line ################ def run(command, allow_failure=False, shell=False, printOutput=False): """ Run shell command (convenience wrapper around subprocess). If printOutput is True then the output is sent to STDOUT and not returned """ out = None logging.debug("{}".format(command)) try: cmd = command if not shell: cmd = command.split() stdout = subprocess.PIPE stderr = subprocess.STDOUT if printOutput: stdout = None p = subprocess.Popen(cmd, shell=shell, stdout=stdout, stderr=stderr) out, _ = p.communicate() if out is not None: out = out.decode('utf-8').strip() if p.returncode != 0: if allow_failure: logging.warn(u"Command '{}' failed with error: {}".format(command, out)) return None else: logging.error(u"Command '{}' failed with error: {}".format(command, out)) sys.exit(1) except OSError as e: if allow_failure: logging.warn("Command '{}' failed with error: {}".format(command, e)) return out else: logging.error("Command '{}' failed with error: {}".format(command, e)) sys.exit(1) else: return out def create_temp_dir(prefix = None): """ Create temporary directory with optional prefix. """ if prefix is None: return tempfile.mkdtemp(prefix="{}-build.".format(PACKAGE_NAME)) else: return tempfile.mkdtemp(prefix=prefix) def increment_minor_version(version): """Return the version with the minor version incremented and patch version set to zero. """ ver_list = version.split('.') if len(ver_list) != 3: logging.warn("Could not determine how to increment version '{}', will just use provided version.".format(version)) return version ver_list[1] = str(int(ver_list[1]) + 1) ver_list[2] = str(0) inc_version = '.'.join(ver_list) logging.debug("Incremented version from '{}' to '{}'.".format(version, inc_version)) return inc_version def get_current_version_tag(): """Retrieve the raw git version tag. """ version = run("git describe --always --tags --abbrev=0") return version def get_current_version(): """Parse version information from git tag output. """ version_tag = get_current_version_tag() # Remove leading 'v' if version_tag[0] == 'v': version_tag = version_tag[1:] # Replace any '-'/'_' with '~' if '-' in version_tag: version_tag = version_tag.replace("-","~") if '_' in version_tag: version_tag = version_tag.replace("_","~") return version_tag def get_current_commit(short=False): """Retrieve the current git commit. """ command = None if short: command = "git log --pretty=format:'%h' -n 1" else: command = "git rev-parse HEAD" out = run(command) return out.strip('\'\n\r ') def get_current_branch(): """Retrieve the current git branch. """ command = "git rev-parse --abbrev-ref HEAD" out = run(command) return out.strip() def local_changes(): """Return True if there are local un-committed changes. """ output = run("git diff-files --ignore-submodules --").strip() if len(output) > 0: return True return False def get_system_arch(): """Retrieve current system architecture. """ arch = os.uname()[4] if arch == "x86_64": arch = "amd64" elif arch == "386": arch = "i386" elif 'arm' in arch: # Prevent uname from reporting full ARM arch (eg 'armv7l') arch = "arm" return arch def get_system_platform(): """Retrieve current system platform. """ if sys.platform.startswith("linux"): return "linux" else: return sys.platform def get_go_version(): """Retrieve version information for Go. """ out = run("go version") matches = re.search('go version go(\S+)', out) if matches is not None: return matches.groups()[0].strip() return None def check_path_for(b): """Check the the user's path for the provided binary. """ def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') full_path = os.path.join(path, b) if os.path.isfile(full_path) and os.access(full_path, os.X_OK): return full_path def check_environ(build_dir = None): """Check environment for common Go variables. """ logging.info("Checking environment...") for v in [ "GOPATH", "GOBIN", "GOROOT" ]: logging.debug("Using '{}' for {}".format(os.environ.get(v), v)) cwd = os.getcwd() if build_dir is None and os.environ.get("GOPATH") and os.environ.get("GOPATH") not in cwd: logging.warn("Your current directory is not under your GOPATH. This may lead to build failures.") return True def check_prereqs(): """Check user path for required dependencies. """ logging.info("Checking for dependencies...") for req in prereqs: if not check_path_for(req): logging.error("Could not find dependency: {}".format(req)) return False return True def upload_packages(packages, bucket_name=None, overwrite=False): """Upload provided package output to AWS S3. """ logging.debug("Uploading files to bucket '{}': {}".format(bucket_name, packages)) try: import boto from boto.s3.key import Key from boto.s3.connection import OrdinaryCallingFormat logging.getLogger("boto").setLevel(logging.WARNING) except ImportError: logging.warn("Cannot upload packages without 'boto' Python library!") return False logging.info("Connecting to AWS S3...") # Up the number of attempts to 10 from default of 1 boto.config.add_section("Boto") boto.config.set("Boto", "metadata_service_num_attempts", "10") c = boto.connect_s3(calling_format=OrdinaryCallingFormat()) if bucket_name is None: bucket_name = DEFAULT_BUCKET bucket = c.get_bucket(bucket_name.split('/')[0]) for p in packages: if '/' in bucket_name: # Allow for nested paths within the bucket name (ex: # bucket/folder). Assuming forward-slashes as path # delimiter. name = os.path.join('/'.join(bucket_name.split('/')[1:]), os.path.basename(p)) else: name = os.path.basename(p) logging.debug("Using key: {}".format(name)) if bucket.get_key(name) is None or overwrite: logging.info("Uploading file {}".format(name)) k = Key(bucket) k.key = name if overwrite: n = k.set_contents_from_filename(p, replace=True) else: n = k.set_contents_from_filename(p, replace=False) k.make_public() else: logging.warn("Not uploading file {}, as it already exists in the target bucket.".format(name)) return True def go_list(vendor=False, relative=False): """ Return a list of packages If vendor is False vendor package are not included If relative is True the package prefix defined by PACKAGE_URL is stripped """ p = subprocess.Popen(["go", "list", "./..."], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() packages = out.split('\n') if packages[-1] == '': packages = packages[:-1] if not vendor: non_vendor = [] for p in packages: if '/vendor/' not in p: non_vendor.append(p) packages = non_vendor if relative: relative_pkgs = [] for p in packages: r = p.replace(PACKAGE_URL, '.') if r != '.': relative_pkgs.append(r) packages = relative_pkgs return packages def build(version=None, platform=None, arch=None, nightly=False, race=False, clean=False, outdir=".", tags=[], static=False): """Build each target for the specified architecture and platform. """ logging.info("Starting build for {}/{}...".format(platform, arch)) logging.info("Using Go version: {}".format(get_go_version())) logging.info("Using git branch: {}".format(get_current_branch())) logging.info("Using git commit: {}".format(get_current_commit())) if static: logging.info("Using statically-compiled output.") if race: logging.info("Race is enabled.") if len(tags) > 0: logging.info("Using build tags: {}".format(','.join(tags))) logging.info("Sending build output to: {}".format(outdir)) if not os.path.exists(outdir): os.makedirs(outdir) elif clean and outdir != '/' and outdir != ".": logging.info("Cleaning build directory '{}' before building.".format(outdir)) shutil.rmtree(outdir) os.makedirs(outdir) logging.info("Using version '{}' for build.".format(version)) tmp_build_dir = create_temp_dir() for target, path in targets.items(): logging.info("Building target: {}".format(target)) build_command = "" # Handle static binary output if static is True or "static_" in arch: if "static_" in arch: static = True arch = arch.replace("static_", "") build_command += "CGO_ENABLED=0 " # Handle variations in architecture output if arch == "i386" or arch == "i686": arch = "386" elif "arm" in arch: arch = "arm" build_command += "GOOS={} GOARCH={} ".format(platform, arch) if "arm" in arch: if arch == "armel": build_command += "GOARM=5 " elif arch == "armhf" or arch == "arm": build_command += "GOARM=6 " elif arch == "arm64": # TODO(rossmcdonald) - Verify this is the correct setting for arm64 build_command += "GOARM=7 " else: logging.error("Invalid ARM architecture specified: {}".format(arch)) logging.error("Please specify either 'armel', 'armhf', or 'arm64'.") return False if platform == 'windows': target = target + '.exe' build_command += "go build -o {} ".format(os.path.join(outdir, target)) if race: build_command += "-race " if len(tags) > 0: build_command += "-tags {} ".format(','.join(tags)) if "1.4" in get_go_version(): if static: build_command += "-ldflags=\"-s -X main.version {} -X main.branch {} -X main.commit {}\" ".format(version, get_current_branch(), get_current_commit()) else: build_command += "-ldflags=\"-X main.version {} -X main.branch {} -X main.commit {}\" ".format(version, get_current_branch(), get_current_commit()) else: # Starting with Go 1.5, the linker flag arguments changed to 'name=value' from 'name value' if static: build_command += "-ldflags=\"-s -X main.version={} -X main.branch={} -X main.commit={}\" ".format(version, get_current_branch(), get_current_commit()) else: build_command += "-ldflags=\"-X main.version={} -X main.branch={} -X main.commit={}\" ".format(version, get_current_branch(), get_current_commit()) if static: build_command += "-a -installsuffix cgo " build_command += path start_time = datetime.utcnow() run(build_command, shell=True) end_time = datetime.utcnow() logging.info("Time taken: {}s".format((end_time - start_time).total_seconds())) return True def generate_md5_from_file(path): """Generate MD5 signature based on the contents of the file at path. """ m = hashlib.md5() with open(path, 'rb') as f: for chunk in iter(lambda: f.read(4096), b""): m.update(chunk) return m.hexdigest() def generate_sig_from_file(path): """Generate a detached GPG signature from the file at path. """ logging.debug("Generating GPG signature for file: {}".format(path)) gpg_path = check_path_for('gpg') if gpg_path is None: logging.warn("gpg binary not found on path! Skipping signature creation.") return False if os.environ.get("GNUPG_HOME") is not None: run('gpg --homedir {} --armor --yes --detach-sign {}'.format(os.environ.get("GNUPG_HOME"), path)) else: run('gpg --armor --detach-sign --yes {}'.format(path)) return True def package(build_output, pkg_name, version, nightly=False, iteration=1, static=False, release=False): """Package the output of the build process. """ outfiles = [] tmp_build_dir = create_temp_dir() logging.debug("Packaging for build output: {}".format(build_output)) logging.info("Using temporary directory: {}".format(tmp_build_dir)) try: for platform in build_output: # Create top-level folder displaying which platform (linux, etc) os.makedirs(os.path.join(tmp_build_dir, platform)) for arch in build_output[platform]: logging.info("Creating packages for {}/{}".format(platform, arch)) # Create second-level directory displaying the architecture (amd64, etc) current_location = build_output[platform][arch] # Create directory tree to mimic file system of package build_root = os.path.join(tmp_build_dir, platform, arch, '{}-{}-{}'.format(PACKAGE_NAME, version, iteration)) os.makedirs(build_root) # Copy packaging scripts to build directory if platform == "windows" or static or "static_" in arch: # For windows and static builds, just copy # binaries to root of package (no other scripts or # directories) package_scripts(build_root, config_only=True) else: create_package_fs(build_root) package_scripts(build_root) for binary in targets: # Copy newly-built binaries to packaging directory if platform == 'windows': binary = binary + '.exe' if platform == 'windows' or static or "static_" in arch: # Where the binary should go in the package filesystem to = os.path.join(build_root, binary) # Where the binary currently is located fr = os.path.join(current_location, binary) else: # Where the binary currently is located fr = os.path.join(current_location, binary) # Where the binary should go in the package filesystem to = os.path.join(build_root, INSTALL_ROOT_DIR[1:], binary) shutil.copy(fr, to) for package_type in supported_packages[platform]: # Package the directory structure for each package type for the platform logging.debug("Packaging directory '{}' as '{}'.".format(build_root, package_type)) name = pkg_name # Reset version, iteration, and current location on each run # since they may be modified below. package_version = version package_iteration = iteration if "static_" in arch: # Remove the "static_" from the displayed arch on the package package_arch = arch.replace("static_", "") else: package_arch = arch if not release and not nightly: # For non-release builds, just use the commit hash as the version package_version = "{}~{}".format(version, get_current_commit(short=True)) package_iteration = "0" package_build_root = build_root current_location = build_output[platform][arch] if package_type in ['zip', 'tar']: # For tars and zips, start the packaging one folder above # the build root (to include the package name) package_build_root = os.path.join('/', '/'.join(build_root.split('/')[:-1])) if nightly: if static or "static_" in arch: name = '{}-static-nightly_{}_{}'.format(name, platform, package_arch) else: name = '{}-nightly_{}_{}'.format(name, platform, package_arch) else: if static or "static_" in arch: name = '{}-{}-static_{}_{}'.format(name, package_version, platform, package_arch) else: name = '{}-{}_{}_{}'.format(name, package_version, platform, package_arch) current_location = os.path.join(os.getcwd(), current_location) if package_type == 'tar': tar_command = "cd {} && tar -cvzf {}.tar.gz ./*".format(package_build_root, name) run(tar_command, shell=True) run("mv {}.tar.gz {}".format(os.path.join(package_build_root, name), current_location), shell=True) outfile = os.path.join(current_location, name + ".tar.gz") outfiles.append(outfile) elif package_type == 'zip': zip_command = "cd {} && zip -r {}.zip ./*".format(package_build_root, name) run(zip_command, shell=True) run("mv {}.zip {}".format(os.path.join(package_build_root, name), current_location), shell=True) outfile = os.path.join(current_location, name + ".zip") outfiles.append(outfile) elif package_type not in ['zip', 'tar'] and static or "static_" in arch: logging.info("Skipping package type '{}' for static builds.".format(package_type)) else: fpm_command = "fpm {} --name {} -a {} -t {} --version {} --iteration {} -C {} -p {} ".format( fpm_common_args, name, package_arch, package_type, package_version, package_iteration, package_build_root, current_location) if package_type == "rpm": fpm_command += "--depends coreutils --rpm-posttrans {}".format(POSTINST_SCRIPT) out = run(fpm_command, shell=True) matches = re.search(':path=>"(.*)"', out) outfile = None if matches is not None: outfile = matches.groups()[0] if outfile is None: logging.warn("Could not determine output from packaging output!") else: if nightly: # Strip nightly version from package name new_outfile = outfile.replace("{}-{}".format(package_version, package_iteration), "nightly") os.rename(outfile, new_outfile) outfile = new_outfile else: if package_type == 'rpm': # rpm's convert any dashes to underscores package_version = package_version.replace("-", "_") new_outfile = outfile.replace("{}-{}".format(package_version, package_iteration), package_version) os.rename(outfile, new_outfile) outfile = new_outfile outfiles.append(os.path.join(os.getcwd(), outfile)) logging.debug("Produced package files: {}".format(outfiles)) return outfiles finally: # Cleanup shutil.rmtree(tmp_build_dir) def main(args): global PACKAGE_NAME if args.release and args.nightly: logging.error("Cannot be both a nightly and a release.") return 1 if args.nightly: args.version = increment_minor_version(args.version) args.version = "{}~n{}".format(args.version, datetime.utcnow().strftime("%Y%m%d%H%M")) args.iteration = 0 # Validate version if not re.match(r'^[-\d\w\.]+', args.version): logging.error("Invalid version {}".format(args.version)) return 1 # Pre-build checks check_environ() if not check_prereqs(): return 1 if args.build_tags is None: args.build_tags = [] else: args.build_tags = args.build_tags.split(',') orig_commit = get_current_commit(short=True) orig_branch = get_current_branch() if args.platform not in supported_builds and args.platform != 'all': logging.error("Invalid build platform: {}".format(target_platform)) return 1 build_output = {} if args.branch != orig_branch and args.commit != orig_commit: logging.error("Can only specify one branch or commit to build from.") return 1 elif args.branch != orig_branch: logging.info("Moving to git branch: {}".format(args.branch)) run("git checkout {}".format(args.branch)) elif args.commit != orig_commit: logging.info("Moving to git commit: {}".format(args.commit)) run("git checkout {}".format(args.commit)) if not args.no_get: if not go_get(): return 1 if args.generate: if not run_generate(): return 1 if args.no_uncommitted: if not check_nochanges(): return 1 if args.test: if not run_tests(args.race, args.parallel, args.timeout, args.no_vet): return 1 platforms = [] single_build = True if args.platform == 'all': platforms = supported_builds.keys() single_build = False else: platforms = [args.platform] for platform in platforms: build_output.update( { platform : {} } ) archs = [] if args.arch == "all": single_build = False archs = supported_builds.get(platform) else: archs = [args.arch] for arch in archs: od = args.outdir if not single_build: od = os.path.join(args.outdir, platform, arch) if not build(version=args.version, platform=platform, arch=arch, nightly=args.nightly, race=args.race, clean=args.clean, outdir=od, tags=args.build_tags, static=args.static): return 1 build_output.get(platform).update( { arch : od } ) # Build packages if args.package: if not check_path_for("fpm"): logging.error("FPM ruby gem required for packaging. Stopping.") return 1 packages = package(build_output, args.name, args.version, nightly=args.nightly, iteration=args.iteration, static=args.static, release=args.release) if args.package_udfs: packages += package_udfs(args.version, args.outdir) if args.sign: logging.debug("Generating GPG signatures for packages: {}".format(packages)) sigs = [] # retain signatures so they can be uploaded with packages for p in packages: if generate_sig_from_file(p): sigs.append(p + '.asc') else: logging.error("Creation of signature for package [{}] failed!".format(p)) return 1 packages += sigs if args.upload: logging.debug("Files staged for upload: {}".format(packages)) if args.nightly: args.upload_overwrite = True if not upload_packages(packages, bucket_name=args.bucket, overwrite=args.upload_overwrite): return 1 logging.info("Packages created:") for p in packages: logging.info("{} (MD5={})".format(p.split('/')[-1:][0], generate_md5_from_file(p))) if orig_branch != get_current_branch(): logging.info("Moving back to original git branch: {}".format(args.branch)) run("git checkout {}".format(orig_branch)) return 0 if __name__ == '__main__': LOG_LEVEL = logging.INFO if '--debug' in sys.argv[1:]: LOG_LEVEL = logging.DEBUG log_format = '[%(levelname)s] %(funcName)s: %(message)s' logging.basicConfig(level=LOG_LEVEL, format=log_format) parser = argparse.ArgumentParser(description='InfluxDB build and packaging script.') parser.add_argument('--verbose','-v','--debug', action='store_true', help='Use debug output') parser.add_argument('--outdir', '-o', metavar='', default='./build/', type=os.path.abspath, help='Output directory') parser.add_argument('--name', '-n', metavar='', default=PACKAGE_NAME, type=str, help='Name to use for package name (when package is specified)') parser.add_argument('--arch', metavar='', type=str, default=get_system_arch(), help='Target architecture for build output') parser.add_argument('--platform', metavar='', type=str, default=get_system_platform(), help='Target platform for build output') parser.add_argument('--branch', metavar='', type=str, default=get_current_branch(), help='Build from a specific branch') parser.add_argument('--commit', metavar='', type=str, default=get_current_commit(short=True), help='Build from a specific commit') parser.add_argument('--version', metavar='', type=str, default=get_current_version(), help='Version information to apply to build output (ex: 0.12.0)') parser.add_argument('--iteration', metavar='', type=str, default="1", help='Package iteration to apply to build output (defaults to 1)') parser.add_argument('--stats', action='store_true', help='Emit build metrics (requires InfluxDB Python client)') parser.add_argument('--stats-server', metavar='', type=str, help='Send build stats to InfluxDB using provided hostname and port') parser.add_argument('--stats-db', metavar='', type=str, help='Send build stats to InfluxDB using provided database name') parser.add_argument('--nightly', action='store_true', help='Mark build output as nightly build (will incremement the minor version)') parser.add_argument('--update', action='store_true', help='Update build dependencies prior to building') parser.add_argument('--package', action='store_true', help='Package binary output') parser.add_argument('--package-udfs', action='store_true', help='Package UDF agents') parser.add_argument('--release', action='store_true', help='Mark build output as release') parser.add_argument('--clean', action='store_true', help='Clean output directory before building') parser.add_argument('--no-get', action='store_true', help='Do not retrieve pinned dependencies when building') parser.add_argument('--no-uncommitted', action='store_true', help='Fail if uncommitted changes exist in the working directory') parser.add_argument('--upload', action='store_true', help='Upload output packages to AWS S3') parser.add_argument('--upload-overwrite','-w', action='store_true', help='Upload output packages to AWS S3') parser.add_argument('--bucket', metavar='', type=str, default=DEFAULT_BUCKET, help='Destination bucket for uploads') parser.add_argument('--generate', action='store_true', help='Run "go generate" before building') parser.add_argument('--build-tags', metavar='', help='Optional build tags to use for compilation') parser.add_argument('--static', action='store_true', help='Create statically-compiled binary output') parser.add_argument('--sign', action='store_true', help='Create GPG detached signatures for packages (when package is specified)') parser.add_argument('--test', action='store_true', help='Run tests (does not produce build output)') parser.add_argument('--no-vet', action='store_true', help='Do not run "go vet" when running tests') parser.add_argument('--race', action='store_true', help='Enable race flag for build output') parser.add_argument('--parallel', metavar='', type=int, help='Number of tests to run simultaneously') parser.add_argument('--timeout', metavar='', type=str, help='Timeout for tests before failing') args = parser.parse_args() print_banner() sys.exit(main(args)) ================================================ FILE: vendor/github.com/influxdata/kapacitor/build.sh ================================================ #!/bin/bash # Run the build utility via Docker set -e # Make sure our working dir is the dir of the script DIR=$(cd $(dirname ${BASH_SOURCE[0]}) && pwd) cd $DIR # Unique number for this build BUILD_NUM=${BUILD_NUM-$RANDOM} # Home dir of the docker user HOME_DIR=/root imagename=kapacitor-builder-img-$BUILD_NUM dataname=kapacitor-data-$BUILD_NUM # Build new docker image docker build -f Dockerfile_build_ubuntu64 -t $imagename $DIR # Build new docker image docker build -f Dockerfile_build_ubuntu64 -t influxdata/kapacitor-builder $DIR # Create data volume with code docker create \ --name $dataname \ -v "$HOME_DIR/go/src/github.com/influxdata/kapacitor" \ $imagename /bin/true docker cp "$DIR/" "$dataname:$HOME_DIR/go/src/github.com/influxdata/" echo "Running build.py" # Run docker docker run \ --rm \ --volumes-from $dataname \ -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ $imagename \ "$@" docker cp "$dataname:$HOME_DIR/go/src/github.com/influxdata/kapacitor/build" \ ./ docker rm -v $dataname ================================================ FILE: vendor/github.com/influxdata/kapacitor/circle-test.sh ================================================ #!/bin/bash # # This is the InfluxDB test script for CircleCI, it is a light wrapper around ./test.sh. # Exit if any command fails set -e # Get dir of script and make it is our working directory. DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) cd $DIR export NO_UNCOMMITTED=true export BUILD_NUM=$CIRCLE_BUILD_NUM # Get number of test environments. count=$(./test.sh count) # Check that we aren't wasting CircleCI nodes. if [ $CIRCLE_NODE_TOTAL -gt $count ] then echo "More CircleCI nodes allocated than tests environments to run!" exit 1 fi # Map CircleCI nodes to test environments. tests=$(seq 0 $((count - 1))) for i in $tests do mine=$(( $i % $CIRCLE_NODE_TOTAL )) if [ $mine -eq $CIRCLE_NODE_INDEX ] then echo "Running test env index: $i" ./test.sh $i fi done ================================================ FILE: vendor/github.com/influxdata/kapacitor/circle.yml ================================================ machine: services: - docker dependencies: pre: # setup ipv6 - sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0 cache_directories: - "~/docker" override: - ./test.sh save test: override: - bash circle-test.sh: parallel: true deployment: release: tag: /v[0-9]+(\.[0-9]+){2}(-(rc|beta)[0-9]+)?/ commands: - ./build.sh --debug --clean --generate --package --package-udfs --upload --bucket=dl.influxdata.com/kapacitor/releases --platform=all --arch=all --release ================================================ FILE: vendor/github.com/influxdata/kapacitor/combine.go ================================================ package kapacitor import ( "fmt" "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" ) type CombineNode struct { node c *pipeline.CombineNode expressions []stateful.Expression scopePools []stateful.ScopePool combination combination } // Create a new CombineNode, which combines a stream with itself dynamically. func newCombineNode(et *ExecutingTask, n *pipeline.CombineNode, l *log.Logger) (*CombineNode, error) { cn := &CombineNode{ c: n, node: node{Node: n, et: et, logger: l}, combination: combination{max: n.Max}, } // Create stateful expressions cn.expressions = make([]stateful.Expression, len(n.Lambdas)) cn.scopePools = make([]stateful.ScopePool, len(n.Lambdas)) for i, lambda := range n.Lambdas { statefulExpr, err := stateful.NewExpression(lambda.Expression) if err != nil { return nil, fmt.Errorf("Failed to compile %v expression: %v", i, err) } cn.expressions[i] = statefulExpr cn.scopePools[i] = stateful.NewScopePool(ast.FindReferenceVariables(lambda.Expression)) } cn.node.runF = cn.runCombine return cn, nil } func (n *CombineNode) runCombine([]byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *CombineNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { expressions := make([]stateful.Expression, len(n.expressions)) for i, expr := range n.expressions { expressions[i] = expr.CopyReset() } return &combineBuffer{ n: n, time: first.Time(), name: first.Name(), groupInfo: group, expressions: expressions, c: n.combination, }, nil } type combineBuffer struct { n *CombineNode time time.Time name string groupInfo edge.GroupInfo points []edge.FieldsTagsTimeSetter expressions []stateful.Expression c combination begin edge.BeginBatchMessage } func (b *combineBuffer) BeginBatch(begin edge.BeginBatchMessage) error { b.n.timer.Start() defer b.n.timer.Stop() b.name = begin.Name() b.time = time.Time{} if s := begin.SizeHint(); s > cap(b.points) { b.points = make([]edge.FieldsTagsTimeSetter, 0, s) } return nil } func (b *combineBuffer) BatchPoint(bp edge.BatchPointMessage) error { b.n.timer.Start() defer b.n.timer.Stop() bp = bp.ShallowCopy() return b.addPoint(bp) } func (b *combineBuffer) EndBatch(end edge.EndBatchMessage) error { b.n.timer.Start() defer b.n.timer.Stop() if err := b.combine(); err != nil { return err } b.points = b.points[0:0] return nil } func (b *combineBuffer) Point(p edge.PointMessage) error { b.n.timer.Start() defer b.n.timer.Stop() p = p.ShallowCopy() return b.addPoint(p) } func (b *combineBuffer) addPoint(p edge.FieldsTagsTimeSetter) error { t := p.Time().Round(b.n.c.Tolerance) p.SetTime(t) if t.Equal(b.time) { b.points = append(b.points, p) } else { if err := b.combine(); err != nil { return err } b.time = t b.points = b.points[0:1] b.points[0] = p } return nil } func (b *combineBuffer) Barrier(barrier edge.BarrierMessage) error { return edge.Forward(b.n.outs, barrier) } func (b *combineBuffer) DeleteGroup(d edge.DeleteGroupMessage) error { return edge.Forward(b.n.outs, d) } // Combine a set of points into all their combinations. func (b *combineBuffer) combine() error { if len(b.points) == 0 { return nil } l := len(b.expressions) // Compute matching result for all points matches := make([]map[int]bool, l) for i := 0; i < l; i++ { matches[i] = make(map[int]bool, len(b.points)) } for idx, p := range b.points { for i := range b.expressions { matched, err := EvalPredicate(b.expressions[i], b.n.scopePools[i], p) if err != nil { b.n.incrementErrorCount() b.n.logger.Println("E! evaluating lambda expression:", err) } matches[i][idx] = matched } } p := edge.NewPointMessage( b.name, "", "", b.groupInfo.Dimensions, nil, nil, time.Time{}, ) dimensions := p.Dimensions().ToSet() set := make([]edge.FieldsTagsTimeSetter, l) return b.c.Do(len(b.points), l, func(indices []int) error { valid := true for s := 0; s < l; s++ { found := false for i := range indices { if matches[s][indices[i]] { set[s] = b.points[indices[i]] indices = append(indices[0:i], indices[i+1:]...) found = true break } } if !found { valid = false break } } if valid { fields, tags, t := b.merge(set, dimensions) np := p.ShallowCopy() np.SetFields(fields) np.SetTags(tags) np.SetTime(t.Round(b.n.c.Tolerance)) b.n.timer.Pause() err := edge.Forward(b.n.outs, np) b.n.timer.Resume() if err != nil { return err } } return nil }) } // Merge a set of points into a single point. func (b *combineBuffer) merge(points []edge.FieldsTagsTimeSetter, dimensions map[string]bool) (models.Fields, models.Tags, time.Time) { fields := make(models.Fields, len(points[0].Fields())*len(points)) tags := make(models.Tags, len(points[0].Tags())*len(points)) for i, p := range points { for field, value := range p.Fields() { fields[b.n.c.Names[i]+b.n.c.Delimiter+field] = value } for tag, value := range p.Tags() { if !dimensions[tag] { tags[b.n.c.Names[i]+b.n.c.Delimiter+tag] = value } else { tags[tag] = value } } } return fields, tags, points[0].Time() } // Type for performing actions on a set of combinations. type combination struct { max int64 } // Do action for each combination, based on combinatorial logic n choose k. // If n choose k > max an error is returned func (c combination) Do(n, k int, f func(indices []int) error) error { if count := c.Count(int64(n), int64(k)); count > c.max { return fmt.Errorf("refusing to perform combination as total combinations %d exceeds max combinations %d", count, c.max) } else if count == -1 { // Nothing to do return nil } indices := make([]int, k) indicesCopy := make([]int, k) for i := 0; i < k; i++ { indices[i] = i } copy(indicesCopy, indices) if err := f(indicesCopy); err != nil { return err } for { i := k - 1 for ; i >= 0; i-- { if indices[i] != i+n-k { break } } if i == -1 { return nil } indices[i]++ for j := i + 1; j < k; j++ { indices[j] = indices[j-1] + 1 } copy(indicesCopy, indices) if err := f(indicesCopy); err != nil { return err } } } // Count the number of possible combinations of n choose k. func (c combination) Count(n, k int64) int64 { if n < k { return -1 } count := int64(1) for i := int64(0); i < k; i++ { count = (count * (n - i)) / (i + 1) } return count } ================================================ FILE: vendor/github.com/influxdata/kapacitor/combine_test.go ================================================ package kapacitor import ( "reflect" "testing" ) func Test_Combination_Count(t *testing.T) { c := combination{max: 1e9} testCases := []struct { n, k, exp int64 }{ { n: 1, k: 0, exp: 1, }, { n: 1, k: 1, exp: 1, }, { n: 2, k: 1, exp: 2, }, { n: 5, k: 2, exp: 10, }, { n: 5, k: 3, exp: 10, }, { n: 52, k: 5, exp: 2598960, }, } for _, tc := range testCases { if exp, got := tc.exp, c.Count(tc.n, tc.k); exp != got { t.Errorf("unexpected combination count for %d choose %d: got %d exp %d", tc.n, tc.k, got, exp) } } } func Test_Combination_Do(t *testing.T) { c := combination{max: 1e9} testCases := []struct { n, k int exp [][]int }{ { n: 1, k: 1, exp: [][]int{{0}}, }, { n: 5, k: 2, exp: [][]int{ {0, 1}, {0, 2}, {0, 3}, {0, 4}, {1, 2}, {1, 3}, {1, 4}, {2, 3}, {2, 4}, {3, 4}, }, }, { n: 5, k: 3, exp: [][]int{ {0, 1, 2}, {0, 1, 3}, {0, 1, 4}, {0, 2, 3}, {0, 2, 4}, {0, 3, 4}, {1, 2, 3}, {1, 2, 4}, {1, 3, 4}, {2, 3, 4}, }, }, { n: 7, k: 5, exp: [][]int{ {0, 1, 2, 3, 4}, {0, 1, 2, 3, 5}, {0, 1, 2, 3, 6}, {0, 1, 2, 4, 5}, {0, 1, 2, 4, 6}, {0, 1, 2, 5, 6}, {0, 1, 3, 4, 5}, {0, 1, 3, 4, 6}, {0, 1, 3, 5, 6}, {0, 1, 4, 5, 6}, {0, 2, 3, 4, 5}, {0, 2, 3, 4, 6}, {0, 2, 3, 5, 6}, {0, 2, 4, 5, 6}, {0, 3, 4, 5, 6}, {1, 2, 3, 4, 5}, {1, 2, 3, 4, 6}, {1, 2, 3, 5, 6}, {1, 2, 4, 5, 6}, {1, 3, 4, 5, 6}, {2, 3, 4, 5, 6}, }, }, } for _, tc := range testCases { i := 0 c.Do(tc.n, tc.k, func(indices []int) error { if i == len(tc.exp) { t.Fatalf("too many combinations returned for %d choose %d: got %v", tc.n, tc.k, indices) } if !reflect.DeepEqual(tc.exp[i], indices) { t.Errorf("unexpected combination set for %d choose %d index %d: got %v exp %v", tc.n, tc.k, i, indices, tc.exp[i]) } i++ return nil }) if i != len(tc.exp) { t.Errorf("not enough combinations returned for %d choose %d", tc.n, tc.k) } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/default.go ================================================ package kapacitor import ( "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) const ( statsFieldsDefaulted = "fields_defaulted" statsTagsDefaulted = "tags_defaulted" ) type DefaultNode struct { node d *pipeline.DefaultNode fieldsDefaulted *expvar.Int tagsDefaulted *expvar.Int } // Create a new DefaultNode which applies a transformation func to each point in a stream and returns a single point. func newDefaultNode(et *ExecutingTask, n *pipeline.DefaultNode, l *log.Logger) (*DefaultNode, error) { dn := &DefaultNode{ node: node{Node: n, et: et, logger: l}, d: n, fieldsDefaulted: new(expvar.Int), tagsDefaulted: new(expvar.Int), } dn.node.runF = dn.runDefault return dn, nil } func (n *DefaultNode) runDefault(snapshot []byte) error { n.statMap.Set(statsFieldsDefaulted, n.fieldsDefaulted) n.statMap.Set(statsTagsDefaulted, n.tagsDefaulted) consumer := edge.NewConsumerWithReceiver( n.ins[0], edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n), ), ) return consumer.Consume() } func (n *DefaultNode) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { begin = begin.ShallowCopy() _, tags := n.setDefaults(nil, begin.Tags()) begin.SetTags(tags) return begin, nil } func (n *DefaultNode) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { bp = bp.ShallowCopy() fields, tags := n.setDefaults(bp.Fields(), bp.Tags()) bp.SetFields(fields) bp.SetTags(tags) return bp, nil } func (n *DefaultNode) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (n *DefaultNode) Point(p edge.PointMessage) (edge.Message, error) { p = p.ShallowCopy() fields, tags := n.setDefaults(p.Fields(), p.Tags()) p.SetFields(fields) p.SetTags(tags) return p, nil } func (n *DefaultNode) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *DefaultNode) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (n *DefaultNode) setDefaults(fields models.Fields, tags models.Tags) (models.Fields, models.Tags) { newFields := fields fieldsCopied := false for field, value := range n.d.Fields { if v := fields[field]; v == nil { if !fieldsCopied { newFields = newFields.Copy() fieldsCopied = true } n.fieldsDefaulted.Add(1) newFields[field] = value } } newTags := tags tagsCopied := false for tag, value := range n.d.Tags { if v := tags[tag]; v == "" { if !tagsCopied { newTags = newTags.Copy() tagsCopied = true } n.tagsDefaulted.Add(1) newTags[tag] = value } } return newFields, newTags } ================================================ FILE: vendor/github.com/influxdata/kapacitor/delete.go ================================================ package kapacitor import ( "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) const ( statsFieldsDeleted = "fields_deleted" statsTagsDeleted = "tags_deleted" ) type DeleteNode struct { node d *pipeline.DeleteNode fieldsDeleted *expvar.Int tagsDeleted *expvar.Int tags map[string]bool } // Create a new DeleteNode which applies a transformation func to each point in a stream and returns a single point. func newDeleteNode(et *ExecutingTask, n *pipeline.DeleteNode, l *log.Logger) (*DeleteNode, error) { tags := make(map[string]bool) for _, tag := range n.Tags { tags[tag] = true } dn := &DeleteNode{ node: node{Node: n, et: et, logger: l}, d: n, fieldsDeleted: new(expvar.Int), tagsDeleted: new(expvar.Int), tags: tags, } dn.node.runF = dn.runDelete return dn, nil } func (n *DeleteNode) runDelete(snapshot []byte) error { n.statMap.Set(statsFieldsDeleted, n.fieldsDeleted) n.statMap.Set(statsTagsDeleted, n.tagsDeleted) consumer := edge.NewConsumerWithReceiver( n.ins[0], edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n), ), ) return consumer.Consume() } func (n *DeleteNode) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { begin = begin.ShallowCopy() _, tags := n.doDeletes(nil, begin.Tags()) begin.SetTags(tags) return begin, nil } func (n *DeleteNode) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { bp = bp.ShallowCopy() fields, tags := n.doDeletes(bp.Fields(), bp.Tags()) bp.SetFields(fields) bp.SetTags(tags) return bp, nil } func (n *DeleteNode) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (n *DeleteNode) Point(p edge.PointMessage) (edge.Message, error) { p = p.ShallowCopy() fields, tags := n.doDeletes(p.Fields(), p.Tags()) p.SetFields(fields) p.SetTags(tags) dims := p.Dimensions() if n.checkForDeletedDimension(dims) { p.SetDimensions(n.deleteDimensions(dims)) } return p, nil } func (n *DeleteNode) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *DeleteNode) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } // checkForDeletedDimension checks if we deleted a group by dimension func (n *DeleteNode) checkForDeletedDimension(dimensions models.Dimensions) bool { for _, dim := range dimensions.TagNames { if n.tags[dim] { return true } } return false } func (n *DeleteNode) deleteDimensions(dims models.Dimensions) models.Dimensions { newTagNames := make([]string, 0, len(dims.TagNames)-1) for _, dim := range dims.TagNames { if !n.tags[dim] { newTagNames = append(newTagNames, dim) } } return models.Dimensions{ TagNames: newTagNames, ByName: dims.ByName, } } func (n *DeleteNode) doDeletes(fields models.Fields, tags models.Tags) (models.Fields, models.Tags) { newFields := fields fieldsCopied := false for _, field := range n.d.Fields { if _, ok := fields[field]; ok { if !fieldsCopied { newFields = newFields.Copy() fieldsCopied = true } n.fieldsDeleted.Add(1) delete(newFields, field) } } newTags := tags tagsCopied := false for _, tag := range n.d.Tags { if _, ok := tags[tag]; ok { if !tagsCopied { newTags = newTags.Copy() tagsCopied = true } n.tagsDeleted.Add(1) delete(newTags, tag) } } return newFields, newTags } ================================================ FILE: vendor/github.com/influxdata/kapacitor/derivative.go ================================================ package kapacitor import ( "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) type DerivativeNode struct { node d *pipeline.DerivativeNode } // Create a new derivative node. func newDerivativeNode(et *ExecutingTask, n *pipeline.DerivativeNode, l *log.Logger) (*DerivativeNode, error) { dn := &DerivativeNode{ node: node{Node: n, et: et, logger: l}, d: n, } // Create stateful expressions dn.node.runF = dn.runDerivative return dn, nil } func (n *DerivativeNode) runDerivative([]byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *DerivativeNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup()), ), nil } func (n *DerivativeNode) newGroup() *derivativeGroup { return &derivativeGroup{ n: n, } } type derivativeGroup struct { n *DerivativeNode previous edge.FieldsTagsTimeGetter } func (g *derivativeGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { if s := begin.SizeHint(); s > 0 { begin = begin.ShallowCopy() begin.SetSizeHint(s - 1) } g.previous = nil return begin, nil } func (g *derivativeGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { np := bp.ShallowCopy() emit := g.doDerivative(bp, np) if emit { return np, nil } return nil, nil } func (g *derivativeGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (g *derivativeGroup) Point(p edge.PointMessage) (edge.Message, error) { np := p.ShallowCopy() emit := g.doDerivative(p, np) if emit { return np, nil } return nil, nil } // doDerivative computes the derivative with respect to g.previous and p. // The resulting derivative value will be set on n. func (g *derivativeGroup) doDerivative(p edge.FieldsTagsTimeGetter, n edge.FieldsTagsTimeSetter) bool { var prevFields, currFields models.Fields var prevTime, currTime time.Time if g.previous != nil { prevFields = g.previous.Fields() prevTime = g.previous.Time() } currFields = p.Fields() currTime = p.Time() value, store, emit := g.n.derivative( prevFields, currFields, prevTime, currTime, ) if store { g.previous = p } if !emit { return false } fields := n.Fields().Copy() fields[g.n.d.As] = value n.SetFields(fields) return true } func (g *derivativeGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *derivativeGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } // derivative calculates the derivative between prev and cur. // Return is the resulting derivative, whether the current point should be // stored as previous, and whether the point result should be emitted. func (n *DerivativeNode) derivative(prev, curr models.Fields, prevTime, currTime time.Time) (float64, bool, bool) { f1, ok := numToFloat(curr[n.d.Field]) if !ok { n.incrementErrorCount() n.logger.Printf("E! cannot apply derivative to type %T", curr[n.d.Field]) return 0, false, false } f0, ok := numToFloat(prev[n.d.Field]) if !ok { // The only time this will fail to parse is if there is no previous. // Because we only return `store=true` if current parses successfully, we will // never get a previous which doesn't parse. return 0, true, false } elapsed := float64(currTime.Sub(prevTime)) if elapsed == 0 { n.incrementErrorCount() n.logger.Printf("E! cannot perform derivative elapsed time was 0") return 0, true, false } diff := f1 - f0 // Drop negative values for non-negative derivatives if n.d.NonNegativeFlag && diff < 0 { return 0, true, false } value := float64(diff) / (elapsed / float64(n.d.Unit)) return value, true, true } func numToFloat(num interface{}) (float64, bool) { switch n := num.(type) { case int64: return float64(n), true case float64: return n, true default: return 0, false } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/doc.go ================================================ /* A data pipeline processing engine. See the README for more complete examples and guides. Code Organization: The pipeline package provides an API for how nodes can be connected to form a pipeline. The individual implementations of each node exist in this kapacitor package. The reason for the separation is to keep the exported API from the pipeline package clean as it is consumed via the TICKscripts (a DSL for Kapacitor). Other Concepts: Stream vs Batch -- Use of the word 'stream' indicates data arrives a single data point at a time. Use of the word 'batch' indicates data arrives in sets or batches or data points. Task -- A task represents a concrete workload to perform. It consists of a pipeline and an identifying name. Basic CRUD operations can be performed on tasks. Task Master -- Responsible for executing a task in a specific environment. Replay -- Replays static datasets against tasks. */ package kapacitor ================================================ FILE: vendor/github.com/influxdata/kapacitor/edge.go ================================================ package kapacitor import ( "errors" "fmt" "log" "sync" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/server/vars" ) const ( statCollected = "collected" statEmitted = "emitted" defaultEdgeBufferSize = 1000 ) var ErrAborted = errors.New("edged aborted") type Edge struct { edge.StatsEdge mu sync.Mutex closed bool statsKey string statMap *expvar.Map logger *log.Logger } func newEdge(taskName, parentName, childName string, t pipeline.EdgeType, size int, logService LogService) edge.StatsEdge { e := edge.NewStatsEdge(edge.NewChannelEdge(t, defaultEdgeBufferSize)) tags := map[string]string{ "task": taskName, "parent": parentName, "child": childName, "type": t.String(), } key, sm := vars.NewStatistic("edges", tags) sm.Set(statCollected, e.CollectedVar()) sm.Set(statEmitted, e.EmittedVar()) name := fmt.Sprintf("%s|%s->%s", taskName, parentName, childName) return &Edge{ StatsEdge: e, statsKey: key, statMap: sm, logger: logService.NewLogger(fmt.Sprintf("[edge:%s] ", name), log.LstdFlags), } } func (e *Edge) Close() error { e.mu.Lock() defer e.mu.Unlock() if e.closed { return nil } e.closed = true vars.DeleteStatistic(e.statsKey) e.logger.Printf("D! closing c: %d e: %d", e.Collected(), e.Emitted(), ) return e.StatsEdge.Close() } ================================================ FILE: vendor/github.com/influxdata/kapacitor/eval.go ================================================ package kapacitor import ( "errors" "fmt" "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" ) type EvalNode struct { node e *pipeline.EvalNode expressions []stateful.Expression refVarList [][]string scopePool stateful.ScopePool tags map[string]bool evalErrors *expvar.Int } // Create a new EvalNode which applies a transformation func to each point in a stream and returns a single point. func newEvalNode(et *ExecutingTask, n *pipeline.EvalNode, l *log.Logger) (*EvalNode, error) { if len(n.AsList) != len(n.Lambdas) { return nil, errors.New("must provide one name per expression via the 'As' property") } en := &EvalNode{ node: node{Node: n, et: et, logger: l}, e: n, } // Create stateful expressions en.expressions = make([]stateful.Expression, len(n.Lambdas)) en.refVarList = make([][]string, len(n.Lambdas)) expressions := make([]ast.Node, len(n.Lambdas)) for i, lambda := range n.Lambdas { expressions[i] = lambda.Expression statefulExpr, err := stateful.NewExpression(lambda.Expression) if err != nil { return nil, fmt.Errorf("Failed to compile %v expression: %v", i, err) } en.expressions[i] = statefulExpr refVars := ast.FindReferenceVariables(lambda.Expression) en.refVarList[i] = refVars } // Create a single pool for the combination of all expressions en.scopePool = stateful.NewScopePool(ast.FindReferenceVariables(expressions...)) // Create map of tags if l := len(n.TagsList); l > 0 { en.tags = make(map[string]bool, l) for _, tag := range n.TagsList { en.tags[tag] = true } } en.node.runF = en.runEval return en, nil } func (n *EvalNode) runEval(snapshot []byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *EvalNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup()), ), nil } func (n *EvalNode) newGroup() *evalGroup { expressions := make([]stateful.Expression, len(n.expressions)) for i, exp := range n.expressions { expressions[i] = exp.CopyReset() } return &evalGroup{ n: n, expressions: expressions, } } func (n *EvalNode) eval(expressions []stateful.Expression, p edge.FieldsTagsTimeSetter) error { vars := n.scopePool.Get() defer n.scopePool.Put(vars) for i, expr := range expressions { err := fillScope(vars, n.refVarList[i], p) if err != nil { return err } v, err := expr.Eval(vars) if err != nil { return err } name := n.e.AsList[i] vars.Set(name, v) } fields := p.Fields() tags := p.Tags() newTags := tags if len(n.tags) > 0 { newTags = newTags.Copy() for tag := range n.tags { v, err := vars.Get(tag) if err != nil { return err } if s, ok := v.(string); !ok { return fmt.Errorf("result of a tag expression must be of type string, got %T", v) } else { newTags[tag] = s } } } var newFields models.Fields if n.e.KeepFlag { if l := len(n.e.KeepList); l != 0 { newFields = make(models.Fields, l) for _, f := range n.e.KeepList { // Try the vars scope first if vars.Has(f) { v, err := vars.Get(f) if err != nil { return err } newFields[f] = v } else if v, ok := fields[f]; ok { // Try the raw fields next, since it may not have been a referenced var. newFields[f] = v } else { return fmt.Errorf("cannot keep field %q, field does not exist", f) } } } else { newFields = make(models.Fields, len(fields)+len(n.e.AsList)) for f, v := range fields { newFields[f] = v } for _, f := range n.e.AsList { v, err := vars.Get(f) if err != nil { return err } newFields[f] = v } } } else { newFields = make(models.Fields, len(n.e.AsList)-len(n.tags)) for _, f := range n.e.AsList { if n.tags[f] { continue } v, err := vars.Get(f) if err != nil { return err } newFields[f] = v } } p.SetFields(newFields) p.SetTags(newTags) return nil } type evalGroup struct { n *EvalNode expressions []stateful.Expression } func (g *evalGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { begin = begin.ShallowCopy() begin.SetSizeHint(0) return begin, nil } func (g *evalGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { bp = bp.ShallowCopy() if g.doEval(bp) { return bp, nil } return nil, nil } func (g *evalGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (g *evalGroup) Point(p edge.PointMessage) (edge.Message, error) { p = p.ShallowCopy() if g.doEval(p) { return p, nil } return nil, nil } func (g *evalGroup) doEval(p edge.FieldsTagsTimeSetter) bool { err := g.n.eval(g.expressions, p) if err != nil { g.n.incrementErrorCount() if !g.n.e.QuietFlag { g.n.logger.Println("E!", err) } // Skip bad point return false } return true } func (g *evalGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *evalGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/expr.go ================================================ package kapacitor import ( "fmt" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" ) // EvalPredicate - Evaluate a given expression as a boolean predicate against a set of fields and tags func EvalPredicate(se stateful.Expression, scopePool stateful.ScopePool, p edge.FieldsTagsTimeGetter) (bool, error) { vars := scopePool.Get() defer scopePool.Put(vars) err := fillScope(vars, scopePool.ReferenceVariables(), p) if err != nil { return false, err } // for function signature check if _, err := se.Type(vars); err != nil { return false, err } return se.EvalBool(vars) } // fillScope - given a scope and reference variables, we fill the exact variables from the now, fields and tags. func fillScope(vars *stateful.Scope, referenceVariables []string, p edge.FieldsTagsTimeGetter) error { now := p.Time() fields := p.Fields() tags := p.Tags() for _, refVariableName := range referenceVariables { if refVariableName == "time" { vars.Set("time", now.Local()) continue } // Support the error with tags/fields collision var fieldValue interface{} var isFieldExists bool var tagValue interface{} var isTagExists bool if fieldValue, isFieldExists = fields[refVariableName]; isFieldExists { vars.Set(refVariableName, fieldValue) } if tagValue, isTagExists = tags[refVariableName]; isTagExists { if isFieldExists { return fmt.Errorf("cannot have field and tags with same name %q", refVariableName) } vars.Set(refVariableName, tagValue) } if !isFieldExists && !isTagExists { if !vars.Has(refVariableName) { vars.Set(refVariableName, ast.MissingValue) } } } return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/flatten.go ================================================ package kapacitor import ( "bytes" "log" "sync" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) type FlattenNode struct { node f *pipeline.FlattenNode bufPool sync.Pool } // Create a new FlattenNode, which takes pairs from parent streams combines them into a single point. func newFlattenNode(et *ExecutingTask, n *pipeline.FlattenNode, l *log.Logger) (*FlattenNode, error) { fn := &FlattenNode{ f: n, node: node{Node: n, et: et, logger: l}, bufPool: sync.Pool{ New: func() interface{} { return &bytes.Buffer{} }, }, } fn.node.runF = fn.runFlatten return fn, nil } func (n *FlattenNode) runFlatten([]byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *FlattenNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { t := first.Time().Round(n.f.Tolerance) return &flattenBuffer{ n: n, time: t, name: first.Name(), groupInfo: group, }, nil } type flattenBuffer struct { n *FlattenNode time time.Time name string groupInfo edge.GroupInfo points []edge.FieldsTagsTimeGetter } func (b *flattenBuffer) BeginBatch(begin edge.BeginBatchMessage) error { b.n.timer.Start() defer b.n.timer.Stop() b.name = begin.Name() b.time = time.Time{} if s := begin.SizeHint(); s > cap(b.points) { b.points = make([]edge.FieldsTagsTimeGetter, 0, s) } begin = begin.ShallowCopy() begin.SetSizeHint(0) b.n.timer.Pause() err := edge.Forward(b.n.outs, begin) b.n.timer.Resume() return err } func (b *flattenBuffer) BatchPoint(bp edge.BatchPointMessage) error { b.n.timer.Start() defer b.n.timer.Stop() t := bp.Time().Round(b.n.f.Tolerance) bp = bp.ShallowCopy() bp.SetTime(t) t, fields, err := b.addPoint(bp) if err != nil { return err } if len(fields) == 0 { return nil } return b.emitBatchPoint(t, fields) } func (b *flattenBuffer) emitBatchPoint(t time.Time, fields models.Fields) error { // Emit batch point flatP := edge.NewBatchPointMessage( fields, b.groupInfo.Tags, t, ) b.n.timer.Pause() err := edge.Forward(b.n.outs, flatP) b.n.timer.Resume() return err } func (b *flattenBuffer) EndBatch(end edge.EndBatchMessage) error { b.n.timer.Start() defer b.n.timer.Stop() if len(b.points) > 0 { fields, err := b.n.flatten(b.points) if err != nil { return err } if err := b.emitBatchPoint(b.time, fields); err != nil { return err } b.points = b.points[0:0] } b.n.timer.Pause() err := edge.Forward(b.n.outs, end) b.n.timer.Resume() return err } func (b *flattenBuffer) Point(p edge.PointMessage) error { b.n.timer.Start() defer b.n.timer.Stop() t := p.Time().Round(b.n.f.Tolerance) p = p.ShallowCopy() p.SetTime(t) t, fields, err := b.addPoint(p) if err != nil { return err } if len(fields) == 0 { return nil } // Emit point flatP := edge.NewPointMessage( b.name, "", "", b.groupInfo.Dimensions, fields, b.groupInfo.Tags, t, ) b.n.timer.Pause() err = edge.Forward(b.n.outs, flatP) b.n.timer.Resume() return err } func (b *flattenBuffer) addPoint(p edge.FieldsTagsTimeGetter) (next time.Time, fields models.Fields, err error) { t := p.Time() if !t.Equal(b.time) { if len(b.points) > 0 { fields, err = b.n.flatten(b.points) if err != nil { return } next = b.time b.points = b.points[0:0] } // Update buffer with new time b.time = t } b.points = append(b.points, p) return } func (b *flattenBuffer) Barrier(barrier edge.BarrierMessage) error { return edge.Forward(b.n.outs, barrier) } func (b *flattenBuffer) DeleteGroup(d edge.DeleteGroupMessage) error { return edge.Forward(b.n.outs, d) } func (n *FlattenNode) flatten(points []edge.FieldsTagsTimeGetter) (models.Fields, error) { fields := make(models.Fields) if len(points) == 0 { return fields, nil } fieldPrefix := n.bufPool.Get().(*bytes.Buffer) defer n.bufPool.Put(fieldPrefix) POINTS: for _, p := range points { tags := p.Tags() for i, tag := range n.f.Dimensions { if v, ok := tags[tag]; ok { if i > 0 { fieldPrefix.WriteString(n.f.Delimiter) } fieldPrefix.WriteString(v) } else { n.incrementErrorCount() n.logger.Printf("E! point missing tag %q for flatten operation", tag) continue POINTS } } l := fieldPrefix.Len() for fname, value := range p.Fields() { if !n.f.DropOriginalFieldNameFlag { if l > 0 { fieldPrefix.WriteString(n.f.Delimiter) } fieldPrefix.WriteString(fname) } fields[fieldPrefix.String()] = value fieldPrefix.Truncate(l) } fieldPrefix.Reset() } return fields, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/gobuild.sh ================================================ #!/bin/bash # This script run inside the Dockerfile_build_ubuntu64_git container and # gets the latests Go source code and compiles it. # Then passes control over to the normal build.py script set -e cd /go/src git fetch --all git checkout $GO_CHECKOUT # Merge in recent changes if we are on a branch # if we checked out a tag just ignore the error git pull || true ./make.bash # Run normal build.py cd "$PROJECT_DIR" exec ./build.py "$@" ================================================ FILE: vendor/github.com/influxdata/kapacitor/group_by.go ================================================ package kapacitor import ( "log" "sort" "sync" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick/ast" ) type GroupByNode struct { node g *pipeline.GroupByNode byName bool tagNames []string begin edge.BeginBatchMessage dimensions models.Dimensions allDimensions bool mu sync.RWMutex lastTime time.Time groups map[models.GroupID]edge.BufferedBatchMessage } // Create a new GroupByNode which splits the stream dynamically based on the specified dimensions. func newGroupByNode(et *ExecutingTask, n *pipeline.GroupByNode, l *log.Logger) (*GroupByNode, error) { gn := &GroupByNode{ node: node{Node: n, et: et, logger: l}, g: n, groups: make(map[models.GroupID]edge.BufferedBatchMessage), } gn.node.runF = gn.runGroupBy gn.allDimensions, gn.tagNames = determineTagNames(n.Dimensions, n.ExcludedDimensions) gn.byName = n.ByMeasurementFlag return gn, nil } func (n *GroupByNode) runGroupBy([]byte) error { valueF := func() int64 { n.mu.RLock() l := len(n.groups) n.mu.RUnlock() return int64(l) } n.statMap.Set(statCardinalityGauge, expvar.NewIntFuncGauge(valueF)) consumer := edge.NewConsumerWithReceiver( n.ins[0], n, ) return consumer.Consume() } func (n *GroupByNode) Point(p edge.PointMessage) error { p = p.ShallowCopy() n.timer.Start() dims := p.Dimensions() dims.ByName = dims.ByName || n.byName dims.TagNames = computeTagNames(p.Tags(), n.allDimensions, n.tagNames, n.g.ExcludedDimensions) p.SetDimensions(dims) n.timer.Stop() if err := edge.Forward(n.outs, p); err != nil { return err } return nil } func (n *GroupByNode) BeginBatch(begin edge.BeginBatchMessage) error { n.timer.Start() defer n.timer.Stop() n.emit(begin.Time()) n.begin = begin n.dimensions = begin.Dimensions() n.dimensions.ByName = n.dimensions.ByName || n.byName return nil } func (n *GroupByNode) BatchPoint(bp edge.BatchPointMessage) error { n.timer.Start() defer n.timer.Stop() n.dimensions.TagNames = computeTagNames(bp.Tags(), n.allDimensions, n.tagNames, n.g.ExcludedDimensions) groupID := models.ToGroupID(n.begin.Name(), bp.Tags(), n.dimensions) group, ok := n.groups[groupID] if !ok { // Create new begin message newBegin := n.begin.ShallowCopy() newBegin.SetTagsAndDimensions(bp.Tags(), n.dimensions) // Create buffer for group batch group = edge.NewBufferedBatchMessage( newBegin, make([]edge.BatchPointMessage, 0, newBegin.SizeHint()), edge.NewEndBatchMessage(), ) n.mu.Lock() n.groups[groupID] = group n.mu.Unlock() } group.SetPoints(append(group.Points(), bp)) return nil } func (n *GroupByNode) EndBatch(end edge.EndBatchMessage) error { return nil } func (n *GroupByNode) Barrier(b edge.BarrierMessage) error { n.timer.Start() err := n.emit(b.Time()) n.timer.Stop() if err != nil { return err } return edge.Forward(n.outs, b) } func (n *GroupByNode) DeleteGroup(d edge.DeleteGroupMessage) error { return edge.Forward(n.outs, d) } // emit sends all groups before time t to children nodes. // The node timer must be started when calling this method. func (n *GroupByNode) emit(t time.Time) error { // TODO: ensure this time comparison works with barrier messages if !t.Equal(n.lastTime) { n.lastTime = t // Emit all groups for id, group := range n.groups { // Update SizeHint since we know the final point count group.Begin().SetSizeHint(len(group.Points())) // Sort points since we didn't guarantee insertion order was sorted sort.Sort(edge.BatchPointMessages(group.Points())) // Send group batch to all children n.timer.Pause() if err := edge.Forward(n.outs, group); err != nil { return err } n.timer.Resume() n.mu.Lock() // Remove from group delete(n.groups, id) n.mu.Unlock() } } return nil } func determineTagNames(dimensions []interface{}, excluded []string) (allDimensions bool, realDimensions []string) { for _, dim := range dimensions { switch d := dim.(type) { case string: realDimensions = append(realDimensions, d) case *ast.StarNode: allDimensions = true } } sort.Strings(realDimensions) realDimensions = filterExcludedTagNames(realDimensions, excluded) return } func filterExcludedTagNames(tagNames, excluded []string) []string { filtered := tagNames[0:0] for _, t := range tagNames { found := false for _, x := range excluded { if x == t { found = true break } } if !found { filtered = append(filtered, t) } } return filtered } func computeTagNames(tags models.Tags, allDimensions bool, tagNames, excluded []string) []string { if allDimensions { return filterExcludedTagNames(models.SortedKeys(tags), excluded) } return tagNames } ================================================ FILE: vendor/github.com/influxdata/kapacitor/http_out.go ================================================ package kapacitor import ( "encoding/json" "log" "net/http" "path" "sync" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/services/httpd" ) type HTTPOutNode struct { node c *pipeline.HTTPOutNode endpoint string mu sync.RWMutex routes []httpd.Route result *models.Result indexes []*httpOutGroup } // Create a new HTTPOutNode which caches the most recent item and exposes it over the HTTP API. func newHTTPOutNode(et *ExecutingTask, n *pipeline.HTTPOutNode, l *log.Logger) (*HTTPOutNode, error) { hn := &HTTPOutNode{ node: node{Node: n, et: et, logger: l}, c: n, result: new(models.Result), } et.registerOutput(hn.c.Endpoint, hn) hn.node.runF = hn.runOut hn.node.stopF = hn.stopOut return hn, nil } func (n *HTTPOutNode) Endpoint() string { return n.endpoint } func (n *HTTPOutNode) runOut([]byte) error { hndl := func(w http.ResponseWriter, req *http.Request) { n.mu.RLock() defer n.mu.RUnlock() if b, err := json.Marshal(n.result); err != nil { httpd.HttpError( w, err.Error(), true, http.StatusInternalServerError, ) } else { _, _ = w.Write(b) } } p := path.Join("/tasks/", n.et.Task.ID, n.c.Endpoint) r := []httpd.Route{{ Method: "GET", Pattern: p, HandlerFunc: hndl, }} n.endpoint = n.et.tm.HTTPDService.URL() + p n.mu.Lock() n.routes = r n.mu.Unlock() err := n.et.tm.HTTPDService.AddRoutes(r) if err != nil { return err } consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } // Update the result structure with a row. func (n *HTTPOutNode) updateResultWithRow(idx int, row *models.Row) { n.mu.Lock() defer n.mu.Unlock() if idx >= len(n.result.Series) { n.incrementErrorCount() n.logger.Printf("E! index out of range for row update %d", idx) return } n.result.Series[idx] = row } func (n *HTTPOutNode) stopOut() { n.mu.Lock() defer n.mu.Unlock() n.et.tm.HTTPDService.DelRoutes(n.routes) } func (n *HTTPOutNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup(group.ID)), ), nil } func (n *HTTPOutNode) newGroup(groupID models.GroupID) *httpOutGroup { n.mu.Lock() defer n.mu.Unlock() idx := len(n.result.Series) n.result.Series = append(n.result.Series, nil) g := &httpOutGroup{ n: n, idx: idx, buffer: new(edge.BatchBuffer), } n.indexes = append(n.indexes, g) return g } func (n *HTTPOutNode) deleteGroup(idx int) { n.mu.Lock() defer n.mu.Unlock() for _, g := range n.indexes[idx+1:] { g.idx-- } n.indexes = append(n.indexes[0:idx], n.indexes[idx+1:]...) n.result.Series = append(n.result.Series[0:idx], n.result.Series[idx+1:]...) } type httpOutGroup struct { n *HTTPOutNode id models.GroupID idx int buffer *edge.BatchBuffer } func (g *httpOutGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { return nil, g.buffer.BeginBatch(begin) } func (g *httpOutGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { return nil, g.buffer.BatchPoint(bp) } func (g *httpOutGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return g.BufferedBatch(g.buffer.BufferedBatchMessage(end)) } func (g *httpOutGroup) BufferedBatch(batch edge.BufferedBatchMessage) (edge.Message, error) { row := batch.ToRow() g.n.updateResultWithRow(g.idx, row) return batch, nil } func (g *httpOutGroup) Point(p edge.PointMessage) (edge.Message, error) { row := p.ToRow() g.n.updateResultWithRow(g.idx, row) return p, nil } func (g *httpOutGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *httpOutGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { g.n.deleteGroup(g.idx) return d, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/http_post.go ================================================ package kapacitor import ( "encoding/json" "fmt" "log" "net/http" "sync" "github.com/influxdata/kapacitor/bufpool" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/services/httppost" ) type HTTPPostNode struct { node c *pipeline.HTTPPostNode endpoint *httppost.Endpoint mu sync.RWMutex bp *bufpool.Pool } // Create a new HTTPPostNode which submits received items via POST to an HTTP endpoint func newHTTPPostNode(et *ExecutingTask, n *pipeline.HTTPPostNode, l *log.Logger) (*HTTPPostNode, error) { hn := &HTTPPostNode{ node: node{Node: n, et: et, logger: l}, c: n, bp: bufpool.New(), } // Should only ever be 0 or 1 from validation of n if len(n.URLs) == 1 { e := httppost.NewEndpoint(n.URLs[0], nil, httppost.BasicAuth{}) hn.endpoint = e } // Should only ever be 0 or 1 from validation of n if len(n.Endpoints) == 1 { endpointName := n.Endpoints[0] e, ok := et.tm.HTTPPostService.Endpoint(endpointName) if !ok { return nil, fmt.Errorf("endpoint '%s' does not exist", endpointName) } hn.endpoint = e } hn.node.runF = hn.runPost return hn, nil } func (n *HTTPPostNode) runPost([]byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *HTTPPostNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { g := &httpPostGroup{ n: n, buffer: new(edge.BatchBuffer), } return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, g), ), nil } type httpPostGroup struct { n *HTTPPostNode buffer *edge.BatchBuffer } func (g *httpPostGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { return nil, g.buffer.BeginBatch(begin) } func (g *httpPostGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { return nil, g.buffer.BatchPoint(bp) } func (g *httpPostGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return g.BufferedBatch(g.buffer.BufferedBatchMessage(end)) } func (g *httpPostGroup) BufferedBatch(batch edge.BufferedBatchMessage) (edge.Message, error) { row := batch.ToRow() g.n.postRow(row) return batch, nil } func (g *httpPostGroup) Point(p edge.PointMessage) (edge.Message, error) { row := p.ToRow() g.n.postRow(row) return p, nil } func (g *httpPostGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *httpPostGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (n *HTTPPostNode) postRow(row *models.Row) { result := new(models.Result) result.Series = []*models.Row{row} body := n.bp.Get() defer n.bp.Put(body) err := json.NewEncoder(body).Encode(result) if err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to marshal row data json: %v", err) return } req, err := n.endpoint.NewHTTPRequest(body) if err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to marshal row data json: %v", err) return } req.Header.Set("Content-Type", "application/json") for k, v := range n.c.Headers { req.Header.Set(k, v) } resp, err := http.DefaultClient.Do(req) if err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to POST row data: %v", err) return } resp.Body.Close() } ================================================ FILE: vendor/github.com/influxdata/kapacitor/influxdb_out.go ================================================ package kapacitor import ( "bytes" "log" "sync" "time" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/influxdb" "github.com/influxdata/kapacitor/pipeline" "github.com/pkg/errors" ) const ( statsInfluxDBPointsWritten = "points_written" statsInfluxDBWriteErrors = "write_errors" ) type InfluxDBOutNode struct { node i *pipeline.InfluxDBOutNode wb *writeBuffer pointsWritten *expvar.Int writeErrors *expvar.Int batchBuffer *edge.BatchBuffer } func newInfluxDBOutNode(et *ExecutingTask, n *pipeline.InfluxDBOutNode, l *log.Logger) (*InfluxDBOutNode, error) { if et.tm.InfluxDBService == nil { return nil, errors.New("no InfluxDB cluster configured cannot use the InfluxDBOutNode") } cli, err := et.tm.InfluxDBService.NewNamedClient(n.Cluster) if err != nil { return nil, errors.Wrap(err, "failed to get InfluxDB client") } in := &InfluxDBOutNode{ node: node{Node: n, et: et, logger: l}, i: n, wb: newWriteBuffer(int(n.Buffer), n.FlushInterval, cli), batchBuffer: new(edge.BatchBuffer), } in.node.runF = in.runOut in.node.stopF = in.stopOut in.wb.i = in return in, nil } func (n *InfluxDBOutNode) runOut([]byte) error { n.pointsWritten = &expvar.Int{} n.writeErrors = &expvar.Int{} n.statMap.Set(statsInfluxDBPointsWritten, n.pointsWritten) n.statMap.Set(statsInfluxDBWriteErrors, n.writeErrors) // Start the write buffer n.wb.start() // Create the database and retention policy if n.i.CreateFlag { err := func() error { cli, err := n.et.tm.InfluxDBService.NewNamedClient(n.i.Cluster) if err != nil { return err } var createDb bytes.Buffer createDb.WriteString("CREATE DATABASE ") createDb.WriteString(influxql.QuoteIdent(n.i.Database)) if n.i.RetentionPolicy != "" { createDb.WriteString(" WITH NAME ") createDb.WriteString(influxql.QuoteIdent(n.i.RetentionPolicy)) } _, err = cli.Query(influxdb.Query{Command: createDb.String()}) if err != nil { return err } return nil }() if err != nil { n.incrementErrorCount() n.logger.Printf("E! failed to create database %q on cluster %q: %v", n.i.Database, n.i.Cluster, err) } } // Setup consumer consumer := edge.NewConsumerWithReceiver( n.ins[0], edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n), ), ) return consumer.Consume() } func (n *InfluxDBOutNode) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { return nil, n.batchBuffer.BeginBatch(begin) } func (n *InfluxDBOutNode) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { return nil, n.batchBuffer.BatchPoint(bp) } func (n *InfluxDBOutNode) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return n.BufferedBatch(n.batchBuffer.BufferedBatchMessage(end)) } func (n *InfluxDBOutNode) BufferedBatch(batch edge.BufferedBatchMessage) (edge.Message, error) { n.write("", "", batch) return batch, nil } func (n *InfluxDBOutNode) Point(p edge.PointMessage) (edge.Message, error) { batch := edge.NewBufferedBatchMessage( edge.NewBeginBatchMessage( p.Name(), p.Tags(), p.Dimensions().ByName, p.Time(), 1, ), []edge.BatchPointMessage{ edge.NewBatchPointMessage( p.Fields(), p.Tags(), p.Time(), ), }, edge.NewEndBatchMessage(), ) n.write(p.Database(), p.RetentionPolicy(), batch) return p, nil } func (n *InfluxDBOutNode) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *InfluxDBOutNode) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (n *InfluxDBOutNode) stopOut() { n.wb.flush() n.wb.abort() } func (n *InfluxDBOutNode) write(db, rp string, batch edge.BufferedBatchMessage) error { if n.i.Database != "" { db = n.i.Database } if n.i.RetentionPolicy != "" { rp = n.i.RetentionPolicy } name := n.i.Measurement if name == "" { name = batch.Name() } points := make([]influxdb.Point, len(batch.Points())) for j, p := range batch.Points() { var tags map[string]string if len(n.i.Tags) > 0 { tags = make(map[string]string, len(p.Tags())+len(n.i.Tags)) for k, v := range p.Tags() { tags[k] = v } for k, v := range n.i.Tags { tags[k] = v } } else { tags = p.Tags() } points[j] = influxdb.Point{ Name: name, Tags: tags, Fields: p.Fields(), Time: p.Time(), } } bpc := influxdb.BatchPointsConfig{ Database: db, RetentionPolicy: rp, WriteConsistency: n.i.WriteConsistency, Precision: n.i.Precision, } n.wb.enqueue(bpc, points) return nil } type writeBuffer struct { size int flushInterval time.Duration errC chan error queue chan queueEntry buffer map[influxdb.BatchPointsConfig]influxdb.BatchPoints flushing chan struct{} flushed chan struct{} stopping chan struct{} wg sync.WaitGroup cli influxdb.Client i *InfluxDBOutNode } type queueEntry struct { bpc influxdb.BatchPointsConfig points []influxdb.Point } func newWriteBuffer(size int, flushInterval time.Duration, cli influxdb.Client) *writeBuffer { return &writeBuffer{ cli: cli, size: size, flushInterval: flushInterval, flushing: make(chan struct{}), flushed: make(chan struct{}), queue: make(chan queueEntry), buffer: make(map[influxdb.BatchPointsConfig]influxdb.BatchPoints), stopping: make(chan struct{}), } } func (w *writeBuffer) enqueue(bpc influxdb.BatchPointsConfig, points []influxdb.Point) { qe := queueEntry{ bpc: bpc, points: points, } select { case w.queue <- qe: case <-w.stopping: } } func (w *writeBuffer) start() { w.wg.Add(1) go w.run() } func (w *writeBuffer) flush() { w.flushing <- struct{}{} <-w.flushed } func (w *writeBuffer) abort() { close(w.stopping) w.wg.Wait() } func (w *writeBuffer) run() { defer w.wg.Done() flushTick := time.NewTicker(w.flushInterval) defer flushTick.Stop() var err error for { select { case qe := <-w.queue: // Read incoming points off queue bp, ok := w.buffer[qe.bpc] if !ok { bp, err = influxdb.NewBatchPoints(qe.bpc) if err != nil { w.i.incrementErrorCount() w.i.logger.Println("E! failed to write points to InfluxDB:", err) break } w.buffer[qe.bpc] = bp } bp.AddPoints(qe.points) // Check if we hit buffer size if len(bp.Points()) >= w.size { err = w.write(bp) if err != nil { w.i.incrementErrorCount() w.i.logger.Println("E! failed to write points to InfluxDB:", err) } delete(w.buffer, qe.bpc) } case <-w.flushing: // Explicit flush called w.writeAll() w.flushed <- struct{}{} case <-flushTick.C: // Flush all points after flush interval timeout w.writeAll() case <-w.stopping: return } } } func (w *writeBuffer) writeAll() { for bpc, bp := range w.buffer { err := w.write(bp) if err != nil { w.i.incrementErrorCount() w.i.logger.Println("E! failed to write points to InfluxDB:", err) } delete(w.buffer, bpc) } } func (w *writeBuffer) write(bp influxdb.BatchPoints) error { err := w.cli.Write(bp) if err != nil { w.i.writeErrors.Add(1) return err } w.i.pointsWritten.Add(int64(len(bp.Points()))) return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/influxql.gen.go ================================================ // Generated by tmpl // https://github.com/benbjohnson/tmpl // // DO NOT EDIT! // Source: influxql.gen.go.tmpl package kapacitor import ( "fmt" "reflect" "time" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) func convertFloatPoint( name string, p edge.FieldsTagsTimeGetter, field string, isSimpleSelector bool, topBottomInfo *pipeline.TopBottomCallInfo, ) (*influxql.FloatPoint, error) { value, ok := p.Fields()[field] if !ok { return nil, fmt.Errorf("field %s missing from point cannot aggregate", field) } typed, ok := value.(float64) if !ok { return nil, fmt.Errorf("field %s has wrong type: got %T exp float64", field, value) } ap := &influxql.FloatPoint{ Name: name, Tags: influxql.NewTags(p.Tags()), Time: p.Time().UnixNano(), Value: typed, } if topBottomInfo != nil { // We need to populate the Aux fields floatPopulateAuxFieldsAndTags(ap, topBottomInfo.FieldsAndTags, p.Fields(), p.Tags()) } if isSimpleSelector { ap.Aux = []interface{}{p.Tags(), p.Fields()} } return ap, nil } type floatPointAggregator struct { field string topBottomInfo *pipeline.TopBottomCallInfo isSimpleSelector bool aggregator influxql.FloatPointAggregator } func floatPopulateAuxFieldsAndTags(ap *influxql.FloatPoint, fieldsAndTags []string, fields models.Fields, tags models.Tags) { ap.Aux = make([]interface{}, len(fieldsAndTags)) for i, name := range fieldsAndTags { if f, ok := fields[name]; ok { ap.Aux[i] = f } else { ap.Aux[i] = tags[name] } } } func (a *floatPointAggregator) AggregatePoint(name string, p edge.FieldsTagsTimeGetter) error { ap, err := convertFloatPoint(name, p, a.field, a.isSimpleSelector, a.topBottomInfo) if err != nil { return nil } a.aggregator.AggregateFloat(ap) return nil } type floatPointEmitter struct { baseReduceContext emitter influxql.FloatPointEmitter isSimpleSelector bool byName bool } func (e *floatPointEmitter) EmitPoint() (edge.PointMessage, error) { slice := e.emitter.Emit() if len(slice) != 1 { return nil, nil } ap := slice[0] var t time.Time if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var fields models.Fields var tags models.Tags if e.isSimpleSelector { tags = ap.Aux[0].(models.Tags) fields = ap.Aux[1].(models.Fields) if e.as != e.field { fields = fields.Copy() fields[e.as] = fields[e.field] delete(fields, e.field) } } else { tags = e.groupInfo.Tags fields = map[string]interface{}{e.as: ap.Value} } return edge.NewPointMessage( e.name, "", "", e.groupInfo.Dimensions, fields, tags, t, ), nil } func (e *floatPointEmitter) EmitBatch() edge.BufferedBatchMessage { slice := e.emitter.Emit() begin := edge.NewBeginBatchMessage( e.name, e.groupInfo.Tags, e.groupInfo.Dimensions.ByName, e.time, len(slice), ) points := make([]edge.BatchPointMessage, len(slice)) var t time.Time for i, ap := range slice { if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var tags models.Tags if l := len(ap.Tags.KeyValues()); l > 0 { // Merge batch and point specific tags tags = make(models.Tags, len(e.groupInfo.Tags)+l) for k, v := range e.groupInfo.Tags { tags[k] = v } for k, v := range ap.Tags.KeyValues() { tags[k] = v } } else { tags = e.groupInfo.Tags } points[i] = edge.NewBatchPointMessage( models.Fields{e.as: ap.Value}, tags, t, ) if t.After(begin.Time()) { begin.SetTime(t) } } return edge.NewBufferedBatchMessage( begin, points, edge.NewEndBatchMessage(), ) } func convertIntegerPoint( name string, p edge.FieldsTagsTimeGetter, field string, isSimpleSelector bool, topBottomInfo *pipeline.TopBottomCallInfo, ) (*influxql.IntegerPoint, error) { value, ok := p.Fields()[field] if !ok { return nil, fmt.Errorf("field %s missing from point cannot aggregate", field) } typed, ok := value.(int64) if !ok { return nil, fmt.Errorf("field %s has wrong type: got %T exp int64", field, value) } ap := &influxql.IntegerPoint{ Name: name, Tags: influxql.NewTags(p.Tags()), Time: p.Time().UnixNano(), Value: typed, } if topBottomInfo != nil { // We need to populate the Aux fields integerPopulateAuxFieldsAndTags(ap, topBottomInfo.FieldsAndTags, p.Fields(), p.Tags()) } if isSimpleSelector { ap.Aux = []interface{}{p.Tags(), p.Fields()} } return ap, nil } type integerPointAggregator struct { field string topBottomInfo *pipeline.TopBottomCallInfo isSimpleSelector bool aggregator influxql.IntegerPointAggregator } func integerPopulateAuxFieldsAndTags(ap *influxql.IntegerPoint, fieldsAndTags []string, fields models.Fields, tags models.Tags) { ap.Aux = make([]interface{}, len(fieldsAndTags)) for i, name := range fieldsAndTags { if f, ok := fields[name]; ok { ap.Aux[i] = f } else { ap.Aux[i] = tags[name] } } } func (a *integerPointAggregator) AggregatePoint(name string, p edge.FieldsTagsTimeGetter) error { ap, err := convertIntegerPoint(name, p, a.field, a.isSimpleSelector, a.topBottomInfo) if err != nil { return nil } a.aggregator.AggregateInteger(ap) return nil } type integerPointEmitter struct { baseReduceContext emitter influxql.IntegerPointEmitter isSimpleSelector bool byName bool } func (e *integerPointEmitter) EmitPoint() (edge.PointMessage, error) { slice := e.emitter.Emit() if len(slice) != 1 { return nil, nil } ap := slice[0] var t time.Time if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var fields models.Fields var tags models.Tags if e.isSimpleSelector { tags = ap.Aux[0].(models.Tags) fields = ap.Aux[1].(models.Fields) if e.as != e.field { fields = fields.Copy() fields[e.as] = fields[e.field] delete(fields, e.field) } } else { tags = e.groupInfo.Tags fields = map[string]interface{}{e.as: ap.Value} } return edge.NewPointMessage( e.name, "", "", e.groupInfo.Dimensions, fields, tags, t, ), nil } func (e *integerPointEmitter) EmitBatch() edge.BufferedBatchMessage { slice := e.emitter.Emit() begin := edge.NewBeginBatchMessage( e.name, e.groupInfo.Tags, e.groupInfo.Dimensions.ByName, e.time, len(slice), ) points := make([]edge.BatchPointMessage, len(slice)) var t time.Time for i, ap := range slice { if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var tags models.Tags if l := len(ap.Tags.KeyValues()); l > 0 { // Merge batch and point specific tags tags = make(models.Tags, len(e.groupInfo.Tags)+l) for k, v := range e.groupInfo.Tags { tags[k] = v } for k, v := range ap.Tags.KeyValues() { tags[k] = v } } else { tags = e.groupInfo.Tags } points[i] = edge.NewBatchPointMessage( models.Fields{e.as: ap.Value}, tags, t, ) if t.After(begin.Time()) { begin.SetTime(t) } } return edge.NewBufferedBatchMessage( begin, points, edge.NewEndBatchMessage(), ) } func convertStringPoint( name string, p edge.FieldsTagsTimeGetter, field string, isSimpleSelector bool, topBottomInfo *pipeline.TopBottomCallInfo, ) (*influxql.StringPoint, error) { value, ok := p.Fields()[field] if !ok { return nil, fmt.Errorf("field %s missing from point cannot aggregate", field) } typed, ok := value.(string) if !ok { return nil, fmt.Errorf("field %s has wrong type: got %T exp string", field, value) } ap := &influxql.StringPoint{ Name: name, Tags: influxql.NewTags(p.Tags()), Time: p.Time().UnixNano(), Value: typed, } if topBottomInfo != nil { // We need to populate the Aux fields stringPopulateAuxFieldsAndTags(ap, topBottomInfo.FieldsAndTags, p.Fields(), p.Tags()) } if isSimpleSelector { ap.Aux = []interface{}{p.Tags(), p.Fields()} } return ap, nil } type stringPointAggregator struct { field string topBottomInfo *pipeline.TopBottomCallInfo isSimpleSelector bool aggregator influxql.StringPointAggregator } func stringPopulateAuxFieldsAndTags(ap *influxql.StringPoint, fieldsAndTags []string, fields models.Fields, tags models.Tags) { ap.Aux = make([]interface{}, len(fieldsAndTags)) for i, name := range fieldsAndTags { if f, ok := fields[name]; ok { ap.Aux[i] = f } else { ap.Aux[i] = tags[name] } } } func (a *stringPointAggregator) AggregatePoint(name string, p edge.FieldsTagsTimeGetter) error { ap, err := convertStringPoint(name, p, a.field, a.isSimpleSelector, a.topBottomInfo) if err != nil { return nil } a.aggregator.AggregateString(ap) return nil } type stringPointEmitter struct { baseReduceContext emitter influxql.StringPointEmitter isSimpleSelector bool byName bool } func (e *stringPointEmitter) EmitPoint() (edge.PointMessage, error) { slice := e.emitter.Emit() if len(slice) != 1 { return nil, nil } ap := slice[0] var t time.Time if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var fields models.Fields var tags models.Tags if e.isSimpleSelector { tags = ap.Aux[0].(models.Tags) fields = ap.Aux[1].(models.Fields) if e.as != e.field { fields = fields.Copy() fields[e.as] = fields[e.field] delete(fields, e.field) } } else { tags = e.groupInfo.Tags fields = map[string]interface{}{e.as: ap.Value} } return edge.NewPointMessage( e.name, "", "", e.groupInfo.Dimensions, fields, tags, t, ), nil } func (e *stringPointEmitter) EmitBatch() edge.BufferedBatchMessage { slice := e.emitter.Emit() begin := edge.NewBeginBatchMessage( e.name, e.groupInfo.Tags, e.groupInfo.Dimensions.ByName, e.time, len(slice), ) points := make([]edge.BatchPointMessage, len(slice)) var t time.Time for i, ap := range slice { if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var tags models.Tags if l := len(ap.Tags.KeyValues()); l > 0 { // Merge batch and point specific tags tags = make(models.Tags, len(e.groupInfo.Tags)+l) for k, v := range e.groupInfo.Tags { tags[k] = v } for k, v := range ap.Tags.KeyValues() { tags[k] = v } } else { tags = e.groupInfo.Tags } points[i] = edge.NewBatchPointMessage( models.Fields{e.as: ap.Value}, tags, t, ) if t.After(begin.Time()) { begin.SetTime(t) } } return edge.NewBufferedBatchMessage( begin, points, edge.NewEndBatchMessage(), ) } func convertBooleanPoint( name string, p edge.FieldsTagsTimeGetter, field string, isSimpleSelector bool, topBottomInfo *pipeline.TopBottomCallInfo, ) (*influxql.BooleanPoint, error) { value, ok := p.Fields()[field] if !ok { return nil, fmt.Errorf("field %s missing from point cannot aggregate", field) } typed, ok := value.(bool) if !ok { return nil, fmt.Errorf("field %s has wrong type: got %T exp bool", field, value) } ap := &influxql.BooleanPoint{ Name: name, Tags: influxql.NewTags(p.Tags()), Time: p.Time().UnixNano(), Value: typed, } if topBottomInfo != nil { // We need to populate the Aux fields booleanPopulateAuxFieldsAndTags(ap, topBottomInfo.FieldsAndTags, p.Fields(), p.Tags()) } if isSimpleSelector { ap.Aux = []interface{}{p.Tags(), p.Fields()} } return ap, nil } type booleanPointAggregator struct { field string topBottomInfo *pipeline.TopBottomCallInfo isSimpleSelector bool aggregator influxql.BooleanPointAggregator } func booleanPopulateAuxFieldsAndTags(ap *influxql.BooleanPoint, fieldsAndTags []string, fields models.Fields, tags models.Tags) { ap.Aux = make([]interface{}, len(fieldsAndTags)) for i, name := range fieldsAndTags { if f, ok := fields[name]; ok { ap.Aux[i] = f } else { ap.Aux[i] = tags[name] } } } func (a *booleanPointAggregator) AggregatePoint(name string, p edge.FieldsTagsTimeGetter) error { ap, err := convertBooleanPoint(name, p, a.field, a.isSimpleSelector, a.topBottomInfo) if err != nil { return nil } a.aggregator.AggregateBoolean(ap) return nil } type booleanPointEmitter struct { baseReduceContext emitter influxql.BooleanPointEmitter isSimpleSelector bool byName bool } func (e *booleanPointEmitter) EmitPoint() (edge.PointMessage, error) { slice := e.emitter.Emit() if len(slice) != 1 { return nil, nil } ap := slice[0] var t time.Time if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var fields models.Fields var tags models.Tags if e.isSimpleSelector { tags = ap.Aux[0].(models.Tags) fields = ap.Aux[1].(models.Fields) if e.as != e.field { fields = fields.Copy() fields[e.as] = fields[e.field] delete(fields, e.field) } } else { tags = e.groupInfo.Tags fields = map[string]interface{}{e.as: ap.Value} } return edge.NewPointMessage( e.name, "", "", e.groupInfo.Dimensions, fields, tags, t, ), nil } func (e *booleanPointEmitter) EmitBatch() edge.BufferedBatchMessage { slice := e.emitter.Emit() begin := edge.NewBeginBatchMessage( e.name, e.groupInfo.Tags, e.groupInfo.Dimensions.ByName, e.time, len(slice), ) points := make([]edge.BatchPointMessage, len(slice)) var t time.Time for i, ap := range slice { if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var tags models.Tags if l := len(ap.Tags.KeyValues()); l > 0 { // Merge batch and point specific tags tags = make(models.Tags, len(e.groupInfo.Tags)+l) for k, v := range e.groupInfo.Tags { tags[k] = v } for k, v := range ap.Tags.KeyValues() { tags[k] = v } } else { tags = e.groupInfo.Tags } points[i] = edge.NewBatchPointMessage( models.Fields{e.as: ap.Value}, tags, t, ) if t.After(begin.Time()) { begin.SetTime(t) } } return edge.NewBufferedBatchMessage( begin, points, edge.NewEndBatchMessage(), ) } // floatReduceContext uses composition to implement the reduceContext interface type floatReduceContext struct { floatPointAggregator floatPointEmitter } // floatIntegerReduceContext uses composition to implement the reduceContext interface type floatIntegerReduceContext struct { floatPointAggregator integerPointEmitter } // floatStringReduceContext uses composition to implement the reduceContext interface type floatStringReduceContext struct { floatPointAggregator stringPointEmitter } // floatBooleanReduceContext uses composition to implement the reduceContext interface type floatBooleanReduceContext struct { floatPointAggregator booleanPointEmitter } // integerFloatReduceContext uses composition to implement the reduceContext interface type integerFloatReduceContext struct { integerPointAggregator floatPointEmitter } // integerReduceContext uses composition to implement the reduceContext interface type integerReduceContext struct { integerPointAggregator integerPointEmitter } // integerStringReduceContext uses composition to implement the reduceContext interface type integerStringReduceContext struct { integerPointAggregator stringPointEmitter } // integerBooleanReduceContext uses composition to implement the reduceContext interface type integerBooleanReduceContext struct { integerPointAggregator booleanPointEmitter } // stringFloatReduceContext uses composition to implement the reduceContext interface type stringFloatReduceContext struct { stringPointAggregator floatPointEmitter } // stringIntegerReduceContext uses composition to implement the reduceContext interface type stringIntegerReduceContext struct { stringPointAggregator integerPointEmitter } // stringReduceContext uses composition to implement the reduceContext interface type stringReduceContext struct { stringPointAggregator stringPointEmitter } // stringBooleanReduceContext uses composition to implement the reduceContext interface type stringBooleanReduceContext struct { stringPointAggregator booleanPointEmitter } // booleanFloatReduceContext uses composition to implement the reduceContext interface type booleanFloatReduceContext struct { booleanPointAggregator floatPointEmitter } // booleanIntegerReduceContext uses composition to implement the reduceContext interface type booleanIntegerReduceContext struct { booleanPointAggregator integerPointEmitter } // booleanStringReduceContext uses composition to implement the reduceContext interface type booleanStringReduceContext struct { booleanPointAggregator stringPointEmitter } // booleanReduceContext uses composition to implement the reduceContext interface type booleanReduceContext struct { booleanPointAggregator booleanPointEmitter } func determineReduceContextCreateFn(method string, kind reflect.Kind, rc pipeline.ReduceCreater) (fn createReduceContextFunc, err error) { switch kind { case reflect.Float64: switch { case rc.CreateFloatReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateFloatReducer() return &floatReduceContext{ floatPointAggregator: floatPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, floatPointEmitter: floatPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateFloatIntegerReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateFloatIntegerReducer() return &floatIntegerReduceContext{ floatPointAggregator: floatPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, integerPointEmitter: integerPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateFloatStringReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateFloatStringReducer() return &floatStringReduceContext{ floatPointAggregator: floatPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, stringPointEmitter: stringPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateFloatBooleanReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateFloatBooleanReducer() return &floatBooleanReduceContext{ floatPointAggregator: floatPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, booleanPointEmitter: booleanPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } default: err = fmt.Errorf("cannot apply %s to float64 field", method) } case reflect.Int64: switch { case rc.CreateIntegerFloatReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateIntegerFloatReducer() return &integerFloatReduceContext{ integerPointAggregator: integerPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, floatPointEmitter: floatPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateIntegerReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateIntegerReducer() return &integerReduceContext{ integerPointAggregator: integerPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, integerPointEmitter: integerPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateIntegerStringReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateIntegerStringReducer() return &integerStringReduceContext{ integerPointAggregator: integerPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, stringPointEmitter: stringPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateIntegerBooleanReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateIntegerBooleanReducer() return &integerBooleanReduceContext{ integerPointAggregator: integerPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, booleanPointEmitter: booleanPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } default: err = fmt.Errorf("cannot apply %s to int64 field", method) } case reflect.String: switch { case rc.CreateStringFloatReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateStringFloatReducer() return &stringFloatReduceContext{ stringPointAggregator: stringPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, floatPointEmitter: floatPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateStringIntegerReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateStringIntegerReducer() return &stringIntegerReduceContext{ stringPointAggregator: stringPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, integerPointEmitter: integerPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateStringReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateStringReducer() return &stringReduceContext{ stringPointAggregator: stringPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, stringPointEmitter: stringPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateStringBooleanReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateStringBooleanReducer() return &stringBooleanReduceContext{ stringPointAggregator: stringPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, booleanPointEmitter: booleanPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } default: err = fmt.Errorf("cannot apply %s to string field", method) } case reflect.Bool: switch { case rc.CreateBooleanFloatReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateBooleanFloatReducer() return &booleanFloatReduceContext{ booleanPointAggregator: booleanPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, floatPointEmitter: floatPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateBooleanIntegerReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateBooleanIntegerReducer() return &booleanIntegerReduceContext{ booleanPointAggregator: booleanPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, integerPointEmitter: integerPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateBooleanStringReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateBooleanStringReducer() return &booleanStringReduceContext{ booleanPointAggregator: booleanPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, stringPointEmitter: stringPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } case rc.CreateBooleanReducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.CreateBooleanReducer() return &booleanReduceContext{ booleanPointAggregator: booleanPointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, booleanPointEmitter: booleanPointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } default: err = fmt.Errorf("cannot apply %s to bool field", method) } default: err = fmt.Errorf("invalid field kind: %v", kind) } return } ================================================ FILE: vendor/github.com/influxdata/kapacitor/influxql.gen.go.tmpl ================================================ package kapacitor import ( "fmt" "time" "reflect" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" ) {{/* Define typed Aggregate/Emit types */}} {{range .}} func convert{{.Name}}Point( name string, p edge.FieldsTagsTimeGetter, field string, isSimpleSelector bool, topBottomInfo *pipeline.TopBottomCallInfo, ) (*influxql.{{.Name}}Point, error) { value, ok := p.Fields()[field] if !ok { return nil, fmt.Errorf("field %s missing from point cannot aggregate", field) } typed, ok := value.({{.Type}}) if !ok { return nil, fmt.Errorf("field %s has wrong type: got %T exp {{.Type}}", field, value) } ap := &influxql.{{.Name}}Point{ Name: name, Tags: influxql.NewTags(p.Tags()), Time: p.Time().UnixNano(), Value: typed, } if topBottomInfo != nil { // We need to populate the Aux fields {{.name}}PopulateAuxFieldsAndTags(ap, topBottomInfo.FieldsAndTags, p.Fields(), p.Tags()) } if isSimpleSelector { ap.Aux = []interface{}{ p.Tags(), p.Fields() } } return ap, nil } type {{.name}}PointAggregator struct { field string topBottomInfo *pipeline.TopBottomCallInfo isSimpleSelector bool aggregator influxql.{{.Name}}PointAggregator } func {{.name}}PopulateAuxFieldsAndTags(ap *influxql.{{.Name}}Point, fieldsAndTags []string, fields models.Fields, tags models.Tags) { ap.Aux = make([]interface{}, len(fieldsAndTags)) for i, name := range fieldsAndTags { if f, ok := fields[name]; ok { ap.Aux[i] = f } else { ap.Aux[i] = tags[name] } } } func (a *{{.name}}PointAggregator) AggregatePoint(name string, p edge.FieldsTagsTimeGetter) error { ap, err := convert{{.Name}}Point(name, p, a.field, a.isSimpleSelector, a.topBottomInfo) if err != nil { return nil } a.aggregator.Aggregate{{.Name}}(ap) return nil } type {{.name}}PointEmitter struct { baseReduceContext emitter influxql.{{.Name}}PointEmitter isSimpleSelector bool byName bool } func (e *{{.name}}PointEmitter) EmitPoint() (edge.PointMessage, error) { slice := e.emitter.Emit() if len(slice) != 1 { return nil, nil } ap := slice[0] var t time.Time if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var fields models.Fields var tags models.Tags if e.isSimpleSelector { tags = ap.Aux[0].(models.Tags) fields = ap.Aux[1].(models.Fields) if e.as != e.field { fields = fields.Copy() fields[e.as] = fields[e.field] delete(fields, e.field) } } else { tags = e.groupInfo.Tags fields = map[string]interface{}{e.as: ap.Value} } return edge.NewPointMessage( e.name, "", "", e.groupInfo.Dimensions, fields, tags, t, ), nil } func (e *{{.name}}PointEmitter) EmitBatch() edge.BufferedBatchMessage { slice := e.emitter.Emit() begin := edge.NewBeginBatchMessage( e.name, e.groupInfo.Tags, e.groupInfo.Dimensions.ByName, e.time, len(slice), ) points := make([]edge.BatchPointMessage, len(slice)) var t time.Time for i, ap := range slice { if e.pointTimes { if ap.Time == influxql.ZeroTime { t = e.time } else { t = time.Unix(0, ap.Time).UTC() } } else { t = e.time } var tags models.Tags if l := len(ap.Tags.KeyValues()); l > 0 { // Merge batch and point specific tags tags = make(models.Tags, len(e.groupInfo.Tags)+l) for k, v := range e.groupInfo.Tags { tags[k] = v } for k, v := range ap.Tags.KeyValues() { tags[k] = v } } else { tags = e.groupInfo.Tags } points[i] = edge.NewBatchPointMessage( models.Fields{e.as: ap.Value}, tags, t, ) if t.After(begin.Time()) { begin.SetTime(t) } } return edge.NewBufferedBatchMessage( begin, points, edge.NewEndBatchMessage(), ) } {{end}} {{/* Define composite types for reduceContext */}} {{with $types := .}} {{range $a := $types}} {{range $e := $types}} // {{$a.name}}{{if ne $a.Name $e.Name}}{{$e.Name}}{{end}}ReduceContext uses composition to implement the reduceContext interface type {{$a.name}}{{if ne $a.Name $e.Name}}{{$e.Name}}{{end}}ReduceContext struct { {{$a.name}}PointAggregator {{$e.name}}PointEmitter } {{end}}{{end}} {{/* Define switch cases for reduceContext contruction */}} func determineReduceContextCreateFn(method string, kind reflect.Kind, rc pipeline.ReduceCreater) (fn createReduceContextFunc, err error) { switch kind { {{range $a := $types}} case {{.Kind}}: switch { {{range $e := $types}} case rc.Create{{$a.Name}}{{if ne $a.Name $e.Name}}{{$e.Name}}{{end}}Reducer != nil: fn = func(c baseReduceContext) reduceContext { a, e := rc.Create{{$a.Name}}{{if ne $a.Name $e.Name}}{{$e.Name}}{{end}}Reducer() return &{{$a.name}}{{if ne $a.Name $e.Name}}{{$e.Name}}{{end}}ReduceContext{ {{$a.name}}PointAggregator: {{$a.name}}PointAggregator{ field: c.field, topBottomInfo: rc.TopBottomCallInfo, isSimpleSelector: rc.IsSimpleSelector, aggregator: a, }, {{$e.name}}PointEmitter: {{$e.name}}PointEmitter{ baseReduceContext: c, emitter: e, isSimpleSelector: rc.IsSimpleSelector, }, } } {{end}} default: err = fmt.Errorf("cannot apply %s to {{$a.Type}} field", method) } {{end}} default: err = fmt.Errorf("invalid field kind: %v", kind) } return } {{end}} ================================================ FILE: vendor/github.com/influxdata/kapacitor/influxql.go ================================================ package kapacitor import ( "fmt" "log" "reflect" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/pkg/errors" ) // tmpl -- go get github.com/benbjohnson/tmpl //go:generate tmpl -data=@tmpldata.json influxql.gen.go.tmpl type createReduceContextFunc func(c baseReduceContext) reduceContext type InfluxQLNode struct { node n *pipeline.InfluxQLNode createFn createReduceContextFunc isStreamTransformation bool currentKind reflect.Kind } func newInfluxQLNode(et *ExecutingTask, n *pipeline.InfluxQLNode, l *log.Logger) (*InfluxQLNode, error) { m := &InfluxQLNode{ node: node{Node: n, et: et, logger: l}, n: n, isStreamTransformation: n.ReduceCreater.IsStreamTransformation, } m.node.runF = m.runInfluxQL return m, nil } type reduceContext interface { AggregatePoint(name string, p edge.FieldsTagsTimeGetter) error EmitPoint() (edge.PointMessage, error) EmitBatch() edge.BufferedBatchMessage } type baseReduceContext struct { as string field string name string groupInfo edge.GroupInfo time time.Time pointTimes bool topBottomInfo *pipeline.TopBottomCallInfo } func (n *InfluxQLNode) runInfluxQL([]byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *InfluxQLNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup(first)), ), nil } func (n *InfluxQLNode) newGroup(first edge.PointMeta) edge.ForwardReceiver { bc := baseReduceContext{ as: n.n.As, field: n.n.Field, name: first.Name(), groupInfo: first.GroupInfo(), time: first.Time(), pointTimes: n.n.PointTimes || n.isStreamTransformation, } g := influxqlGroup{ n: n, bc: bc, } if n.isStreamTransformation { return &influxqlStreamingTransformGroup{ influxqlGroup: g, } } return &g } type influxqlGroup struct { n *InfluxQLNode bc baseReduceContext rc reduceContext batchSize int name string begin edge.BeginBatchMessage } func (g *influxqlGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { g.begin = begin g.batchSize = 0 g.bc.time = begin.Time() g.rc = nil return nil, nil } func (g *influxqlGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { if g.rc == nil { if err := g.realizeReduceContextFromFields(bp.Fields()); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E!", err) return nil, nil } } g.batchSize++ if err := g.rc.AggregatePoint(g.begin.Name(), bp); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to aggregate point in batch:", err) } return nil, nil } func (g *influxqlGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { if g.batchSize == 0 && !g.n.n.ReduceCreater.IsEmptyOK { // Do not call Emit on the reducer since it can't handle empty batches. return nil, nil } if g.rc == nil { // Assume float64 type since we do not have any data. if err := g.realizeReduceContext(reflect.Float64); err != nil { return nil, err } } m, err := g.n.emit(g.rc) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to emit batch:", err) return nil, nil } return m, nil } func (g *influxqlGroup) Point(p edge.PointMessage) (edge.Message, error) { if p.Time().Equal(g.bc.time) { g.aggregatePoint(p) } else { // Time has elapsed, emit current context var msg edge.Message if g.rc != nil { m, err := g.n.emit(g.rc) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to emit stream:", err) } msg = m } // Reset context g.bc.name = p.Name() g.bc.time = p.Time() g.rc = nil // Aggregate the current point g.aggregatePoint(p) return msg, nil } return nil, nil } func (g *influxqlGroup) aggregatePoint(p edge.PointMessage) { if g.rc == nil { if err := g.realizeReduceContextFromFields(p.Fields()); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E!", err) return } } err := g.rc.AggregatePoint(p.Name(), p) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to aggregate point:", err) } } func (g *influxqlGroup) getFieldKind(fields models.Fields) (reflect.Kind, error) { f, exists := fields[g.bc.field] if !exists { return reflect.Invalid, fmt.Errorf("field %q missing from point", g.bc.field) } return reflect.TypeOf(f).Kind(), nil } func (g *influxqlGroup) realizeReduceContextFromFields(fields models.Fields) error { k, err := g.getFieldKind(fields) if err != nil { return err } return g.realizeReduceContext(k) } func (g *influxqlGroup) realizeReduceContext(kind reflect.Kind) error { createFn, err := g.n.getCreateFn(kind) if err != nil { return err } g.rc = createFn(g.bc) return nil } func (g *influxqlGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *influxqlGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } type influxqlStreamingTransformGroup struct { influxqlGroup } func (g *influxqlStreamingTransformGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { g.begin = begin.ShallowCopy() g.begin.SetSizeHint(0) g.bc.time = begin.Time() g.rc = nil return begin, nil } func (g *influxqlStreamingTransformGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { if g.rc == nil { if err := g.realizeReduceContextFromFields(bp.Fields()); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E!", err) return nil, nil } } if err := g.rc.AggregatePoint(g.begin.Name(), bp); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to aggregate batch point:", err) } if ep, err := g.rc.EmitPoint(); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to emit batch point:", err) } else if ep != nil { return edge.NewBatchPointMessage( ep.Fields(), ep.Tags(), ep.Time(), ), nil } return nil, nil } func (g *influxqlStreamingTransformGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (g *influxqlStreamingTransformGroup) Point(p edge.PointMessage) (edge.Message, error) { if g.rc == nil { if err := g.realizeReduceContextFromFields(p.Fields()); err != nil { g.n.incrementErrorCount() g.n.logger.Println("E!", err) // Skip point return nil, nil } } err := g.rc.AggregatePoint(p.Name(), p) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to aggregate point:", err) } m, err := g.n.emit(g.rc) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! failed to emit stream:", err) return nil, nil } return m, nil } func (g *influxqlStreamingTransformGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *InfluxQLNode) getCreateFn(kind reflect.Kind) (createReduceContextFunc, error) { changed := n.currentKind != kind if !changed && n.createFn != nil { return n.createFn, nil } n.currentKind = kind createFn, err := determineReduceContextCreateFn(n.n.Method, kind, n.n.ReduceCreater) if err != nil { return nil, errors.Wrapf(err, "invalid influxql func %s with field %s", n.n.Method, n.n.Field) } n.createFn = createFn return n.createFn, nil } func (n *InfluxQLNode) emit(context reduceContext) (edge.Message, error) { switch n.Provides() { case pipeline.StreamEdge: return context.EmitPoint() case pipeline.BatchEdge: return context.EmitBatch(), nil } return nil, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/join.go ================================================ package kapacitor import ( "fmt" "log" "sync" "time" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/pkg/errors" ) type JoinNode struct { node j *pipeline.JoinNode fill influxql.FillOption fillValue interface{} groupsMu sync.RWMutex groups map[models.GroupID]*joinGroup // Represents the lower bound of times per group per source lowMarks map[srcGroup]time.Time // Buffer for caching points that need to be matched with specific points. matchGroupsBuffer map[models.GroupID][]srcPoint // Buffer for caching specific points until their match arrivces. specificGroupsBuffer map[models.GroupID][]srcPoint reported map[int]bool allReported bool } // Create a new JoinNode, which takes pairs from parent streams combines them into a single point. func newJoinNode(et *ExecutingTask, n *pipeline.JoinNode, l *log.Logger) (*JoinNode, error) { jn := &JoinNode{ j: n, node: node{Node: n, et: et, logger: l}, groups: make(map[models.GroupID]*joinGroup), matchGroupsBuffer: make(map[models.GroupID][]srcPoint), specificGroupsBuffer: make(map[models.GroupID][]srcPoint), lowMarks: make(map[srcGroup]time.Time), reported: make(map[int]bool), } // Set fill switch fill := n.Fill.(type) { case string: switch fill { case "null": jn.fill = influxql.NullFill case "none": jn.fill = influxql.NoFill default: return nil, fmt.Errorf("unexpected fill option %s", fill) } case int64, float64: jn.fill = influxql.NumberFill jn.fillValue = fill default: jn.fill = influxql.NoFill } jn.node.runF = jn.runJoin return jn, nil } func (n *JoinNode) runJoin([]byte) error { consumer := edge.NewMultiConsumerWithStats(n.ins, n) valueF := func() int64 { n.groupsMu.RLock() l := len(n.groups) n.groupsMu.RUnlock() return int64(l) } n.statMap.Set(statCardinalityGauge, expvar.NewIntFuncGauge(valueF)) return consumer.Consume() } func (n *JoinNode) BufferedBatch(src int, batch edge.BufferedBatchMessage) error { return n.doMessage(src, batch) } func (n *JoinNode) Point(src int, p edge.PointMessage) error { return n.doMessage(src, p) } func (n *JoinNode) Barrier(src int, b edge.BarrierMessage) error { return edge.Forward(n.outs, b) } func (n *JoinNode) Finish() error { // No more points are coming signal all groups to finish up. for _, group := range n.groups { if err := group.Finish(); err != nil { return err } } return nil } type messageMeta interface { edge.Message edge.PointMeta } type srcPoint struct { Src int Msg messageMeta } func (n *JoinNode) doMessage(src int, m messageMeta) error { n.timer.Start() defer n.timer.Stop() if len(n.j.Dimensions) > 0 { // Match points with their group based on join dimensions. n.matchPoints(srcPoint{Src: src, Msg: m}) } else { // Just send point on to group, we are not joining on specific dimensions. group := n.getOrCreateGroup(m.GroupID()) group.Collect(src, m) } return nil } // The purpose of this method is to match more specific points // with the less specific points as they arrive. // // Where 'more specific' means, that a point has more dimensions than the join.on dimensions. func (n *JoinNode) matchPoints(p srcPoint) { // Specific points may be sent to the joinset without a matching point, but not the other way around. // This is because the specific points have the needed specific tag data. // The joinset will later handle the fill inner/outer join operations. if !n.allReported { n.reported[p.Src] = true n.allReported = len(n.reported) == len(n.ins) } t := p.Msg.Time().Round(n.j.Tolerance) groupId := models.ToGroupID( p.Msg.Name(), p.Msg.GroupInfo().Tags, models.Dimensions{ ByName: p.Msg.Dimensions().ByName, TagNames: n.j.Dimensions, }, ) // Update current srcGroup lowMark srcG := srcGroup{src: p.Src, groupId: groupId} n.lowMarks[srcG] = t // Determine lowMark, the oldest time per parent per group. var lowMark time.Time if n.allReported { for s := 0; s < len(n.ins); s++ { sg := srcGroup{src: s, groupId: groupId} if lm := n.lowMarks[sg]; lowMark.IsZero() || lm.Before(lowMark) { lowMark = lm } } } // Check for cached specific points that can now be sent alone. if n.allReported { // Send all cached specific point that won't match anymore. var i int buf := n.specificGroupsBuffer[groupId] l := len(buf) for i = 0; i < l; i++ { st := buf[i].Msg.Time().Round(n.j.Tolerance) if st.Before(lowMark) { // Send point by itself since it won't get a match. n.sendSpecificPoint(buf[i]) } else { break } } // Remove all sent points. n.specificGroupsBuffer[groupId] = buf[i:] } if len(p.Msg.Dimensions().TagNames) > len(n.j.Dimensions) { // We have a specific point and three options: // 1. Find the cached match point and send both to group. // 2. Cache the specific point for later. // 3. Send the specific point alone if it is no longer possible that a match will arrive. // Search for a match. // Also purge any old match points. matches := n.matchGroupsBuffer[groupId] matched := false var i int l := len(matches) for i = 0; i < l; i++ { match := matches[i] pt := match.Msg.Time().Round(n.j.Tolerance) if pt.Equal(t) { // Option 1, send both points n.sendMatchPoint(p, match) matched = true } if !pt.Before(lowMark) { break } } if n.allReported { // Can't trust lowMark until all parents have reported. // Remove any unneeded match points. n.matchGroupsBuffer[groupId] = matches[i:] } // If the point didn't match that leaves us with options 2 and 3. if !matched { if n.allReported && t.Before(lowMark) { // Option 3 // Send this specific point by itself since it won't get a match. n.sendSpecificPoint(p) } else { // Option 2 // Cache this point for when its match arrives. n.specificGroupsBuffer[groupId] = append(n.specificGroupsBuffer[groupId], p) } } } else { // Cache match point. n.matchGroupsBuffer[groupId] = append(n.matchGroupsBuffer[groupId], p) // Send all specific points that match, to the group. var i int buf := n.specificGroupsBuffer[groupId] l := len(buf) for i = 0; i < l; i++ { st := buf[i].Msg.Time().Round(n.j.Tolerance) if st.Equal(t) { n.sendMatchPoint(buf[i], p) } else { break } } // Remove all sent points n.specificGroupsBuffer[groupId] = buf[i:] } } // Add the specific tags from the specific point to the matched point // and then send both on to the group. func (n *JoinNode) sendMatchPoint(specific, matched srcPoint) { var newMatched messageMeta switch msg := matched.Msg.(type) { case edge.BufferedBatchMessage: b := msg.ShallowCopy() b.SetBegin(b.Begin().ShallowCopy()) b.Begin().SetTags(specific.Msg.GroupInfo().Tags) newMatched = b case edge.PointMessage: p := msg.ShallowCopy() info := specific.Msg.GroupInfo() p.SetTagsAndDimensions(info.Tags, info.Dimensions) newMatched = p } group := n.getOrCreateGroup(specific.Msg.GroupID()) // Collect specific point group.Collect(specific.Src, specific.Msg) // Collect new matched point group.Collect(matched.Src, newMatched) } // Send only the specific point to the group func (n *JoinNode) sendSpecificPoint(specific srcPoint) { group := n.getOrCreateGroup(specific.Msg.GroupID()) group.Collect(specific.Src, specific.Msg) } // safely get the group for the point or create one if it doesn't exist. func (n *JoinNode) getOrCreateGroup(groupID models.GroupID) *joinGroup { group := n.groups[groupID] if group == nil { group = n.newGroup(len(n.ins)) n.groupsMu.Lock() n.groups[groupID] = group n.groupsMu.Unlock() } return group } func (n *JoinNode) newGroup(count int) *joinGroup { return &joinGroup{ n: n, sets: make(map[time.Time][]*joinset), head: make([]time.Time, count), } } // handles emitting joined sets once enough data has arrived from parents. type joinGroup struct { n *JoinNode sets map[time.Time][]*joinset head []time.Time oldestTime time.Time } func (g *joinGroup) Finish() error { return g.emitAll() } // Collect a point from a given parent. // emit the oldest set if we have collected enough data. func (g *joinGroup) Collect(src int, p timeMessage) error { t := p.Time().Round(g.n.j.Tolerance) if t.Before(g.oldestTime) || g.oldestTime.IsZero() { g.oldestTime = t } var set *joinset sets := g.sets[t] if len(sets) == 0 { set = g.newJoinset(t) sets = append(sets, set) g.sets[t] = sets } for i := 0; i < len(sets); i++ { if !sets[i].Has(src) { set = sets[i] break } } if set == nil { set = g.newJoinset(t) sets = append(sets, set) g.sets[t] = sets } set.Set(src, p) // Update head g.head[src] = t onlyReadySets := false for _, t := range g.head { if !t.After(g.oldestTime) { onlyReadySets = true break } } err := g.emit(onlyReadySets) if err != nil { return err } return nil } func (g *joinGroup) newJoinset(t time.Time) *joinset { return newJoinset( g.n, g.n.j.StreamName, g.n.fill, g.n.fillValue, g.n.j.Names, g.n.j.Delimiter, g.n.j.Tolerance, t, g.n.logger, ) } // emit a set and update the oldestTime. func (g *joinGroup) emit(onlyReadySets bool) error { sets := g.sets[g.oldestTime] i := 0 for ; i < len(sets); i++ { if sets[i].Ready() || !onlyReadySets { err := g.emitJoinedSet(sets[i]) if err != nil { return err } } else { break } } if i == len(sets) { delete(g.sets, g.oldestTime) } else { g.sets[g.oldestTime] = sets[i:] } g.oldestTime = time.Time{} for t := range g.sets { if g.oldestTime.IsZero() || t.Before(g.oldestTime) { g.oldestTime = t } } return nil } // emit sets until we have none left. func (g *joinGroup) emitAll() error { var lastErr error for len(g.sets) > 0 { err := g.emit(false) if err != nil { lastErr = err } } return lastErr } // emit a single joined set func (g *joinGroup) emitJoinedSet(set *joinset) error { if set.name == "" { set.name = set.First().(edge.NameGetter).Name() } switch g.n.Wants() { case pipeline.StreamEdge: p, err := set.JoinIntoPoint() if err != nil { return errors.Wrap(err, "failed to join into point") } if p != nil { if err := edge.Forward(g.n.outs, p); err != nil { return err } } case pipeline.BatchEdge: b, err := set.JoinIntoBatch() if err != nil { return errors.Wrap(err, "failed to join into batch") } if b != nil { if err := edge.Forward(g.n.outs, b); err != nil { return err } } } return nil } // A groupId and its parent type srcGroup struct { src int groupId models.GroupID } // represents a set of points or batches from the same joined time type joinset struct { j *JoinNode name string fill influxql.FillOption fillValue interface{} prefixes []string delimiter string time time.Time tolerance time.Duration values []edge.Message expected int size int finished int first int logger *log.Logger } func newJoinset( n *JoinNode, name string, fill influxql.FillOption, fillValue interface{}, prefixes []string, delimiter string, tolerance time.Duration, time time.Time, l *log.Logger, ) *joinset { expected := len(prefixes) return &joinset{ j: n, name: name, fill: fill, fillValue: fillValue, prefixes: prefixes, delimiter: delimiter, expected: expected, values: make([]edge.Message, expected), first: expected, time: time, tolerance: tolerance, logger: l, } } func (js *joinset) Ready() bool { return js.size == js.expected } func (js *joinset) Has(i int) bool { return js.values[i] != nil } // add a point to the set from a given parent index. func (js *joinset) Set(i int, v edge.Message) { if i < js.first { js.first = i } js.values[i] = v js.size++ } // a valid point in the set func (js *joinset) First() edge.Message { return js.values[js.first] } // join all points into a single point func (js *joinset) JoinIntoPoint() (edge.PointMessage, error) { first, ok := js.First().(edge.PointMessage) if !ok { return nil, fmt.Errorf("unexpected type of first value %T", js.First()) } firstFields := first.Fields() fields := make(models.Fields, js.size*len(firstFields)) for i, v := range js.values { if v == nil { switch js.fill { case influxql.NullFill: for k := range firstFields { fields[js.prefixes[i]+js.delimiter+k] = nil } case influxql.NumberFill: for k := range firstFields { fields[js.prefixes[i]+js.delimiter+k] = js.fillValue } default: // inner join no valid point possible return nil, nil } } else { p, ok := v.(edge.FieldGetter) if !ok { return nil, fmt.Errorf("unexpected type %T", v) } for k, v := range p.Fields() { fields[js.prefixes[i]+js.delimiter+k] = v } } } np := edge.NewPointMessage( js.name, "", "", first.Dimensions(), fields, first.GroupInfo().Tags, js.time, ) return np, nil } // join all batches the set into a single batch func (js *joinset) JoinIntoBatch() (edge.BufferedBatchMessage, error) { first, ok := js.First().(edge.BufferedBatchMessage) if !ok { return nil, fmt.Errorf("unexpected type of first value %T", js.First()) } newBegin := edge.NewBeginBatchMessage( js.name, first.Tags(), first.Dimensions().ByName, js.time, 0, ) newPoints := make([]edge.BatchPointMessage, 0, len(first.Points())) empty := make([]bool, js.expected) emptyCount := 0 indexes := make([]int, js.expected) var fieldNames []string BATCH_POINT: for emptyCount < js.expected { set := make([]edge.BatchPointMessage, js.expected) setTime := time.Time{} count := 0 for i, batch := range js.values { if empty[i] { continue } if batch == nil { emptyCount++ empty[i] = true continue } b, ok := batch.(edge.BufferedBatchMessage) if !ok { return nil, fmt.Errorf("unexpected type of batch value %T", batch) } if indexes[i] == len(b.Points()) { emptyCount++ empty[i] = true continue } bp := b.Points()[indexes[i]] t := bp.Time().Round(js.tolerance) if setTime.IsZero() { setTime = t } if t.Before(setTime) { // We need to backup setTime = t for j := range set { if set[j] != nil { indexes[j]-- } set[j] = nil } set[i] = bp indexes[i]++ count = 1 } else if t.Equal(setTime) { if fieldNames == nil { for k := range bp.Fields() { fieldNames = append(fieldNames, k) } } set[i] = bp indexes[i]++ count++ } } // we didn't get any points from any group we must be empty // skip this set if count == 0 { continue } // Join all batch points in set fields := make(models.Fields, js.expected*len(fieldNames)) for i, bp := range set { if bp == nil { switch js.fill { case influxql.NullFill: for _, k := range fieldNames { fields[js.prefixes[i]+js.delimiter+k] = nil } case influxql.NumberFill: for _, k := range fieldNames { fields[js.prefixes[i]+js.delimiter+k] = js.fillValue } default: // inner join no valid point possible continue BATCH_POINT } } else { for k, v := range bp.Fields() { fields[js.prefixes[i]+js.delimiter+k] = v } } } bp := edge.NewBatchPointMessage( fields, newBegin.Tags(), setTime, ) newPoints = append(newPoints, bp) } newBegin.SetSizeHint(len(newPoints)) return edge.NewBufferedBatchMessage( newBegin, newPoints, edge.NewEndBatchMessage(), ), nil } type durationVar struct { expvar.Int } func (d *durationVar) String() string { return time.Duration(d.IntValue()).String() } ================================================ FILE: vendor/github.com/influxdata/kapacitor/kapacitor_loopback.go ================================================ package kapacitor import ( "fmt" "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) const ( statsKapacitorLoopbackPointsWritten = "points_written" ) type KapacitorLoopbackNode struct { node k *pipeline.KapacitorLoopbackNode pointsWritten *expvar.Int begin edge.BeginBatchMessage } func newKapacitorLoopbackNode(et *ExecutingTask, n *pipeline.KapacitorLoopbackNode, l *log.Logger) (*KapacitorLoopbackNode, error) { kn := &KapacitorLoopbackNode{ node: node{Node: n, et: et, logger: l}, k: n, } kn.node.runF = kn.runOut // Check that a loop has not been created within this task for _, dbrp := range et.Task.DBRPs { if dbrp.Database == n.Database && dbrp.RetentionPolicy == n.RetentionPolicy { return nil, fmt.Errorf("loop detected on dbrp: %v", dbrp) } } return kn, nil } func (n *KapacitorLoopbackNode) runOut([]byte) error { n.pointsWritten = &expvar.Int{} n.statMap.Set(statsInfluxDBPointsWritten, n.pointsWritten) consumer := edge.NewConsumerWithReceiver( n.ins[0], n, ) return consumer.Consume() } func (n *KapacitorLoopbackNode) Point(p edge.PointMessage) error { n.timer.Start() defer n.timer.Stop() p = p.ShallowCopy() if n.k.Database != "" { p.SetDatabase(n.k.Database) } if n.k.RetentionPolicy != "" { p.SetRetentionPolicy(n.k.RetentionPolicy) } if n.k.Measurement != "" { p.SetName(n.k.Measurement) } if len(n.k.Tags) > 0 { tags := p.Tags().Copy() for k, v := range n.k.Tags { tags[k] = v } p.SetTags(tags) } n.timer.Pause() err := n.et.tm.WriteKapacitorPoint(p) n.timer.Resume() if err != nil { n.incrementErrorCount() n.logger.Println("E! failed to write point over loopback") } else { n.pointsWritten.Add(1) } return nil } func (n *KapacitorLoopbackNode) BeginBatch(begin edge.BeginBatchMessage) error { n.begin = begin return nil } func (n *KapacitorLoopbackNode) BatchPoint(bp edge.BatchPointMessage) error { tags := bp.Tags() if len(n.k.Tags) > 0 { tags = bp.Tags().Copy() for k, v := range n.k.Tags { tags[k] = v } } p := edge.NewPointMessage( n.begin.Name(), n.k.Database, n.k.RetentionPolicy, models.Dimensions{}, bp.Fields(), tags, bp.Time(), ) n.timer.Pause() err := n.et.tm.WriteKapacitorPoint(p) n.timer.Resume() if err != nil { n.incrementErrorCount() n.logger.Println("E! failed to write point over loopback") } else { n.pointsWritten.Add(1) } return nil } func (n *KapacitorLoopbackNode) EndBatch(edge.EndBatchMessage) error { return nil } func (n *KapacitorLoopbackNode) Barrier(edge.BarrierMessage) error { return nil } func (n *KapacitorLoopbackNode) DeleteGroup(edge.DeleteGroupMessage) error { return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/list-deps ================================================ #!/bin/bash # Make sure we are in the dir of the script DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) cd $DIR # List all dependent packages and whether they have been vendored. deps() { local package packages allDeps paths # Get the current package package=$(go list .) # Get the sub packages excluding vendored packages packages=$(go list ./... | grep -v "^$package/vendor") allDeps=$(go list -f '{{ join .Deps "\n"}}' $packages) for dep in $allDeps do # Skip standard lib deps paths=(${dep//\// }) if ! [[ "${paths[0]}" =~ .*\..* ]] then continue fi # Skip deps from within current package if [[ "$dep" =~ ^$package ]] then if [[ "$dep" =~ ^$package/vendor ]] then # Rewrite vendored deps as normal deps dep="v ${dep/$package\/vendor\//}" else continue fi else dep="n $dep" fi echo $dep done } # Deduplicate and sort the output deps | sort -k 2 -u ================================================ FILE: vendor/github.com/influxdata/kapacitor/log.go ================================================ package kapacitor import ( "bytes" "encoding/json" "fmt" "log" "strings" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/wlog" ) type LogNode struct { node key string buf bytes.Buffer enc *json.Encoder batchBuffer *edge.BatchBuffer } // Create a new LogNode which logs all data it receives func newLogNode(et *ExecutingTask, n *pipeline.LogNode, l *log.Logger) (*LogNode, error) { level, ok := wlog.StringToLevel[strings.ToUpper(n.Level)] if !ok { return nil, fmt.Errorf("invalid log level %s", n.Level) } nn := &LogNode{ node: node{Node: n, et: et, logger: l}, key: fmt.Sprintf("%c! %s", wlog.ReverseLevels[level], n.Prefix), batchBuffer: new(edge.BatchBuffer), } nn.enc = json.NewEncoder(&nn.buf) nn.node.runF = nn.runLog return nn, nil } func (n *LogNode) runLog([]byte) error { consumer := edge.NewConsumerWithReceiver( n.ins[0], edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n), ), ) return consumer.Consume() } func (n *LogNode) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { return nil, n.batchBuffer.BeginBatch(begin) } func (n *LogNode) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { return nil, n.batchBuffer.BatchPoint(bp) } func (n *LogNode) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return n.BufferedBatch(n.batchBuffer.BufferedBatchMessage(end)) } func (n *LogNode) BufferedBatch(batch edge.BufferedBatchMessage) (edge.Message, error) { n.buf.Reset() if err := n.enc.Encode(batch); err != nil { n.incrementErrorCount() n.logger.Println("E!", err) return batch, nil } n.logger.Println(n.key, n.buf.String()) return batch, nil } func (n *LogNode) Point(p edge.PointMessage) (edge.Message, error) { n.buf.Reset() if err := n.enc.Encode(p); err != nil { n.incrementErrorCount() n.logger.Println("E!", err) return p, nil } n.logger.Println(n.key, n.buf.String()) return p, nil } func (n *LogNode) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *LogNode) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/metaclient.go ================================================ package kapacitor import ( "errors" "time" "github.com/influxdata/influxdb/services/meta" ) type NoopMetaClient struct{} func (m *NoopMetaClient) WaitForLeader(d time.Duration) error { return nil } func (m *NoopMetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) { return nil, nil } func (m *NoopMetaClient) CreateDatabaseWithRetentionPolicy(name string, rpi *meta.RetentionPolicySpec) (*meta.DatabaseInfo, error) { return nil, nil } func (m *NoopMetaClient) CreateRetentionPolicy(database string, rpi *meta.RetentionPolicySpec) (*meta.RetentionPolicyInfo, error) { return nil, nil } func (m *NoopMetaClient) Database(name string) *meta.DatabaseInfo { return &meta.DatabaseInfo{ Name: name, } } func (m *NoopMetaClient) RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) { return nil, nil } func (m *NoopMetaClient) Authenticate(username, password string) (ui *meta.UserInfo, err error) { return nil, errors.New("not authenticated") } func (m *NoopMetaClient) Users() ([]meta.UserInfo, error) { return nil, errors.New("no user") } ================================================ FILE: vendor/github.com/influxdata/kapacitor/node.go ================================================ package kapacitor import ( "bytes" "expvar" "fmt" "log" "runtime" "sync" "sync/atomic" "time" "github.com/influxdata/kapacitor/edge" kexpvar "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/server/vars" "github.com/influxdata/kapacitor/timer" "github.com/pkg/errors" ) const ( statErrorCount = "errors" statCardinalityGauge = "working_cardinality" statAverageExecTime = "avg_exec_time_ns" ) // A node that can be in an executor. type Node interface { pipeline.Node addParentEdge(edge.StatsEdge) init() // start the node and its children start(snapshot []byte) stop() // snapshot running state snapshot() ([]byte, error) restore(snapshot []byte) error // wait for the node to finish processing and return any errors Wait() error // link specified child linkChild(c Node) error addParent(p Node) // close children edges closeChildEdges() // abort parent edges abortParentEdges() // executing dot edot(buf *bytes.Buffer, labels bool) nodeStatsByGroup() map[models.GroupID]nodeStats collectedCount() int64 emittedCount() int64 incrementErrorCount() stats() map[string]interface{} } //implementation of Node type node struct { pipeline.Node et *ExecutingTask parents []Node children []Node runF func(snapshot []byte) error stopF func() errCh chan error err error finishedMu sync.Mutex finished bool ins []edge.StatsEdge outs []edge.StatsEdge logger *log.Logger timer timer.Timer statsKey string statMap *kexpvar.Map nodeErrors *kexpvar.Int } func (n *node) addParentEdge(e edge.StatsEdge) { n.ins = append(n.ins, e) } func (n *node) abortParentEdges() { for _, in := range n.ins { in.Abort() } } func (n *node) init() { tags := map[string]string{ "task": n.et.Task.ID, "node": n.Name(), "type": n.et.Task.Type.String(), "kind": n.Desc(), } n.statsKey, n.statMap = vars.NewStatistic("nodes", tags) avgExecVar := &MaxDuration{} n.statMap.Set(statAverageExecTime, avgExecVar) n.nodeErrors = &kexpvar.Int{} n.statMap.Set(statErrorCount, n.nodeErrors) n.statMap.Set(statCardinalityGauge, kexpvar.NewIntFuncGauge(nil)) n.timer = n.et.tm.TimingService.NewTimer(avgExecVar) n.errCh = make(chan error, 1) } func (n *node) start(snapshot []byte) { go func() { var err error defer func() { // Always close children edges n.closeChildEdges() // Propagate error up if err != nil { // Handle panic in runF r := recover() if r != nil { trace := make([]byte, 512) n := runtime.Stack(trace, false) err = fmt.Errorf("%s: Trace:%s", r, string(trace[:n])) } n.abortParentEdges() n.logger.Println("E!", err) err = errors.Wrap(err, n.Name()) } n.errCh <- err }() // Run node err = n.runF(snapshot) }() } func (n *node) stop() { if n.stopF != nil { n.stopF() } vars.DeleteStatistic(n.statsKey) } // no-op snapshot func (n *node) snapshot() (b []byte, err error) { return } // no-op restore func (n *node) restore([]byte) error { return nil } func (n *node) Wait() error { n.finishedMu.Lock() defer n.finishedMu.Unlock() if !n.finished { n.finished = true n.err = <-n.errCh } return n.err } func (n *node) addChild(c Node) (edge.StatsEdge, error) { if n.Provides() != c.Wants() { return nil, fmt.Errorf("cannot add child mismatched edges: %s:%s -> %s:%s", n.Name(), n.Provides(), c.Name(), c.Wants()) } if n.Provides() == pipeline.NoEdge { return nil, fmt.Errorf("cannot add child no edge expected: %s:%s -> %s:%s", n.Name(), n.Provides(), c.Name(), c.Wants()) } n.children = append(n.children, c) edge := newEdge(n.et.Task.ID, n.Name(), c.Name(), n.Provides(), defaultEdgeBufferSize, n.et.tm.LogService) if edge == nil { return nil, fmt.Errorf("unknown edge type %s", n.Provides()) } c.addParentEdge(edge) return edge, nil } func (n *node) addParent(p Node) { n.parents = append(n.parents, p) } func (n *node) linkChild(c Node) error { // add child edge, err := n.addChild(c) if err != nil { return err } // add parent c.addParent(n) // store edge to child n.outs = append(n.outs, edge) return nil } func (n *node) closeChildEdges() { for _, child := range n.outs { child.Close() } } func (n *node) edot(buf *bytes.Buffer, labels bool) { if labels { // Print all stats on node. buf.WriteString( fmt.Sprintf("\n%s [xlabel=\"", n.Name(), ), ) i := 0 n.statMap.DoSorted(func(kv expvar.KeyValue) { if i != 0 { // NOTE: A literal \r, indicates a newline right justified in graphviz syntax. buf.WriteString(`\r`) } i++ var s string if sv, ok := kv.Value.(kexpvar.StringVar); ok { s = sv.StringValue() } else { s = kv.Value.String() } buf.WriteString( fmt.Sprintf("%s=%s", kv.Key, s, ), ) }) buf.Write([]byte("\"];\n")) for i, c := range n.children { buf.Write([]byte( fmt.Sprintf("%s -> %s [label=\"processed=%d\"];\n", n.Name(), c.Name(), n.outs[i].Collected(), ), )) } } else { // Print all stats on node. buf.Write([]byte( fmt.Sprintf("\n%s [", n.Name(), ), )) n.statMap.DoSorted(func(kv expvar.KeyValue) { var s string if sv, ok := kv.Value.(kexpvar.StringVar); ok { s = sv.StringValue() } else { s = kv.Value.String() } buf.Write([]byte( fmt.Sprintf("%s=\"%s\" ", kv.Key, s, ), )) }) buf.Write([]byte("];\n")) for i, c := range n.children { buf.Write([]byte( fmt.Sprintf("%s -> %s [processed=\"%d\"];\n", n.Name(), c.Name(), n.outs[i].Collected(), ), )) } } } // node collected count is the sum of emitted counts of parent edges func (n *node) collectedCount() (count int64) { for _, in := range n.ins { count += in.Emitted() } return } // node emitted count is the sum of collected counts of children edges func (n *node) emittedCount() (count int64) { for _, out := range n.outs { count += out.Collected() } return } // node increment error count increments a nodes error_count stat func (n *node) incrementErrorCount() { n.nodeErrors.Add(1) } func (n *node) stats() map[string]interface{} { stats := make(map[string]interface{}) n.statMap.Do(func(kv expvar.KeyValue) { switch v := kv.Value.(type) { case kexpvar.IntVar: stats[kv.Key] = v.IntValue() case kexpvar.FloatVar: stats[kv.Key] = v.FloatValue() default: stats[kv.Key] = v.String() } }) return stats } // Statistics for a node type nodeStats struct { Fields models.Fields Tags models.Tags Dimensions models.Dimensions } // Return a copy of the current node statistics. // If if no groups have been seen yet a NilGroup will be created with zero stats. func (n *node) nodeStatsByGroup() (stats map[models.GroupID]nodeStats) { // Get the counts for just one output. stats = make(map[models.GroupID]nodeStats) if len(n.outs) > 0 { n.outs[0].ReadGroupStats(func(g *edge.GroupStats) { stats[g.GroupInfo.ID] = nodeStats{ Fields: models.Fields{ // A node's emitted count is the collected count of its output. "emitted": g.Collected, }, Tags: g.GroupInfo.Tags, Dimensions: g.GroupInfo.Dimensions, } }) } if len(stats) == 0 { // If we have no groups/stats add nil group with emitted = 0 stats[""] = nodeStats{ Fields: models.Fields{ "emitted": int64(0), }, } } return } // MaxDuration is a 64-bit int variable representing a duration in nanoseconds,that satisfies the expvar.Var interface. // When setting a value it will only be set if it is greater than the current value. type MaxDuration struct { d int64 setter timer.Setter } func (v *MaxDuration) String() string { return `"` + v.StringValue() + `"` } func (v *MaxDuration) StringValue() string { return time.Duration(v.IntValue()).String() } func (v *MaxDuration) IntValue() int64 { return atomic.LoadInt64(&v.d) } // Set sets value if it is greater than current value. // If set was successful and a setter exists, will pass on value to setter. func (v *MaxDuration) Set(next int64) { for { cur := v.IntValue() if next > cur { if atomic.CompareAndSwapInt64(&v.d, cur, next) { if v.setter != nil { v.setter.Set(next) } return } } else { return } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/noop.go ================================================ package kapacitor import ( "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" ) type NoOpNode struct { node } // Create a new NoOpNode which does nothing with the data and just passes it through. func newNoOpNode(et *ExecutingTask, n *pipeline.NoOpNode, l *log.Logger) (*NoOpNode, error) { nn := &NoOpNode{ node: node{Node: n, et: et, logger: l}, } nn.node.runF = nn.runNoOp return nn, nil } func (n *NoOpNode) runNoOp([]byte) error { for m, ok := n.ins[0].Emit(); ok; m, ok = n.ins[0].Emit() { if err := edge.Forward(n.outs, m); err != nil { return err } } return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/output.go ================================================ package kapacitor // An output of a pipeline. Still need to improve this interface to expose different types of outputs. type Output interface { Endpoint() string } ================================================ FILE: vendor/github.com/influxdata/kapacitor/query.go ================================================ package kapacitor import ( "fmt" "time" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/kapacitor/tick/ast" "github.com/pkg/errors" ) type Query struct { startTL *influxql.TimeLiteral stopTL *influxql.TimeLiteral groupByTimeDL *influxql.DurationLiteral groupByOffsetDL *influxql.DurationLiteral stmt *influxql.SelectStatement alignGroup bool } func NewQuery(queryString string) (*Query, error) { query := &Query{} // Parse and validate query q, err := influxql.ParseQuery(queryString) if err != nil { return nil, errors.Wrap(err, "failed to parse InfluxQL query") } if l := len(q.Statements); l != 1 { return nil, fmt.Errorf("query must be a single select statement, got %d statements", l) } var ok bool query.stmt, ok = q.Statements[0].(*influxql.SelectStatement) if !ok { return nil, fmt.Errorf("query is not a select statement %q", q) } // Add in time condition nodes query.startTL = &influxql.TimeLiteral{} startExpr := &influxql.BinaryExpr{ Op: influxql.GTE, LHS: &influxql.VarRef{Val: "time"}, RHS: query.startTL, } query.stopTL = &influxql.TimeLiteral{} stopExpr := &influxql.BinaryExpr{ Op: influxql.LT, LHS: &influxql.VarRef{Val: "time"}, RHS: query.stopTL, } if query.stmt.Condition != nil { query.stmt.Condition = &influxql.BinaryExpr{ Op: influxql.AND, LHS: query.stmt.Condition, RHS: &influxql.BinaryExpr{ Op: influxql.AND, LHS: startExpr, RHS: stopExpr, }, } } else { query.stmt.Condition = &influxql.BinaryExpr{ Op: influxql.AND, LHS: startExpr, RHS: stopExpr, } } return query, nil } // Return the db rp pairs of the query func (q *Query) DBRPs() ([]DBRP, error) { dbrps := make([]DBRP, len(q.stmt.Sources)) for i, s := range q.stmt.Sources { m, ok := s.(*influxql.Measurement) if !ok { return nil, fmt.Errorf("unknown query source %T", s) } dbrps[i] = DBRP{ Database: m.Database, RetentionPolicy: m.RetentionPolicy, } } return dbrps, nil } // Set the start time of the query func (q *Query) StartTime() time.Time { return q.startTL.Val } // Set the stop time of the query func (q *Query) StopTime() time.Time { return q.stopTL.Val } // Set the start time of the query func (q *Query) SetStartTime(s time.Time) { q.startTL.Val = s if q.alignGroup && q.groupByTimeDL != nil && q.groupByOffsetDL != nil { q.groupByOffsetDL.Val = s.Sub(time.Unix(0, 0)) % q.groupByTimeDL.Val } } // Set the stop time of the query func (q *Query) SetStopTime(s time.Time) { q.stopTL.Val = s } // Deep clone this query func (q *Query) Clone() (*Query, error) { n := &Query{ stmt: q.stmt.Clone(), alignGroup: q.alignGroup, } // Find the start/stop time literals var err error influxql.WalkFunc(n.stmt.Condition, func(qlNode influxql.Node) { if bn, ok := qlNode.(*influxql.BinaryExpr); ok { switch bn.Op { case influxql.GTE: if vf, ok := bn.LHS.(*influxql.VarRef); !ok || vf.Val != "time" { return } if tl, ok := bn.RHS.(*influxql.TimeLiteral); ok { // We have a "time" >= 'time literal' if n.startTL == nil { n.startTL = tl } else { err = errors.New("invalid query, found multiple start time conditions") } } case influxql.LT: if vf, ok := bn.LHS.(*influxql.VarRef); !ok || vf.Val != "time" { return } if tl, ok := bn.RHS.(*influxql.TimeLiteral); ok { // We have a "time" < 'time literal' if n.stopTL == nil { n.stopTL = tl } else { err = errors.New("invalid query, found multiple stop time conditions") } } } } }) influxql.WalkFunc(n.stmt.Dimensions, func(qlNode influxql.Node) { if cn, ok := qlNode.(*influxql.Call); ok { if cn.Name == "time" { if dln, ok := cn.Args[0].(*influxql.DurationLiteral); ok { n.groupByTimeDL = &influxql.DurationLiteral{ Val: dln.Val, } } if don, ok := cn.Args[1].(*influxql.DurationLiteral); ok { n.groupByOffsetDL = &influxql.DurationLiteral{ Val: don.Val, } } } } }) if n.startTL == nil { err = errors.New("invalid query, missing start time condition") } if n.stopTL == nil { err = errors.New("invalid query, missing stop time condition") } return n, err } // Set the dimensions on the query func (q *Query) Dimensions(dims []interface{}) error { q.stmt.Dimensions = q.stmt.Dimensions[:0] q.groupByTimeDL = nil q.groupByOffsetDL = &influxql.DurationLiteral{ Val: 0, } // Add in dimensions hasTime := false for _, d := range dims { switch dim := d.(type) { case time.Duration: if hasTime { return fmt.Errorf("groupBy cannot have more than one time dimension") } // Add time dimension hasTime = true q.groupByTimeDL = &influxql.DurationLiteral{ Val: dim, } if q.alignGroup { q.SetStartTime(q.StartTime()) } q.stmt.Dimensions = append(q.stmt.Dimensions, &influxql.Dimension{ Expr: &influxql.Call{ Name: "time", Args: []influxql.Expr{ q.groupByTimeDL, q.groupByOffsetDL, }, }, }) case string: q.stmt.Dimensions = append(q.stmt.Dimensions, &influxql.Dimension{ Expr: &influxql.VarRef{ Val: dim, }, }) case *ast.StarNode: q.stmt.Dimensions = append(q.stmt.Dimensions, &influxql.Dimension{ Expr: &influxql.Wildcard{}, }) case TimeDimension: if hasTime { return fmt.Errorf("groupBy cannot have more than one time dimension") } // Add time dimension hasTime = true q.groupByTimeDL = &influxql.DurationLiteral{ Val: dim.Length, } q.groupByOffsetDL.Val = dim.Offset if q.alignGroup { q.SetStartTime(q.StartTime()) } q.stmt.Dimensions = append(q.stmt.Dimensions, &influxql.Dimension{ Expr: &influxql.Call{ Name: "time", Args: []influxql.Expr{ q.groupByTimeDL, q.groupByOffsetDL, }, }, }) default: return fmt.Errorf("invalid dimension type:%T, must be string or time.Duration", d) } } return nil } func (q *Query) IsGroupedByTime() bool { return q.groupByTimeDL != nil } func (q *Query) AlignGroup() { q.alignGroup = true } func (q *Query) Fill(option influxql.FillOption, value interface{}) { q.stmt.Fill = option q.stmt.FillValue = value } func (q *Query) String() string { return q.stmt.String() } type TimeDimension struct { Length time.Duration Offset time.Duration } func groupByTime(length time.Duration, offset ...time.Duration) (TimeDimension, error) { var o time.Duration if l := len(offset); l == 1 { o = offset[0] } else if l != 0 { return TimeDimension{}, fmt.Errorf("time() function expects 1 or 2 args, got %d", l+1) } return TimeDimension{ Length: length, Offset: o, }, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/query_test.go ================================================ package kapacitor_test import ( "fmt" "testing" "time" "github.com/influxdata/kapacitor" ) func TestQuery_Clone(t *testing.T) { testCases := []string{ "SELECT usage FROM telegraf.autogen.cpu", "SELECT mean(usage) FROM telegraf.autogen.cpu WHERE host = 'serverA'", "SELECT mean(usage) FROM telegraf.autogen.cpu WHERE host = 'serverA' AND dc = 'slc'", "SELECT mean(usage) FROM telegraf.autogen.cpu WHERE host = 'serverA' AND dc = 'slc' OR product = 'login'", "SELECT mean(usage) FROM telegraf.autogen.cpu WHERE host = 'serverA' AND (dc = 'slc' OR product = 'login')", } equal := func(q0, q1 *kapacitor.Query) error { if got, exp := q0.String(), q1.String(); got != exp { return fmt.Errorf("unequal query string: got %s exp %s", got, exp) } if got, exp := q0.StartTime(), q1.StartTime(); got != exp { return fmt.Errorf("unequal query start time: got %v exp %v", got, exp) } if got, exp := q0.StopTime(), q1.StopTime(); got != exp { return fmt.Errorf("unequal query stop time: got %v exp %v", got, exp) } if got, exp := q0.IsGroupedByTime(), q1.IsGroupedByTime(); got != exp { return fmt.Errorf("unequal query IsGroupedByTime: got %v exp %v", got, exp) } return nil } for _, query := range testCases { q, err := kapacitor.NewQuery(query) if err != nil { t.Fatal(err) } clone, err := q.Clone() if err != nil { t.Fatal(err) } if err := equal(clone, q); err != nil { t.Error(err) } // Modify original start time start := time.Date(1975, 1, 1, 0, 0, 0, 0, time.UTC) q.SetStartTime(start) if err := equal(clone, q); err == nil { t.Errorf("equal after modification: got %v", clone) } // Modify clone in the same way clone.SetStartTime(start) if err := equal(clone, q); err != nil { t.Error(err) } // Re-clone clone, err = q.Clone() if err != nil { t.Fatal(err) } if err := equal(clone, q); err != nil { t.Error(err) } // Modify original stop time stop := time.Date(1975, 1, 2, 0, 0, 0, 0, time.UTC) q.SetStopTime(stop) if err := equal(clone, q); err == nil { t.Errorf("equal after modification: got %v", clone) } // Modify clone in the same way clone.SetStopTime(stop) if err := equal(clone, q); err != nil { t.Error(err) } // Re-clone clone, err = q.Clone() if err != nil { t.Fatal(err) } if err := equal(clone, q); err != nil { t.Error(err) } // Set dimensions q.Dimensions([]interface{}{time.Hour}) if err := equal(clone, q); err == nil { t.Errorf("equal after modification: got %v", clone) } // Set dimesions on the clone in the same way clone.Dimensions([]interface{}{time.Hour}) if err := equal(clone, q); err != nil { t.Error(err) } // Re-clone clone, err = q.Clone() if err != nil { t.Fatal(err) } if err := equal(clone, q); err != nil { t.Error(err) } // Set group align and dimensions q.AlignGroup() q.Dimensions([]interface{}{kapacitor.TimeDimension{ Length: time.Minute, Offset: time.Second, }}) if err := equal(clone, q); err == nil { t.Errorf("equal after modification: got %v", clone) return } // Set group align and dimesions on the clone in the same way clone.AlignGroup() clone.Dimensions([]interface{}{kapacitor.TimeDimension{ Length: time.Minute, Offset: time.Second, }}) if err := equal(clone, q); err != nil { t.Error(err) } // Re-clone clone, err = q.Clone() if err != nil { t.Fatal(err) } if err := equal(clone, q); err != nil { t.Error(err) } } } func TestQuery_IsGroupedByTime(t *testing.T) { q, err := kapacitor.NewQuery("SELECT usage FROM telegraf.autogen.cpu") if err != nil { t.Fatal(err) } q.Dimensions([]interface{}{time.Hour}) if !q.IsGroupedByTime() { t.Error("expected query to be grouped by time") } q, err = kapacitor.NewQuery("SELECT usage FROM telegraf.autogen.cpu") if err != nil { t.Fatal(err) } q.Dimensions([]interface{}{kapacitor.TimeDimension{Length: time.Hour, Offset: time.Minute}}) if !q.IsGroupedByTime() { t.Error("expected query to be grouped by time") } q.Dimensions([]interface{}{"host"}) if q.IsGroupedByTime() { t.Error("expected query to not be grouped by time") } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/replay.go ================================================ package kapacitor import ( "bufio" "encoding/json" "fmt" "io" "time" dbmodels "github.com/influxdata/influxdb/models" "github.com/influxdata/kapacitor/clock" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" ) // Replay stream data from a channel source. func ReplayStreamFromChan(clck clock.Clock, points <-chan edge.PointMessage, collector StreamCollector, recTime bool) <-chan error { errC := make(chan error, 1) go func() { errC <- replayStreamFromChan(clck, points, collector, recTime) }() return errC } // Replay stream data from an IO source. func ReplayStreamFromIO(clck clock.Clock, data io.ReadCloser, collector StreamCollector, recTime bool, precision string) <-chan error { allErrs := make(chan error, 2) errC := make(chan error, 1) points := make(chan edge.PointMessage) go func() { allErrs <- replayStreamFromChan(clck, points, collector, recTime) }() go func() { allErrs <- readPointsFromIO(data, points, precision) }() go func() { for i := 0; i < cap(allErrs); i++ { err := <-allErrs if err != nil { errC <- err return } } errC <- nil }() return errC } func replayStreamFromChan(clck clock.Clock, points <-chan edge.PointMessage, collector StreamCollector, recTime bool) error { defer collector.Close() start := time.Time{} var diff time.Duration zero := clck.Zero() for p := range points { if start.IsZero() { start = p.Time() diff = zero.Sub(start) } waitTime := p.Time().Add(diff).UTC() if !recTime { p = p.ShallowCopy() p.SetTime(waitTime) } clck.Until(waitTime) err := collector.CollectPoint(p) if err != nil { return err } } return nil } func readPointsFromIO(data io.ReadCloser, points chan<- edge.PointMessage, precision string) error { defer data.Close() defer close(points) now := time.Time{} in := bufio.NewScanner(data) for in.Scan() { db := in.Text() if !in.Scan() { return fmt.Errorf("invalid replay file format, expected another line") } rp := in.Text() if !in.Scan() { return fmt.Errorf("invalid replay file format, expected another line") } mps, err := dbmodels.ParsePointsWithPrecision( in.Bytes(), now, precision, ) if err != nil { return err } mp := mps[0] p := edge.NewPointMessage( mp.Name(), db, rp, models.Dimensions{}, models.Fields(mp.Fields()), models.Tags(mp.Tags().Map()), mp.Time().UTC(), ) points <- p } return nil } // Replay batch data from a channel source. func ReplayBatchFromChan(clck clock.Clock, batches []<-chan edge.BufferedBatchMessage, collectors []BatchCollector, recTime bool) <-chan error { errC := make(chan error, 1) if e, g := len(batches), len(collectors); e != g { errC <- fmt.Errorf("unexpected number of batch collectors. exp %d got %d", e, g) return errC } allErrs := make(chan error, len(batches)) for i := range batches { go func(collector BatchCollector, batches <-chan edge.BufferedBatchMessage, clck clock.Clock, recTime bool) { allErrs <- replayBatchFromChan(clck, batches, collector, recTime) }(collectors[i], batches[i], clck, recTime) } go func() { // Wait for each one to finish and report first error if any for i := 0; i < cap(allErrs); i++ { err := <-allErrs if err != nil { errC <- err return } } errC <- nil }() return errC } // Replay batch data from an IO source. func ReplayBatchFromIO(clck clock.Clock, data []io.ReadCloser, collectors []BatchCollector, recTime bool) <-chan error { errC := make(chan error, 1) if e, g := len(data), len(collectors); e != g { errC <- fmt.Errorf("unexpected number of batch collectors. exp %d got %d", e, g) return errC } allErrs := make(chan error, len(data)*2) for i := range data { batches := make(chan edge.BufferedBatchMessage) go func(collector BatchCollector, batches <-chan edge.BufferedBatchMessage, clck clock.Clock, recTime bool) { allErrs <- replayBatchFromChan(clck, batches, collector, recTime) }(collectors[i], batches, clck, recTime) go func(data io.ReadCloser, batches chan<- edge.BufferedBatchMessage) { allErrs <- readBatchFromIO(data, batches) }(data[i], batches) } go func() { // Wait for each one to finish and report first error if any for i := 0; i < cap(allErrs); i++ { err := <-allErrs if err != nil { errC <- err return } } errC <- nil }() return errC } // Replay the batch data from a single source func replayBatchFromChan(clck clock.Clock, batches <-chan edge.BufferedBatchMessage, collector BatchCollector, recTime bool) error { defer collector.Close() // Find relative times var start, tmax time.Time var diff time.Duration zero := clck.Zero() for b := range batches { if len(b.Points()) == 0 { // Emit empty batch if b.Begin().Time().IsZero() { // Set tmax to last batch if not set. b.Begin().SetTime(tmax) } else { tmax = b.Begin().Time().UTC() b.Begin().SetTime(tmax) } if err := collector.CollectBatch(b); err != nil { return err } continue } points := b.Points() if start.IsZero() { start = points[0].Time() diff = zero.Sub(start) } var lastTime time.Time if !recTime { for i := range points { points[i].SetTime(points[i].Time().Add(diff).UTC()) } lastTime = points[len(points)-1].Time() } else { lastTime = points[len(points)-1].Time().Add(diff).UTC() } clck.Until(lastTime) if lpt := points[len(points)-1].Time(); b.Begin().Time().Before(lpt) { b.Begin().SetTime(lpt) } tmax = b.Begin().Time().UTC() b.Begin().SetTime(tmax) if err := collector.CollectBatch(b); err != nil { return err } } return nil } // Replay the batch data from a single source func readBatchFromIO(data io.ReadCloser, batches chan<- edge.BufferedBatchMessage) error { defer close(batches) defer data.Close() dec := edge.NewBufferedBatchMessageDecoder(data) for dec.More() { b, err := dec.Decode() if err != nil { return err } if len(b.Points()) == 0 { // do nothing continue } batches <- b } return nil } func WritePointForRecording(w io.Writer, p edge.PointMessage, precision string) error { if _, err := fmt.Fprintf(w, "%s\n%s\n", p.Database(), p.RetentionPolicy()); err != nil { return err } if _, err := w.Write(p.Bytes(precision)); err != nil { return err } if _, err := w.Write([]byte("\n")); err != nil { return err } return nil } func WriteBatchForRecording(w io.Writer, b edge.BufferedBatchMessage) error { enc := json.NewEncoder(w) err := enc.Encode(b) if err != nil { return err } return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/result.go ================================================ package kapacitor import ( "encoding/json" "io" "io/ioutil" "time" "github.com/influxdata/influxdb/influxql" ) // The result from an output. type Result influxql.Result // Unmarshal a Result object from JSON. func ResultFromJSON(in io.Reader) (r Result) { b, err := ioutil.ReadAll(in) if err != nil { r.Err = err return } _ = json.Unmarshal(b, &r) // Convert all times to time.Time ConvertResultTimes(&r) return } func ConvertResultTimes(r *Result) { for _, series := range r.Series { for i, v := range series.Values { for j, c := range series.Columns { if c == "time" { tStr, ok := v[j].(string) if !ok { continue } t, err := time.Parse(time.RFC3339, tStr) if err != nil { continue } series.Values[i][j] = t break } } } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/sample.go ================================================ package kapacitor import ( "errors" "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) type SampleNode struct { node s *pipeline.SampleNode counts map[models.GroupID]int64 duration time.Duration } // Create a new SampleNode which filters data from a source. func newSampleNode(et *ExecutingTask, n *pipeline.SampleNode, l *log.Logger) (*SampleNode, error) { sn := &SampleNode{ node: node{Node: n, et: et, logger: l}, s: n, counts: make(map[models.GroupID]int64), duration: n.Duration, } sn.node.runF = sn.runSample if n.Duration == 0 && n.N == 0 { return nil, errors.New("invalid sample rate: must be positive integer or duration") } return sn, nil } func (n *SampleNode) runSample([]byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *SampleNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup()), ), nil } func (n *SampleNode) newGroup() *sampleGroup { return &sampleGroup{ n: n, } } type sampleGroup struct { n *SampleNode count int64 } func (g *sampleGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { g.count = 0 return begin, nil } func (g *sampleGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { keep := g.n.shouldKeep(g.count, bp.Time()) g.count++ if keep { return bp, nil } return nil, nil } func (g *sampleGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (g *sampleGroup) Point(p edge.PointMessage) (edge.Message, error) { keep := g.n.shouldKeep(g.count, p.Time()) g.count++ if keep { return p, nil } return nil, nil } func (g *sampleGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *sampleGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (n *SampleNode) shouldKeep(count int64, t time.Time) bool { if n.duration != 0 { keepTime := t.Truncate(n.duration) return t.Equal(keepTime) } else { return count%n.s.N == 0 } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/shift.go ================================================ package kapacitor import ( "errors" "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" ) type ShiftNode struct { node s *pipeline.ShiftNode shift time.Duration } // Create a new ShiftNode which shifts points and batches in time. func newShiftNode(et *ExecutingTask, n *pipeline.ShiftNode, l *log.Logger) (*ShiftNode, error) { sn := &ShiftNode{ node: node{Node: n, et: et, logger: l}, s: n, shift: n.Shift, } sn.node.runF = sn.runShift if n.Shift == 0 { return nil, errors.New("invalid shift value: must be non zero duration") } return sn, nil } func (n *ShiftNode) runShift([]byte) error { consumer := edge.NewConsumerWithReceiver( n.ins[0], edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n), ), ) return consumer.Consume() } func (n *ShiftNode) doShift(t edge.TimeSetter) { t.SetTime(t.Time().Add(n.shift)) } func (n *ShiftNode) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { begin = begin.ShallowCopy() n.doShift(begin) return begin, nil } func (n *ShiftNode) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { bp = bp.ShallowCopy() n.doShift(bp) return bp, nil } func (n *ShiftNode) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (n *ShiftNode) Point(p edge.PointMessage) (edge.Message, error) { p = p.ShallowCopy() n.doShift(p) return p, nil } func (n *ShiftNode) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *ShiftNode) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/state_tracking.go ================================================ package kapacitor import ( "fmt" "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" ) type stateTracker interface { track(t time.Time, inState bool) interface{} reset() } type stateTrackingGroup struct { n *StateTrackingNode stateful.Expression tracker stateTracker } type StateTrackingNode struct { node as string expr stateful.Expression scopePool stateful.ScopePool newTracker func() stateTracker } func (n *StateTrackingNode) runStateTracking(_ []byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *StateTrackingNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup()), ), nil } func (n *StateTrackingNode) newGroup() *stateTrackingGroup { // Create a new tracking group g := &stateTrackingGroup{ n: n, } g.Expression = n.expr.CopyReset() g.tracker = n.newTracker() return g } func (g *stateTrackingGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { g.tracker.reset() return begin, nil } func (g *stateTrackingGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { bp = bp.ShallowCopy() err := g.track(bp) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! error while evaluating expression:", err) return nil, nil } return bp, nil } func (g *stateTrackingGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (g *stateTrackingGroup) Point(p edge.PointMessage) (edge.Message, error) { p = p.ShallowCopy() err := g.track(p) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! error while evaluating expression:", err) return nil, nil } return p, nil } func (g *stateTrackingGroup) track(p edge.FieldsTagsTimeSetter) error { pass, err := EvalPredicate(g.Expression, g.n.scopePool, p) if err != nil { return err } fields := p.Fields().Copy() fields[g.n.as] = g.tracker.track(p.Time(), pass) p.SetFields(fields) return nil } func (g *stateTrackingGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *stateTrackingGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } type stateDurationTracker struct { sd *pipeline.StateDurationNode startTime time.Time } func (sdt *stateDurationTracker) reset() { sdt.startTime = time.Time{} } func (sdt *stateDurationTracker) track(t time.Time, inState bool) interface{} { if !inState { sdt.startTime = time.Time{} return float64(-1) } if sdt.startTime.IsZero() { sdt.startTime = t } return float64(t.Sub(sdt.startTime)) / float64(sdt.sd.Unit) } func newStateDurationNode(et *ExecutingTask, sd *pipeline.StateDurationNode, l *log.Logger) (*StateTrackingNode, error) { if sd.Lambda == nil { return nil, fmt.Errorf("nil expression passed to StateDurationNode") } // Validate lambda expression expr, err := stateful.NewExpression(sd.Lambda.Expression) if err != nil { return nil, err } n := &StateTrackingNode{ node: node{Node: sd, et: et, logger: l}, as: sd.As, newTracker: func() stateTracker { return &stateDurationTracker{sd: sd} }, expr: expr, scopePool: stateful.NewScopePool(ast.FindReferenceVariables(sd.Lambda.Expression)), } n.node.runF = n.runStateTracking return n, nil } type stateCountTracker struct { count int64 } func (sct *stateCountTracker) reset() { sct.count = 0 } func (sct *stateCountTracker) track(t time.Time, inState bool) interface{} { if !inState { sct.count = 0 return int64(-1) } sct.count++ return sct.count } func newStateCountNode(et *ExecutingTask, sc *pipeline.StateCountNode, l *log.Logger) (*StateTrackingNode, error) { if sc.Lambda == nil { return nil, fmt.Errorf("nil expression passed to StateCountNode") } // Validate lambda expression expr, err := stateful.NewExpression(sc.Lambda.Expression) if err != nil { return nil, err } n := &StateTrackingNode{ node: node{Node: sc, et: et, logger: l}, as: sc.As, newTracker: func() stateTracker { return &stateCountTracker{} }, expr: expr, scopePool: stateful.NewScopePool(ast.FindReferenceVariables(sc.Lambda.Expression)), } n.node.runF = n.runStateTracking return n, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/stats.go ================================================ package kapacitor import ( "fmt" "log" "sync" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" ) type StatsNode struct { node s *pipeline.StatsNode en Node closing chan struct{} closed bool mu sync.Mutex } // Create a new FromNode which filters data from a source. func newStatsNode(et *ExecutingTask, n *pipeline.StatsNode, l *log.Logger) (*StatsNode, error) { // Lookup the executing node for stats. en := et.lookup[n.SourceNode.ID()] if en == nil { return nil, fmt.Errorf("no node found for %s", n.SourceNode.Name()) } sn := &StatsNode{ node: node{Node: n, et: et, logger: l}, s: n, en: en, closing: make(chan struct{}), } sn.node.runF = sn.runStats sn.node.stopF = sn.stopStats return sn, nil } func (n *StatsNode) runStats([]byte) error { if n.s.AlignFlag { // Wait till we are roughly aligned with the interval. now := time.Now() next := now.Truncate(n.s.Interval).Add(n.s.Interval) after := time.NewTicker(next.Sub(now)) select { case <-after.C: after.Stop() case <-n.closing: after.Stop() return nil } if err := n.emit(now); err != nil { return err } } ticker := time.NewTicker(n.s.Interval) defer ticker.Stop() for { select { case <-n.closing: return nil case now := <-ticker.C: if err := n.emit(now); err != nil { return err } } } } // Emit a set of stats data points. func (n *StatsNode) emit(now time.Time) error { n.timer.Start() defer n.timer.Stop() name := "stats" t := now.UTC() if n.s.AlignFlag { t = t.Round(n.s.Interval) } stats := n.en.nodeStatsByGroup() for _, stat := range stats { point := edge.NewPointMessage( name, "", "", stat.Dimensions, stat.Fields, stat.Tags, t, ) n.timer.Pause() for _, out := range n.outs { err := out.Collect(point) if err != nil { return err } } n.timer.Resume() } return nil } func (n *StatsNode) stopStats() { n.mu.Lock() defer n.mu.Unlock() if !n.closed { n.closed = true close(n.closing) } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/stream.go ================================================ package kapacitor import ( "errors" "fmt" "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" ) type StreamNode struct { node s *pipeline.StreamNode } // Create a new StreamNode which copies all data to children func newStreamNode(et *ExecutingTask, n *pipeline.StreamNode, l *log.Logger) (*StreamNode, error) { sn := &StreamNode{ node: node{Node: n, et: et, logger: l}, s: n, } sn.node.runF = sn.runSourceStream return sn, nil } func (n *StreamNode) runSourceStream([]byte) error { for m, ok := n.ins[0].Emit(); ok; m, ok = n.ins[0].Emit() { for _, child := range n.outs { err := child.Collect(m) if err != nil { return err } } } return nil } type FromNode struct { node s *pipeline.FromNode expression stateful.Expression scopePool stateful.ScopePool tagNames []string allDimensions bool db string rp string name string } // Create a new FromNode which filters data from a source. func newFromNode(et *ExecutingTask, n *pipeline.FromNode, l *log.Logger) (*FromNode, error) { sn := &FromNode{ node: node{Node: n, et: et, logger: l}, s: n, db: n.Database, rp: n.RetentionPolicy, name: n.Measurement, } sn.node.runF = sn.runStream sn.allDimensions, sn.tagNames = determineTagNames(n.Dimensions, nil) if n.Lambda != nil { expr, err := stateful.NewExpression(n.Lambda.Expression) if err != nil { return nil, fmt.Errorf("Failed to compile from expression: %v", err) } sn.expression = expr sn.scopePool = stateful.NewScopePool(ast.FindReferenceVariables(n.Lambda.Expression)) } return sn, nil } func (n *FromNode) runStream([]byte) error { consumer := edge.NewConsumerWithReceiver( n.ins[0], edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n), ), ) return consumer.Consume() } func (n *FromNode) BeginBatch(edge.BeginBatchMessage) (edge.Message, error) { return nil, errors.New("from does not support batch data") } func (n *FromNode) BatchPoint(edge.BatchPointMessage) (edge.Message, error) { return nil, errors.New("from does not support batch data") } func (n *FromNode) EndBatch(edge.EndBatchMessage) (edge.Message, error) { return nil, errors.New("from does not support batch data") } func (n *FromNode) Point(p edge.PointMessage) (edge.Message, error) { if n.matches(p) { p = p.ShallowCopy() if n.s.Truncate != 0 { p.SetTime(p.Time().Truncate(n.s.Truncate)) } if n.s.Round != 0 { p.SetTime(p.Time().Round(n.s.Round)) } p.SetDimensions(models.Dimensions{ ByName: n.s.GroupByMeasurementFlag, TagNames: computeTagNames(p.Tags(), n.allDimensions, n.tagNames, nil), }) return p, nil } return nil, nil } func (n *FromNode) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (n *FromNode) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (n *FromNode) matches(p edge.PointMessage) bool { if n.db != "" && p.Database() != n.db { return false } if n.rp != "" && p.RetentionPolicy() != n.rp { return false } if n.name != "" && p.Name() != n.name { return false } if n.expression != nil { if pass, err := EvalPredicate(n.expression, n.scopePool, p); err != nil { n.incrementErrorCount() n.logger.Println("E! error while evaluating WHERE expression:", err) return false } else { return pass } } return true } ================================================ FILE: vendor/github.com/influxdata/kapacitor/task.go ================================================ package kapacitor import ( "bytes" "errors" "fmt" "log" "math/rand" "sync" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" ) // The type of a task type TaskType int const ( StreamTask TaskType = iota BatchTask ) func (t TaskType) String() string { switch t { case StreamTask: return "stream" case BatchTask: return "batch" default: return "unknown" } } func (t TaskType) MarshalText() ([]byte, error) { return []byte(t.String()), nil } func (t *TaskType) UnmarshalText(text []byte) error { switch string(text) { case "stream": *t = StreamTask case "batch": *t = BatchTask default: return fmt.Errorf("unknown task type %s", string(text)) } return nil } type DBRP struct { Database string `json:"db"` RetentionPolicy string `json:"rp"` } func CreateDBRPMap(dbrps []DBRP) map[DBRP]bool { dbMap := make(map[DBRP]bool, len(dbrps)) for _, dbrp := range dbrps { dbMap[dbrp] = true } return dbMap } func (d DBRP) String() string { return fmt.Sprintf("%q.%q", d.Database, d.RetentionPolicy) } // The complete definition of a task, its id, pipeline and type. type Task struct { ID string Pipeline *pipeline.Pipeline Type TaskType DBRPs []DBRP SnapshotInterval time.Duration } func (t *Task) Dot() []byte { return t.Pipeline.Dot(t.ID) } // returns all the measurements from a FromNode func (t *Task) Measurements() []string { measurements := make([]string, 0) _ = t.Pipeline.Walk(func(node pipeline.Node) error { switch streamNode := node.(type) { case *pipeline.FromNode: measurements = append(measurements, streamNode.Measurement) } return nil }) return measurements } // ---------------------------------- // ExecutingTask // A task that is ready for execution. type ExecutingTask struct { tm *TaskMaster Task *Task source Node outputs map[string]Output // node lookup from pipeline.ID -> Node lookup map[pipeline.ID]Node nodes []Node stopping chan struct{} wg sync.WaitGroup logger *log.Logger // Mutex for throughput var tmu sync.RWMutex throughput float64 } // Create a new task from a defined kapacitor. func NewExecutingTask(tm *TaskMaster, t *Task) (*ExecutingTask, error) { l := tm.LogService.NewLogger(fmt.Sprintf("[task:%s] ", t.ID), log.LstdFlags) et := &ExecutingTask{ tm: tm, Task: t, outputs: make(map[string]Output), lookup: make(map[pipeline.ID]Node), logger: l, } err := et.link() if err != nil { return nil, err } return et, nil } // walks the entire pipeline applying function f. func (et *ExecutingTask) walk(f func(n Node) error) error { for _, n := range et.nodes { err := f(n) if err != nil { return err } } return nil } // walks the entire pipeline in reverse order applying function f. func (et *ExecutingTask) rwalk(f func(n Node) error) error { for i := len(et.nodes) - 1; i >= 0; i-- { err := f(et.nodes[i]) if err != nil { return err } } return nil } // Link all the nodes together based on the task pipeline. func (et *ExecutingTask) link() error { // Walk Pipeline and create equivalent executing nodes err := et.Task.Pipeline.Walk(func(n pipeline.Node) error { l := et.tm.LogService.NewLogger( fmt.Sprintf("[%s:%s] ", et.Task.ID, n.Name()), log.LstdFlags, ) en, err := et.createNode(n, l) if err != nil { return err } et.lookup[n.ID()] = en // Save the walk order et.nodes = append(et.nodes, en) // Duplicate the Edges for _, p := range n.Parents() { ep := et.lookup[p.ID()] err := ep.linkChild(en) if err != nil { return err } } return err }) if err != nil { return err } // The first node is always the source node et.source = et.nodes[0] return nil } // Start the task. func (et *ExecutingTask) start(ins []edge.StatsEdge, snapshot *TaskSnapshot) error { for _, in := range ins { et.source.addParentEdge(in) } validSnapshot := false if snapshot != nil { err := et.walk(func(n Node) error { _, ok := snapshot.NodeSnapshots[n.Name()] if !ok { return fmt.Errorf("task pipeline changed not using snapshot") } return nil }) validSnapshot = err == nil } err := et.walk(func(n Node) error { if validSnapshot { n.start(snapshot.NodeSnapshots[n.Name()]) } else { n.start(nil) } return nil }) if err != nil { return err } et.stopping = make(chan struct{}) if et.Task.SnapshotInterval > 0 { et.wg.Add(1) go et.runSnapshotter() } // Start calcThroughput et.wg.Add(1) go et.calcThroughput() return nil } func (et *ExecutingTask) stop() (err error) { close(et.stopping) _ = et.walk(func(n Node) error { n.stop() e := n.Wait() if e != nil { err = e } return nil }) et.wg.Wait() return } var ErrWrongTaskType = errors.New("wrong task type") // Instruct source batch node to start querying and sending batches of data func (et *ExecutingTask) StartBatching() error { if et.Task.Type != BatchTask { return ErrWrongTaskType } batcher := et.source.(*BatchNode) err := et.checkDBRPs(batcher) if err != nil { batcher.Abort() return err } batcher.Start() return nil } func (et *ExecutingTask) BatchCount() (int, error) { if et.Task.Type != BatchTask { return 0, ErrWrongTaskType } batcher := et.source.(*BatchNode) return batcher.Count(), nil } // Get the next `num` batch queries that the batcher will run starting at time `start`. func (et *ExecutingTask) BatchQueries(start, stop time.Time) ([]BatchQueries, error) { if et.Task.Type != BatchTask { return nil, ErrWrongTaskType } batcher := et.source.(*BatchNode) err := et.checkDBRPs(batcher) if err != nil { return nil, err } return batcher.Queries(start, stop) } // Check that the task allows access to DBRPs func (et *ExecutingTask) checkDBRPs(batcher *BatchNode) error { dbMap := CreateDBRPMap(et.Task.DBRPs) dbrps, err := batcher.DBRPs() if err != nil { return err } for _, dbrp := range dbrps { if !dbMap[dbrp] { return fmt.Errorf("batch query is not allowed to request data from %v", dbrp) } } return nil } // Stop all stats nodes func (et *ExecutingTask) StopStats() { _ = et.walk(func(n Node) error { if s, ok := n.(*StatsNode); ok { s.stopStats() } return nil }) } // Wait till the task finishes and return any error func (et *ExecutingTask) Wait() error { return et.rwalk(func(n Node) error { return n.Wait() }) } // Get a named output. func (et *ExecutingTask) GetOutput(name string) (Output, error) { if o, ok := et.outputs[name]; ok { return o, nil } else { return nil, fmt.Errorf("unknown output %s", name) } } // Register a named output. func (et *ExecutingTask) registerOutput(name string, o Output) { et.outputs[name] = o } type ExecutionStats struct { TaskStats map[string]interface{} NodeStats map[string]map[string]interface{} } func (et *ExecutingTask) ExecutionStats() (ExecutionStats, error) { executionStats := ExecutionStats{ TaskStats: make(map[string]interface{}), NodeStats: make(map[string]map[string]interface{}), } // Fill the task stats executionStats.TaskStats["throughput"] = et.getThroughput() // Fill the nodes stats err := et.walk(func(node Node) error { nodeStats := node.stats() // Add collected and emitted nodeStats["collected"] = node.collectedCount() nodeStats["emitted"] = node.emittedCount() executionStats.NodeStats[node.Name()] = nodeStats return nil }) if err != nil { return executionStats, err } return executionStats, nil } // Return a graphviz .dot formatted byte array. // Label edges with relavant execution information. func (et *ExecutingTask) EDot(labels bool) []byte { var buf bytes.Buffer buf.WriteString("digraph ") buf.WriteString(et.Task.ID) buf.WriteString(" {\n") // Write graph attributes unit := "points" if et.Task.Type == BatchTask { unit = "batches" } buf.WriteString("graph [") if labels { buf.WriteString( fmt.Sprintf("label=\"Throughput: %0.2f %s/s\" forcelabels=true pad=\"0.8,0.5\"", et.getThroughput(), unit, ), ) } else { buf.WriteString( fmt.Sprintf("throughput=\"%0.2f %s/s\"", et.getThroughput(), unit, ), ) } buf.WriteString("];\n") _ = et.walk(func(n Node) error { n.edot(&buf, labels) return nil }) buf.Write([]byte("}")) return buf.Bytes() } // Return the current throughput value. func (et *ExecutingTask) getThroughput() float64 { et.tmu.RLock() defer et.tmu.RUnlock() return et.throughput } func (et *ExecutingTask) calcThroughput() { defer et.wg.Done() var previous int64 last := time.Now() ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-ticker.C: current := et.source.collectedCount() now := time.Now() elapsed := float64(now.Sub(last)) / float64(time.Second) et.tmu.Lock() et.throughput = float64(current-previous) / elapsed et.tmu.Unlock() last = now previous = current case <-et.stopping: return } } } // Create a node from a given pipeline node. func (et *ExecutingTask) createNode(p pipeline.Node, l *log.Logger) (n Node, err error) { switch t := p.(type) { case *pipeline.FromNode: n, err = newFromNode(et, t, l) case *pipeline.StreamNode: n, err = newStreamNode(et, t, l) case *pipeline.BatchNode: n, err = newBatchNode(et, t, l) case *pipeline.QueryNode: n, err = newQueryNode(et, t, l) case *pipeline.WindowNode: n, err = newWindowNode(et, t, l) case *pipeline.HTTPOutNode: n, err = newHTTPOutNode(et, t, l) case *pipeline.HTTPPostNode: n, err = newHTTPPostNode(et, t, l) case *pipeline.InfluxDBOutNode: n, err = newInfluxDBOutNode(et, t, l) case *pipeline.KapacitorLoopbackNode: n, err = newKapacitorLoopbackNode(et, t, l) case *pipeline.AlertNode: n, err = newAlertNode(et, t, l) case *pipeline.GroupByNode: n, err = newGroupByNode(et, t, l) case *pipeline.UnionNode: n, err = newUnionNode(et, t, l) case *pipeline.JoinNode: n, err = newJoinNode(et, t, l) case *pipeline.FlattenNode: n, err = newFlattenNode(et, t, l) case *pipeline.EvalNode: n, err = newEvalNode(et, t, l) case *pipeline.WhereNode: n, err = newWhereNode(et, t, l) case *pipeline.SampleNode: n, err = newSampleNode(et, t, l) case *pipeline.DerivativeNode: n, err = newDerivativeNode(et, t, l) case *pipeline.UDFNode: n, err = newUDFNode(et, t, l) case *pipeline.StatsNode: n, err = newStatsNode(et, t, l) case *pipeline.ShiftNode: n, err = newShiftNode(et, t, l) case *pipeline.NoOpNode: n, err = newNoOpNode(et, t, l) case *pipeline.InfluxQLNode: n, err = newInfluxQLNode(et, t, l) case *pipeline.LogNode: n, err = newLogNode(et, t, l) case *pipeline.DefaultNode: n, err = newDefaultNode(et, t, l) case *pipeline.DeleteNode: n, err = newDeleteNode(et, t, l) case *pipeline.CombineNode: n, err = newCombineNode(et, t, l) case *pipeline.K8sAutoscaleNode: n, err = newK8sAutoscaleNode(et, t, l) case *pipeline.SwarmAutoscaleNode: n, err = newSwarmAutoscaleNode(et, t, l) case *pipeline.StateDurationNode: n, err = newStateDurationNode(et, t, l) case *pipeline.StateCountNode: n, err = newStateCountNode(et, t, l) default: return nil, fmt.Errorf("unknown pipeline node type %T", p) } if err == nil && n != nil { n.init() } return n, err } type TaskSnapshot struct { NodeSnapshots map[string][]byte } func (et *ExecutingTask) Snapshot() (*TaskSnapshot, error) { snapshot := &TaskSnapshot{ NodeSnapshots: make(map[string][]byte), } err := et.walk(func(n Node) error { data, err := n.snapshot() if err != nil { return err } snapshot.NodeSnapshots[n.Name()] = data return nil }) if err != nil { return nil, err } return snapshot, nil } func (et *ExecutingTask) runSnapshotter() { defer et.wg.Done() // Wait random duration to splay snapshot events across interval select { case <-time.After(time.Duration(rand.Float64() * float64(et.Task.SnapshotInterval))): case <-et.stopping: return } ticker := time.NewTicker(et.Task.SnapshotInterval) defer ticker.Stop() for { select { case <-ticker.C: snapshot, err := et.Snapshot() if err != nil { et.logger.Println("E! failed to snapshot task", et.Task.ID, err) break } size := 0 for _, data := range snapshot.NodeSnapshots { size += len(data) } // Only save the snapshot if it has content if size > 0 { err = et.tm.TaskStore.SaveSnapshot(et.Task.ID, snapshot) if err != nil { et.logger.Println("E! failed to save task snapshot", et.Task.ID, err) } } case <-et.stopping: return } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/task_master.go ================================================ package kapacitor import ( "errors" "fmt" "log" "sync" "time" imodels "github.com/influxdata/influxdb/models" "github.com/influxdata/kapacitor/alert" "github.com/influxdata/kapacitor/command" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/expvar" "github.com/influxdata/kapacitor/influxdb" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/server/vars" alertservice "github.com/influxdata/kapacitor/services/alert" "github.com/influxdata/kapacitor/services/alerta" "github.com/influxdata/kapacitor/services/hipchat" "github.com/influxdata/kapacitor/services/httpd" "github.com/influxdata/kapacitor/services/httppost" k8s "github.com/influxdata/kapacitor/services/k8s/client" "github.com/influxdata/kapacitor/services/mqtt" "github.com/influxdata/kapacitor/services/opsgenie" "github.com/influxdata/kapacitor/services/pagerduty" "github.com/influxdata/kapacitor/services/pushover" "github.com/influxdata/kapacitor/services/sensu" "github.com/influxdata/kapacitor/services/slack" "github.com/influxdata/kapacitor/services/smtp" "github.com/influxdata/kapacitor/services/snmptrap" swarm "github.com/influxdata/kapacitor/services/swarm/client" "github.com/influxdata/kapacitor/services/telegram" "github.com/influxdata/kapacitor/services/victorops" "github.com/influxdata/kapacitor/tick" "github.com/influxdata/kapacitor/tick/stateful" "github.com/influxdata/kapacitor/timer" "github.com/influxdata/kapacitor/udf" ) const ( statPointsReceived = "points_received" MainTaskMaster = "main" ) type LogService interface { NewLogger(prefix string, flag int) *log.Logger } type UDFService interface { List() []string Info(name string) (udf.Info, bool) Create(name, taskID, nodeID string, l *log.Logger, abortCallback func()) (udf.Interface, error) } var ErrTaskMasterClosed = errors.New("TaskMaster is closed") var ErrTaskMasterOpen = errors.New("TaskMaster is open") type deleteHook func(*TaskMaster) // An execution framework for a set of tasks. type TaskMaster struct { // Unique id for this task master instance id string ServerInfo vars.Infoer HTTPDService interface { AddRoutes([]httpd.Route) error DelRoutes([]httpd.Route) URL() string } TaskStore interface { SaveSnapshot(id string, snapshot *TaskSnapshot) error HasSnapshot(id string) bool LoadSnapshot(id string) (*TaskSnapshot, error) } DeadmanService pipeline.DeadmanService UDFService UDFService AlertService interface { alertservice.AnonHandlerRegistrar alertservice.Events alertservice.TopicPersister } InfluxDBService interface { NewNamedClient(name string) (influxdb.Client, error) } SMTPService interface { Global() bool StateChangesOnly() bool Handler(smtp.HandlerConfig, *log.Logger) alert.Handler } MQTTService interface { Handler(mqtt.HandlerConfig, *log.Logger) alert.Handler } OpsGenieService interface { Global() bool Handler(opsgenie.HandlerConfig, *log.Logger) alert.Handler } VictorOpsService interface { Global() bool Handler(victorops.HandlerConfig, *log.Logger) alert.Handler } PagerDutyService interface { Global() bool Handler(pagerduty.HandlerConfig, *log.Logger) alert.Handler } PushoverService interface { Handler(pushover.HandlerConfig, *log.Logger) alert.Handler } HTTPPostService interface { Handler(httppost.HandlerConfig, *log.Logger) alert.Handler Endpoint(string) (*httppost.Endpoint, bool) } SlackService interface { Global() bool StateChangesOnly() bool Handler(slack.HandlerConfig, *log.Logger) alert.Handler } SNMPTrapService interface { Handler(snmptrap.HandlerConfig, *log.Logger) (alert.Handler, error) } TelegramService interface { Global() bool StateChangesOnly() bool Handler(telegram.HandlerConfig, *log.Logger) alert.Handler } HipChatService interface { Global() bool StateChangesOnly() bool Handler(hipchat.HandlerConfig, *log.Logger) alert.Handler } AlertaService interface { DefaultHandlerConfig() alerta.HandlerConfig Handler(alerta.HandlerConfig, *log.Logger) (alert.Handler, error) } SensuService interface { Handler(sensu.HandlerConfig, *log.Logger) (alert.Handler, error) } TalkService interface { Handler(*log.Logger) alert.Handler } TimingService interface { NewTimer(timer.Setter) timer.Timer } K8sService interface { Client(string) (k8s.Client, error) } SwarmService interface { Client(string) (swarm.Client, error) } LogService LogService Commander command.Commander DefaultRetentionPolicy string // Incoming streams writePointsIn StreamCollector writesClosed bool writesMu sync.RWMutex // Forks of incoming streams // We are mapping from (db, rp, measurement) to map of task ids to their edges // The outer map (from dbrp&measurement) is for fast access on forkPoint // While the inner map is for handling fork deletions better (see taskToForkKeys) forks map[forkKey]map[string]edge.Edge // Stats for number of points each fork has received forkStats map[forkKey]*expvar.Int // Task to fork keys is map to help in deletes, in deletes // we have only the task id, and they are called after the task is deleted from TaskMaster.tasks taskToForkKeys map[string][]forkKey // Set of incoming batches batches map[string][]BatchCollector // Executing tasks tasks map[string]*ExecutingTask // DeleteHooks for tasks deleteHooks map[string][]deleteHook logger *log.Logger closed bool drained bool mu sync.RWMutex wg sync.WaitGroup } type forkKey struct { Database string RetentionPolicy string Measurement string } // Create a new Executor with a given clock. func NewTaskMaster(id string, info vars.Infoer, l LogService) *TaskMaster { return &TaskMaster{ id: id, forks: make(map[forkKey]map[string]edge.Edge), forkStats: make(map[forkKey]*expvar.Int), taskToForkKeys: make(map[string][]forkKey), batches: make(map[string][]BatchCollector), tasks: make(map[string]*ExecutingTask), deleteHooks: make(map[string][]deleteHook), LogService: l, ServerInfo: info, logger: l.NewLogger(fmt.Sprintf("[task_master:%s] ", id), log.LstdFlags), closed: true, TimingService: noOpTimingService{}, } } // Returns a new TaskMaster instance with the same services as the current one. func (tm *TaskMaster) New(id string) *TaskMaster { n := NewTaskMaster(id, tm.ServerInfo, tm.LogService) n.DefaultRetentionPolicy = tm.DefaultRetentionPolicy n.HTTPDService = tm.HTTPDService n.TaskStore = tm.TaskStore n.DeadmanService = tm.DeadmanService n.UDFService = tm.UDFService n.AlertService = tm.AlertService n.InfluxDBService = tm.InfluxDBService n.SMTPService = tm.SMTPService n.MQTTService = tm.MQTTService n.OpsGenieService = tm.OpsGenieService n.VictorOpsService = tm.VictorOpsService n.PagerDutyService = tm.PagerDutyService n.PushoverService = tm.PushoverService n.SlackService = tm.SlackService n.TelegramService = tm.TelegramService n.SNMPTrapService = tm.SNMPTrapService n.HipChatService = tm.HipChatService n.AlertaService = tm.AlertaService n.SensuService = tm.SensuService n.TalkService = tm.TalkService n.TimingService = tm.TimingService n.K8sService = tm.K8sService n.Commander = tm.Commander return n } func (tm *TaskMaster) ID() string { return tm.id } func (tm *TaskMaster) Open() (err error) { tm.mu.Lock() defer tm.mu.Unlock() if !tm.closed { return ErrTaskMasterOpen } tm.closed = false tm.drained = false tm.writePointsIn, err = tm.stream("write_points") if err != nil { tm.closed = true return } tm.logger.Println("I! opened") return } func (tm *TaskMaster) StopTasks() { tm.mu.Lock() defer tm.mu.Unlock() for _, et := range tm.tasks { _ = tm.stopTask(et.Task.ID) } } func (tm *TaskMaster) Close() error { tm.mu.Lock() closed := tm.closed tm.mu.Unlock() if closed { return ErrTaskMasterClosed } tm.Drain() tm.mu.Lock() defer tm.mu.Unlock() tm.closed = true for _, et := range tm.tasks { _ = tm.stopTask(et.Task.ID) } tm.logger.Println("I! closed") return nil } func (tm *TaskMaster) Drain() { tm.waitForForks() tm.mu.Lock() defer tm.mu.Unlock() for id, _ := range tm.taskToForkKeys { tm.delFork(id) } } // Create a new template in the context of a TaskMaster func (tm *TaskMaster) NewTemplate( id, script string, tt TaskType, ) (*Template, error) { t := &Template{ id: id, } scope := tm.CreateTICKScope() var srcEdge pipeline.EdgeType switch tt { case StreamTask: srcEdge = pipeline.StreamEdge case BatchTask: srcEdge = pipeline.BatchEdge } tp, err := pipeline.CreateTemplatePipeline(script, srcEdge, scope, tm.DeadmanService) if err != nil { return nil, err } t.tp = tp return t, nil } // Create a new task in the context of a TaskMaster func (tm *TaskMaster) NewTask( id, script string, tt TaskType, dbrps []DBRP, snapshotInterval time.Duration, vars map[string]tick.Var, ) (*Task, error) { t := &Task{ ID: id, Type: tt, DBRPs: dbrps, SnapshotInterval: snapshotInterval, } scope := tm.CreateTICKScope() var srcEdge pipeline.EdgeType switch tt { case StreamTask: srcEdge = pipeline.StreamEdge case BatchTask: srcEdge = pipeline.BatchEdge } p, err := pipeline.CreatePipeline(script, srcEdge, scope, tm.DeadmanService, vars) if err != nil { return nil, err } // A task will always have a stream or batch node. // If it doesn't have anything more then the task does nothing with the data. if p.Len() <= 1 { return nil, fmt.Errorf("task does nothing") } t.Pipeline = p return t, nil } func (tm *TaskMaster) waitForForks() { tm.mu.Lock() drained := tm.drained tm.mu.Unlock() if drained { return } tm.mu.Lock() tm.drained = true tm.mu.Unlock() tm.writesMu.Lock() tm.writesClosed = true tm.writesMu.Unlock() // Close the write points in stream tm.writePointsIn.Close() // Don't hold the lock while we wait tm.wg.Wait() } func (tm *TaskMaster) CreateTICKScope() *stateful.Scope { scope := stateful.NewScope() scope.Set("time", groupByTime) // Add dynamic methods to the scope for UDFs if tm.UDFService != nil { for _, f := range tm.UDFService.List() { f := f info, _ := tm.UDFService.Info(f) scope.SetDynamicMethod( f, func(self interface{}, args ...interface{}) (interface{}, error) { parent, ok := self.(pipeline.Node) if !ok { return nil, fmt.Errorf("cannot call %s on %T", f, self) } udf := pipeline.NewUDF( parent, f, info.Wants, info.Provides, info.Options, ) return udf, nil }, ) } } return scope } func (tm *TaskMaster) StartTask(t *Task) (*ExecutingTask, error) { tm.mu.Lock() defer tm.mu.Unlock() if tm.closed { return nil, errors.New("task master is closed cannot start a task") } tm.logger.Println("D! Starting task:", t.ID) et, err := NewExecutingTask(tm, t) if err != nil { return nil, err } var ins []edge.StatsEdge switch et.Task.Type { case StreamTask: e, err := tm.newFork(et.Task.ID, et.Task.DBRPs, et.Task.Measurements()) if err != nil { return nil, err } ins = []edge.StatsEdge{e} case BatchTask: count, err := et.BatchCount() if err != nil { return nil, err } ins = make([]edge.StatsEdge, count) for i := 0; i < count; i++ { in := newEdge(t.ID, "batch", fmt.Sprintf("batch%d", i), pipeline.BatchEdge, defaultEdgeBufferSize, tm.LogService) ins[i] = in tm.batches[t.ID] = append(tm.batches[t.ID], &batchCollector{edge: in}) } } var snapshot *TaskSnapshot if tm.TaskStore.HasSnapshot(t.ID) { snapshot, err = tm.TaskStore.LoadSnapshot(t.ID) if err != nil { return nil, err } } err = et.start(ins, snapshot) if err != nil { return nil, err } tm.tasks[et.Task.ID] = et tm.logger.Println("I! Started task:", t.ID) tm.logger.Println("D!", string(t.Dot())) return et, nil } func (tm *TaskMaster) BatchCollectors(id string) []BatchCollector { return tm.batches[id] } func (tm *TaskMaster) StopTask(id string) error { tm.mu.Lock() defer tm.mu.Unlock() return tm.stopTask(id) } func (tm *TaskMaster) DeleteTask(id string) error { tm.mu.Lock() defer tm.mu.Unlock() if err := tm.stopTask(id); err != nil { return err } tm.deleteTask(id) return nil } // internal stopTask function. The caller must have acquired // the lock in order to call this function func (tm *TaskMaster) stopTask(id string) (err error) { if et, ok := tm.tasks[id]; ok { delete(tm.tasks, id) switch et.Task.Type { case StreamTask: tm.delFork(id) case BatchTask: delete(tm.batches, id) } err = et.stop() if err != nil { tm.logger.Println("E! Stopped task:", id, err) } else { tm.logger.Println("I! Stopped task:", id) } } return } // internal deleteTask function. The caller must have acquired // the lock in order to call this function func (tm *TaskMaster) deleteTask(id string) { hooks := tm.deleteHooks[id] for _, deleteHook := range hooks { deleteHook(tm) } } func (tm *TaskMaster) registerDeleteHookForTask(id string, hook deleteHook) { tm.mu.Lock() defer tm.mu.Unlock() tm.deleteHooks[id] = append(tm.deleteHooks[id], hook) } func (tm *TaskMaster) IsExecuting(id string) bool { tm.mu.RLock() defer tm.mu.RUnlock() _, executing := tm.tasks[id] return executing } func (tm *TaskMaster) ExecutionStats(id string) (ExecutionStats, error) { tm.mu.RLock() defer tm.mu.RUnlock() task, executing := tm.tasks[id] if !executing { return ExecutionStats{}, nil } return task.ExecutionStats() } func (tm *TaskMaster) ExecutingDot(id string, labels bool) string { tm.mu.RLock() defer tm.mu.RUnlock() et, executing := tm.tasks[id] if executing { return string(et.EDot(labels)) } return "" } func (tm *TaskMaster) Stream(name string) (StreamCollector, error) { tm.mu.Lock() defer tm.mu.Unlock() return tm.stream(name) } func (tm *TaskMaster) stream(name string) (StreamCollector, error) { if tm.closed { return nil, ErrTaskMasterClosed } in := newEdge(fmt.Sprintf("task_master:%s", tm.id), name, "stream", pipeline.StreamEdge, defaultEdgeBufferSize, tm.LogService) se := &streamEdge{edge: in} tm.wg.Add(1) go func() { defer tm.wg.Done() tm.runForking(se) }() return se, nil } type StreamCollector interface { CollectPoint(edge.PointMessage) error Close() error } type StreamEdge interface { CollectPoint(edge.PointMessage) error EmitPoint() (edge.PointMessage, bool) Close() error } type streamEdge struct { edge edge.Edge } func (s *streamEdge) CollectPoint(p edge.PointMessage) error { return s.edge.Collect(p) } func (s *streamEdge) EmitPoint() (edge.PointMessage, bool) { m, ok := s.edge.Emit() if !ok { return nil, false } p, ok := m.(edge.PointMessage) if !ok { panic("impossible to receive non PointMessage message") } return p, true } func (s *streamEdge) Close() error { return s.edge.Close() } func (tm *TaskMaster) runForking(in StreamEdge) { for p, ok := in.EmitPoint(); ok; p, ok = in.EmitPoint() { tm.forkPoint(p) } } func (tm *TaskMaster) forkPoint(p edge.PointMessage) { tm.mu.RLock() locked := true defer func() { if locked { tm.mu.RUnlock() } }() // Create the fork keys - which is (db, rp, measurement) key := forkKey{ Database: p.Database(), RetentionPolicy: p.RetentionPolicy(), Measurement: p.Name(), } // If we have empty measurement in this db,rp we need to send it all // the points emptyMeasurementKey := forkKey{ Database: p.Database(), RetentionPolicy: p.RetentionPolicy(), Measurement: "", } // Merge the results to the forks map for _, edge := range tm.forks[key] { _ = edge.Collect(p) } for _, edge := range tm.forks[emptyMeasurementKey] { _ = edge.Collect(p) } c, ok := tm.forkStats[key] if !ok { // Release read lock tm.mu.RUnlock() locked = false // Get write lock tm.mu.Lock() // Now with write lock check again c, ok = tm.forkStats[key] if !ok { // Create statistics c = &expvar.Int{} tm.forkStats[key] = c } tm.mu.Unlock() tags := map[string]string{ "task_master": tm.id, "database": key.Database, "retention_policy": key.RetentionPolicy, "measurement": key.Measurement, } _, statMap := vars.NewStatistic("ingress", tags) statMap.Set(statPointsReceived, c) } c.Add(1) } func (tm *TaskMaster) WritePoints(database, retentionPolicy string, consistencyLevel imodels.ConsistencyLevel, points []imodels.Point) error { tm.writesMu.RLock() defer tm.writesMu.RUnlock() if tm.writesClosed { return ErrTaskMasterClosed } if retentionPolicy == "" { retentionPolicy = tm.DefaultRetentionPolicy } for _, mp := range points { p := edge.NewPointMessage( mp.Name(), database, retentionPolicy, models.Dimensions{}, models.Fields(mp.Fields()), models.Tags(mp.Tags().Map()), mp.Time(), ) err := tm.writePointsIn.CollectPoint(p) if err != nil { return err } } return nil } func (tm *TaskMaster) WriteKapacitorPoint(p edge.PointMessage) error { tm.writesMu.RLock() defer tm.writesMu.RUnlock() if tm.writesClosed { return ErrTaskMasterClosed } p = p.ShallowCopy() p.SetDimensions(models.Dimensions{}) return tm.writePointsIn.CollectPoint(p) } func (tm *TaskMaster) NewFork(taskName string, dbrps []DBRP, measurements []string) (edge.StatsEdge, error) { tm.mu.Lock() defer tm.mu.Unlock() return tm.newFork(taskName, dbrps, measurements) } func forkKeys(dbrps []DBRP, measurements []string) []forkKey { keys := make([]forkKey, 0) for _, dbrp := range dbrps { for _, measurement := range measurements { key := forkKey{ RetentionPolicy: dbrp.RetentionPolicy, Database: dbrp.Database, Measurement: measurement, } keys = append(keys, key) } } return keys } // internal newFork, must have acquired lock before calling. func (tm *TaskMaster) newFork(taskName string, dbrps []DBRP, measurements []string) (edge.StatsEdge, error) { if tm.closed { return nil, ErrTaskMasterClosed } e := newEdge(taskName, "stream", "stream0", pipeline.StreamEdge, defaultEdgeBufferSize, tm.LogService) for _, key := range forkKeys(dbrps, measurements) { tm.taskToForkKeys[taskName] = append(tm.taskToForkKeys[taskName], key) // Add the task to the tasksMap if it doesn't exists tasksMap, ok := tm.forks[key] if !ok { tasksMap = make(map[string]edge.Edge, 0) } // Add the edge to task map tasksMap[taskName] = e // update the task map in the forks tm.forks[key] = tasksMap } return e, nil } func (tm *TaskMaster) DelFork(id string) { tm.mu.Lock() defer tm.mu.Unlock() tm.delFork(id) } // internal delFork function, must have lock to call func (tm *TaskMaster) delFork(id string) { // mark if we already closed the edge because the edge is replicated // by it's fork keys (db,rp,measurement) isEdgeClosed := false // Find the fork keys for _, key := range tm.taskToForkKeys[id] { // check if the edge exists edge, ok := tm.forks[key][id] if ok { // Only close the edge if we are already didn't closed it if edge != nil && !isEdgeClosed { isEdgeClosed = true edge.Close() } // remove the task in fork map delete(tm.forks[key], id) } } // remove mapping from task id to it's keys delete(tm.taskToForkKeys, id) } func (tm *TaskMaster) SnapshotTask(id string) (*TaskSnapshot, error) { tm.mu.RLock() et, ok := tm.tasks[id] tm.mu.RUnlock() if ok { return et.Snapshot() } return nil, fmt.Errorf("task %s is not running or does not exist", id) } type noOpTimingService struct{} func (noOpTimingService) NewTimer(timer.Setter) timer.Timer { return timer.NewNoOp() } type TaskMasterLookup struct { sync.Mutex taskMasters map[string]*TaskMaster } func NewTaskMasterLookup() *TaskMasterLookup { return &TaskMasterLookup{ taskMasters: make(map[string]*TaskMaster), } } func (tml *TaskMasterLookup) Get(id string) *TaskMaster { tml.Lock() defer tml.Unlock() return tml.taskMasters[id] } func (tml *TaskMasterLookup) Main() *TaskMaster { return tml.Get(MainTaskMaster) } func (tml *TaskMasterLookup) Set(tm *TaskMaster) { tml.Lock() defer tml.Unlock() tml.taskMasters[tm.id] = tm } func (tml *TaskMasterLookup) Delete(tm *TaskMaster) { tml.Lock() defer tml.Unlock() delete(tml.taskMasters, tm.id) } type BatchCollector interface { CollectBatch(edge.BufferedBatchMessage) error Close() error } type batchCollector struct { edge edge.Edge } func (c *batchCollector) CollectBatch(batch edge.BufferedBatchMessage) error { return c.edge.Collect(batch) } func (c *batchCollector) Close() error { return c.edge.Close() } ================================================ FILE: vendor/github.com/influxdata/kapacitor/template.go ================================================ package kapacitor import ( "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick" ) type Template struct { id string tp *pipeline.TemplatePipeline } func (t *Template) Vars() map[string]tick.Var { return t.tp.Vars() } func (t *Template) Dot() string { return string(t.tp.Dot(t.id)) } ================================================ FILE: vendor/github.com/influxdata/kapacitor/test.sh ================================================ #!/bin/bash # # This is the Kapacitor test script. # This script can run tests in different environments. # # Usage: ./test.sh # Corresponding environments for environment_index: # 0: normal 64bit tests # 1: race enabled 64bit tests # 2: normal 32bit tests # count: print the number of test environments # *: to run all tests in parallel containers # # Logs from the test runs will be saved in OUTPUT_DIR, which defaults to ./test-logs # set -eo pipefail # Get dir of script and make it is our working directory. DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) cd $DIR # Unique number for this build BUILD_NUM=${BUILD_NUM-$RANDOM} # Index for which test environment to use ENVIRONMENT_INDEX=$1 # Set the default OUTPUT_DIR OUTPUT_DIR=${OUTPUT_DIR-./test-logs} # Set the default DOCKER_SAVE_DIR DOCKER_SAVE_DIR=${DOCKER_SAVE_DIR-$HOME/docker} # Set default parallelism PARALLELISM=${PARALLELISM-1} # Set default timeout TIMEOUT=${TIMEOUT-480s} # No uncommitted changes NO_UNCOMMITTED=${NO_UNCOMMITTED-false} # Home dir of the docker user HOME_DIR=/root no_uncomitted_arg="$no_uncommitted_arg" if [ ! $NO_UNCOMMITTED ] then no_uncomitted_arg="" fi # Update this value if you add a new test environment. ENV_COUNT=3 # Default return code 0 rc=0 # Convert dockerfile name to valid docker image tag name. function filename2imagename { echo ${1/Dockerfile/kapacitor} } # Run a test in a docker container # Usage: run_test_docker function run_test_docker { local dockerfile=$1 local imagename=$(filename2imagename "$dockerfile") shift local name=$1 shift local logfile="$OUTPUT_DIR/${name}.log" imagename="$imagename-$BUILD_NUM" dataname="kapacitor-data-$BUILD_NUM" echo "Building docker image $imagename" docker build -f "$dockerfile" -t "$imagename" . echo "Running test in docker $name with args $@" # Create data volume with code docker create \ --name $dataname \ -v "$HOME_DIR/go/src/github.com/influxdata/kapacitor" \ $imagename /bin/true docker cp "$DIR/" "$dataname:$HOME_DIR/go/src/github.com/influxdata/" # Run tests in docker docker run \ --rm \ --volumes-from $dataname \ -e "GORACE=$GORACE" \ -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \ -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \ "$imagename" \ "--parallel=$PARALLELISM" \ "--timeout=$TIMEOUT" \ "$@" \ 2>&1 | tee "$logfile" # Copy results back out docker cp \ "$dataname:$HOME_DIR/go/src/github.com/influxdata/kapacitor/build" \ ./ # Remove the data and builder containers docker rm -v $dataname } if [ ! -d "$OUTPUT_DIR" ] then mkdir -p "$OUTPUT_DIR" fi # Run the tests. case $ENVIRONMENT_INDEX in 0) # 64 bit tests run_test_docker Dockerfile_build_ubuntu64 test_64bit --test --generate $no_uncommitted_arg rc=$? ;; 1) # 64 bit race tests GORACE="halt_on_error=1" run_test_docker Dockerfile_build_ubuntu64 test_64bit_race --test --generate $no_uncommitted_arg --race rc=$? ;; 2) # 32 bit tests run_test_docker Dockerfile_build_ubuntu32 test_32bit --test --generate $no_uncommitted_arg --arch=i386 rc=$? ;; "count") echo $ENV_COUNT ;; *) echo "No individual test environment specified running tests for all $ENV_COUNT environments." # Run all test environments pids=() for t in $(seq 0 "$(($ENV_COUNT - 1))") do $0 $t 2>&1 > /dev/null & # add PID to list pids+=($!) done echo "Started all tests. Follow logs in ${OUTPUT_DIR}. Waiting..." # Wait for all tests to finish for pid in "${pids[@]}" do wait $pid rc=$(($? + $rc)) done # Check if all tests passed if [ $rc -eq 0 ] then echo "All test have passed" else echo "Some tests failed check logs in $OUTPUT_DIR for results" fi ;; esac exit $rc ================================================ FILE: vendor/github.com/influxdata/kapacitor/tmpldata.json ================================================ [ { "Name":"Float", "name":"float", "Type":"float64", "Kind":"reflect.Float64", "Nil":"0", "Zero":"float64(0)" }, { "Name":"Integer", "name":"integer", "Type":"int64", "Kind":"reflect.Int64", "Nil":"0", "Zero":"int64(0)" }, { "Name":"String", "name":"string", "Type":"string", "Kind":"reflect.String", "Nil":"\"\"", "Zero":"\"\"" }, { "Name":"Boolean", "name":"boolean", "Type":"bool", "Kind":"reflect.Bool", "Nil":"false", "Zero":"false" } ] ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/README.md ================================================ # UDF Agents and Servers A UDF is a User Defined Function, meaning that you can write your own functions/algorithms and plug them into Kapacitor. Your custom function runs in its own process and Kapacitor communicates with it via a defined protocol, see [udf.proto](https://github.com/influxdata/kapacitor/blob/master/udf/udf.proto). To facilitate working with the protocol several `agents` have been written in various languages that abstract the protocol communication through an interface in the respective languages. You can find those agent implementations in this directory and subdirectories based on language name. Example uses of the agents can be found in the `examples` directory. These examples are working examples and are executed as part of the testing suite, see [server_test.go](https://github.com/influxdata/kapacitor/blob/master/cmd/kapacitord/run/server_test.go). ## Child process vs Socket There are two approaches for writing UDFs. * A child process based approach where Kapacitor spawns a child process and communicates over STDIN/STDOUT. * A socket based approach where you start the UDF process externally and Kapacitor connects to it over a socket. For the socket based approach there will only ever be one instance of your UDF process running. Each use of the UDF in a TICKscript will be a new connection the socket. Where as each use of a process based UDF means a new child process is spawned for each. ## Design The protocol for communicating with Kapacitor consists of Request and Response messages. The agents wrap the communication and serialization and expose an interface that needs to be implemented to handle each request/response. In addition to the request/response paradigm agents provide a way to stream data back to Kapacitor. Your UDF is in control of when new points or batches are sent back to Kapacitor. ### Agents and Servers There are two main objects provided in the current implementations, an `Agent` and a `Server`. The `Agent` is responsible for managing the communication over input and output streams. The `Server` is responsible for accepting new connections and creating new `Agents` to handle those new connections. Both process based and socket based UDFs will need to use an `Agent` to handle the communication/serialization aspects of the protocol. Only socket based UDFs need use the `Server`. ## Writing an Agent for a new Language The UDF protocol is designed to be simple and consists of reading and writing protocol buffer messages. In order to write a UDF in the language of your choice your language must have protocol buffer support and be able to read and write to a socket. The basic steps are: 0. Add the language to the `udf/io.go` generate comment so the udf.proto code exists for your language. 1. Implement a Varint encoder/decoder, this is trivial see the python implementation. 2. Implement a method for reading and writing streamed protobuf messages. See `udf.proto` for more details. 3. Create an interface for handling each of the request/responses. 4. Write a loop for reading from an input stream and calling the handler interface, and write responses to an output stream. 5. Provide an thread safe mechanism for writing points and batches to the output stream independent of the handler interface. This is easily accomplished with a synchronized write method, see the python implementation. 6. Implement the examples using your new agent. 7. Add your example to the test suite in `cmd/kapacitord/run/server_test.go`. For process based UDFs it is expected that the process terminate after STDIN is closed and the remaining requests processed. After STDIN is closed, the agent process can continue to send Responses to Kapacitor as long as a keepalive timeout does not occur. Once a keepalive timeout is reached and after a 2*keepalive_time grace period, if the process has not terminated then it will be forcefully terminated. ## Docker It is expected that the example can run inside the test suite. Since generating different protocol buffer code requires different plugins and libraries to run we make use of Docker to provide the necessary environment. This makes testing the code easier as the developer does not have to install each supported language locally. ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/agent.go ================================================ package agent import ( "bufio" "errors" "fmt" "io" "sync" ) // The Agent calls the appropriate methods on the Handler as it receives requests over a socket. // // Returning an error from any method will cause the Agent to stop and an ErrorResponse to be sent. // Some *Response objects (like SnapshotResponse) allow for returning their own error within the object itself. // These types of errors will not stop the Agent and Kapacitor will deal with them appropriately. // // The Handler is called from a single goroutine, meaning methods will not be called concurrently. // // To write Points/Batches back to the Agent/Kapacitor use the Agent.Responses channel. type Handler interface { // Return the InfoResponse. Describing the properties of this Handler Info() (*InfoResponse, error) // Initialize the Handler with the provided options. Init(*InitRequest) (*InitResponse, error) // Create a snapshot of the running state of the handler. Snapshot() (*SnapshotResponse, error) // Restore a previous snapshot. Restore(*RestoreRequest) (*RestoreResponse, error) // A batch has begun. BeginBatch(*BeginBatch) error // A point has arrived. Point(*Point) error // The batch is complete. EndBatch(*EndBatch) error // Gracefully stop the Handler. // No other methods will be called. Stop() } // Go implementation of a Kapacitor UDF agent. // This agent is responsible for reading and writing // messages over a socket. // // The Agent requires a Handler object in order to fulfill requests. type Agent struct { in io.ReadCloser out io.WriteCloser outGroup sync.WaitGroup outResponses chan *Response responses chan *Response // A channel for writing Responses, specifically Batch and Point responses. Responses chan<- *Response writeErrC chan error readErrC chan error // The handler for requests. Handler Handler } // Create a new Agent is the provided in/out objects. // To create an Agent that reads from STDIN/STDOUT of the process use New(os.Stdin, os.Stdout) func New(in io.ReadCloser, out io.WriteCloser) *Agent { s := &Agent{ in: in, out: out, outResponses: make(chan *Response), responses: make(chan *Response), } s.Responses = s.responses return s } // Start the Agent, you must set an Handler on the agent before starting. func (a *Agent) Start() error { if a.Handler == nil { return errors.New("must set a Handler on the agent before starting") } a.readErrC = make(chan error, 1) a.writeErrC = make(chan error, 1) a.outGroup.Add(1) go func() { defer a.outGroup.Done() err := a.readLoop() if err != nil { a.outResponses <- &Response{ Message: &Response_Error{ Error: &ErrorResponse{Error: err.Error()}, }, } } a.readErrC <- err }() go func() { a.writeErrC <- a.writeLoop() }() a.outGroup.Add(1) go func() { defer a.outGroup.Done() a.forwardResponses() }() return nil } // Wait for the Agent to terminate. // The Agent will not terminate till the Responses channel is closed. // You will need to close this channel externally, typically in the Stop method for the Handler. // The Agent will terminate if the In reader is closed or an error occurs. func (a *Agent) Wait() error { a.outGroup.Wait() close(a.outResponses) for a.readErrC != nil || a.writeErrC != nil { select { case err := <-a.readErrC: a.readErrC = nil if err != nil { return fmt.Errorf("read error: %s", err) } case err := <-a.writeErrC: a.writeErrC = nil if err != nil { return fmt.Errorf("write error: %s", err) } } } return nil } func (a *Agent) readLoop() error { defer a.Handler.Stop() defer a.in.Close() in := bufio.NewReader(a.in) var buf []byte request := &Request{} for { err := ReadMessage(&buf, in, request) if err == io.EOF { break } if err != nil { return err } // Hand message to handler var res *Response switch msg := request.Message.(type) { case *Request_Info: info, err := a.Handler.Info() if err != nil { return err } res = &Response{} res.Message = &Response_Info{ Info: info, } case *Request_Init: init, err := a.Handler.Init(msg.Init) if err != nil { return err } res = &Response{} res.Message = &Response_Init{ Init: init, } case *Request_Keepalive: res = &Response{ Message: &Response_Keepalive{ Keepalive: &KeepaliveResponse{ Time: msg.Keepalive.Time, }, }, } case *Request_Snapshot: snapshot, err := a.Handler.Snapshot() if err != nil { return err } res = &Response{} res.Message = &Response_Snapshot{ Snapshot: snapshot, } case *Request_Restore: restore, err := a.Handler.Restore(msg.Restore) if err != nil { return err } res = &Response{} res.Message = &Response_Restore{ Restore: restore, } case *Request_Begin: err := a.Handler.BeginBatch(msg.Begin) if err != nil { return err } case *Request_Point: err := a.Handler.Point(msg.Point) if err != nil { return err } case *Request_End: err := a.Handler.EndBatch(msg.End) if err != nil { return err } } if res != nil { a.outResponses <- res } } return nil } func (a *Agent) writeLoop() error { defer a.out.Close() for response := range a.outResponses { err := WriteMessage(response, a.out) if err != nil { return err } } return nil } func (a *Agent) forwardResponses() { for r := range a.responses { a.outResponses <- r } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/io.go ================================================ package agent import ( "encoding/binary" "fmt" "io" "github.com/golang/protobuf/proto" ) //go:generate protoc --go_out=./ --python_out=./py/kapacitor/udf/ udf.proto // Interface for reading messages // If you have an io.Reader // wrap your reader in a bufio Reader // to stasify this interface. // // Example: // brr := bufio.NewReader(reader) type ByteReadReader interface { io.Reader io.ByteReader } // Write the message to the io.Writer with a varint size header. func WriteMessage(msg proto.Message, w io.Writer) error { // marshal message data, err := proto.Marshal(msg) if err != nil { return err } varint := make([]byte, binary.MaxVarintLen32) n := binary.PutUvarint(varint, uint64(len(data))) _, err = w.Write(varint[:n]) if err != nil { return err } _, err = w.Write(data) if err != nil { return err } return nil } // Read a message from io.ByteReader by first reading a varint size, // and then reading and decoding the message object. // If buf is not big enough a new buffer will be allocated to replace buf. func ReadMessage(buf *[]byte, r ByteReadReader, msg proto.Message) error { size, err := binary.ReadUvarint(r) if err != nil { return err } if cap(*buf) < int(size) { *buf = make([]byte, size) } b := (*buf)[:size] read := uint64(0) for read != size { n, err := r.Read(b[read:]) if err == io.EOF { return fmt.Errorf("unexpected EOF, expected %d more bytes", size) } if err != nil { return err } read += uint64(n) } err = proto.Unmarshal(b, msg) if err != nil { return err } return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/io_test.go ================================================ package agent_test import ( "bytes" "reflect" "testing" "github.com/influxdata/kapacitor/udf/agent" ) func TestMessage_ReadWrite(t *testing.T) { req := &agent.Request{} req.Message = &agent.Request_Keepalive{ Keepalive: &agent.KeepaliveRequest{ Time: 42, }, } var buf bytes.Buffer err := agent.WriteMessage(req, &buf) if err != nil { t.Fatal(err) } nreq := &agent.Request{} var b []byte err = agent.ReadMessage(&b, &buf, nreq) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(req, nreq) { t.Errorf("unexpected request: \ngot %v\nexp %v", nreq, req) } } func TestMessage_ReadWriteMultiple(t *testing.T) { req := &agent.Request{} req.Message = &agent.Request_Keepalive{ Keepalive: &agent.KeepaliveRequest{ Time: 42, }, } var buf bytes.Buffer var count int = 1e4 for i := 0; i < count; i++ { err := agent.WriteMessage(req, &buf) if err != nil { t.Fatal(err) } } nreq := &agent.Request{} var b []byte for i := 0; i < count; i++ { err := agent.ReadMessage(&b, &buf, nreq) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(req, nreq) { t.Fatalf("unexpected request: i:%d \ngot %v\nexp %v", i, nreq, req) } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/server.go ================================================ package agent import ( "net" "os" "os/signal" "sync" ) // A server accepts connections on a listener and // spawns new Agents for each connection. type Server struct { listener net.Listener accepter Accepter mu sync.Mutex stopped bool stopping chan struct{} wg sync.WaitGroup } type Accepter interface { // Accept new connections from the listener and handle them accordingly. // The typical action is to create a new Agent with the connection as both its in and out objects. Accept(net.Conn) } // Create a new server. func NewServer(l net.Listener, a Accepter) *Server { return &Server{ listener: l, accepter: a, stopping: make(chan struct{}), } } // Server starts the server and blocks. func (s *Server) Serve() error { s.mu.Lock() if s.stopped { s.mu.Unlock() return nil } s.wg.Add(1) s.mu.Unlock() defer s.wg.Done() return s.run() } // Stop closes the listener and stops all server activity. func (s *Server) Stop() { s.mu.Lock() if s.stopped { s.mu.Unlock() return } s.stopped = true s.listener.Close() s.mu.Unlock() close(s.stopping) s.wg.Wait() } // StopOnSignals registers a signal handler to stop the Server for the given signals. func (s *Server) StopOnSignals(signals ...os.Signal) { s.mu.Lock() if s.stopped { s.mu.Unlock() return } s.wg.Add(1) s.mu.Unlock() c := make(chan os.Signal) signal.Notify(c, signals...) go func() { defer s.wg.Done() select { case <-s.stopping: case <-c: s.Stop() } }() } func (s *Server) run() error { conns := make(chan net.Conn) errC := make(chan error, 1) s.wg.Add(1) go func() { defer s.wg.Done() for { conn, err := s.listener.Accept() if err != nil { errC <- err } conns <- conn } }() for { select { case <-s.stopping: return nil case err := <-errC: s.mu.Lock() stopped := s.stopped s.mu.Unlock() if stopped { return nil } return err case conn := <-conns: s.accepter.Accept(conn) } } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/udf.pb.go ================================================ // Code generated by protoc-gen-go. // source: udf.proto // DO NOT EDIT! /* Package agent is a generated protocol buffer package. It is generated from these files: udf.proto It has these top-level messages: InfoRequest InfoResponse OptionInfo InitRequest Option OptionValue InitResponse SnapshotRequest SnapshotResponse RestoreRequest RestoreResponse KeepaliveRequest KeepaliveResponse ErrorResponse BeginBatch Point EndBatch Request Response */ package agent import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type EdgeType int32 const ( EdgeType_STREAM EdgeType = 0 EdgeType_BATCH EdgeType = 1 ) var EdgeType_name = map[int32]string{ 0: "STREAM", 1: "BATCH", } var EdgeType_value = map[string]int32{ "STREAM": 0, "BATCH": 1, } func (x EdgeType) String() string { return proto.EnumName(EdgeType_name, int32(x)) } func (EdgeType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } type ValueType int32 const ( ValueType_BOOL ValueType = 0 ValueType_INT ValueType = 1 ValueType_DOUBLE ValueType = 2 ValueType_STRING ValueType = 3 ValueType_DURATION ValueType = 4 ) var ValueType_name = map[int32]string{ 0: "BOOL", 1: "INT", 2: "DOUBLE", 3: "STRING", 4: "DURATION", } var ValueType_value = map[string]int32{ "BOOL": 0, "INT": 1, "DOUBLE": 2, "STRING": 3, "DURATION": 4, } func (x ValueType) String() string { return proto.EnumName(ValueType_name, int32(x)) } func (ValueType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } // Request that the process return information about available Options. type InfoRequest struct { } func (m *InfoRequest) Reset() { *m = InfoRequest{} } func (m *InfoRequest) String() string { return proto.CompactTextString(m) } func (*InfoRequest) ProtoMessage() {} func (*InfoRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } type InfoResponse struct { Wants EdgeType `protobuf:"varint,1,opt,name=wants,enum=agent.EdgeType" json:"wants,omitempty"` Provides EdgeType `protobuf:"varint,2,opt,name=provides,enum=agent.EdgeType" json:"provides,omitempty"` Options map[string]*OptionInfo `protobuf:"bytes,3,rep,name=options" json:"options,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` } func (m *InfoResponse) Reset() { *m = InfoResponse{} } func (m *InfoResponse) String() string { return proto.CompactTextString(m) } func (*InfoResponse) ProtoMessage() {} func (*InfoResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } func (m *InfoResponse) GetOptions() map[string]*OptionInfo { if m != nil { return m.Options } return nil } type OptionInfo struct { ValueTypes []ValueType `protobuf:"varint,1,rep,name=valueTypes,enum=agent.ValueType" json:"valueTypes,omitempty"` } func (m *OptionInfo) Reset() { *m = OptionInfo{} } func (m *OptionInfo) String() string { return proto.CompactTextString(m) } func (*OptionInfo) ProtoMessage() {} func (*OptionInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } // Request that the process initialize itself with the provided options. type InitRequest struct { Options []*Option `protobuf:"bytes,1,rep,name=options" json:"options,omitempty"` TaskID string `protobuf:"bytes,2,opt,name=taskID" json:"taskID,omitempty"` NodeID string `protobuf:"bytes,3,opt,name=nodeID" json:"nodeID,omitempty"` } func (m *InitRequest) Reset() { *m = InitRequest{} } func (m *InitRequest) String() string { return proto.CompactTextString(m) } func (*InitRequest) ProtoMessage() {} func (*InitRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (m *InitRequest) GetOptions() []*Option { if m != nil { return m.Options } return nil } type Option struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` Values []*OptionValue `protobuf:"bytes,2,rep,name=values" json:"values,omitempty"` } func (m *Option) Reset() { *m = Option{} } func (m *Option) String() string { return proto.CompactTextString(m) } func (*Option) ProtoMessage() {} func (*Option) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *Option) GetValues() []*OptionValue { if m != nil { return m.Values } return nil } type OptionValue struct { Type ValueType `protobuf:"varint,1,opt,name=type,enum=agent.ValueType" json:"type,omitempty"` // Types that are valid to be assigned to Value: // *OptionValue_BoolValue // *OptionValue_IntValue // *OptionValue_DoubleValue // *OptionValue_StringValue // *OptionValue_DurationValue Value isOptionValue_Value `protobuf_oneof:"value"` } func (m *OptionValue) Reset() { *m = OptionValue{} } func (m *OptionValue) String() string { return proto.CompactTextString(m) } func (*OptionValue) ProtoMessage() {} func (*OptionValue) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } type isOptionValue_Value interface { isOptionValue_Value() } type OptionValue_BoolValue struct { BoolValue bool `protobuf:"varint,2,opt,name=boolValue,oneof"` } type OptionValue_IntValue struct { IntValue int64 `protobuf:"varint,3,opt,name=intValue,oneof"` } type OptionValue_DoubleValue struct { DoubleValue float64 `protobuf:"fixed64,4,opt,name=doubleValue,oneof"` } type OptionValue_StringValue struct { StringValue string `protobuf:"bytes,5,opt,name=stringValue,oneof"` } type OptionValue_DurationValue struct { DurationValue int64 `protobuf:"varint,6,opt,name=durationValue,oneof"` } func (*OptionValue_BoolValue) isOptionValue_Value() {} func (*OptionValue_IntValue) isOptionValue_Value() {} func (*OptionValue_DoubleValue) isOptionValue_Value() {} func (*OptionValue_StringValue) isOptionValue_Value() {} func (*OptionValue_DurationValue) isOptionValue_Value() {} func (m *OptionValue) GetValue() isOptionValue_Value { if m != nil { return m.Value } return nil } func (m *OptionValue) GetBoolValue() bool { if x, ok := m.GetValue().(*OptionValue_BoolValue); ok { return x.BoolValue } return false } func (m *OptionValue) GetIntValue() int64 { if x, ok := m.GetValue().(*OptionValue_IntValue); ok { return x.IntValue } return 0 } func (m *OptionValue) GetDoubleValue() float64 { if x, ok := m.GetValue().(*OptionValue_DoubleValue); ok { return x.DoubleValue } return 0 } func (m *OptionValue) GetStringValue() string { if x, ok := m.GetValue().(*OptionValue_StringValue); ok { return x.StringValue } return "" } func (m *OptionValue) GetDurationValue() int64 { if x, ok := m.GetValue().(*OptionValue_DurationValue); ok { return x.DurationValue } return 0 } // XXX_OneofFuncs is for the internal use of the proto package. func (*OptionValue) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { return _OptionValue_OneofMarshaler, _OptionValue_OneofUnmarshaler, _OptionValue_OneofSizer, []interface{}{ (*OptionValue_BoolValue)(nil), (*OptionValue_IntValue)(nil), (*OptionValue_DoubleValue)(nil), (*OptionValue_StringValue)(nil), (*OptionValue_DurationValue)(nil), } } func _OptionValue_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { m := msg.(*OptionValue) // value switch x := m.Value.(type) { case *OptionValue_BoolValue: t := uint64(0) if x.BoolValue { t = 1 } b.EncodeVarint(2<<3 | proto.WireVarint) b.EncodeVarint(t) case *OptionValue_IntValue: b.EncodeVarint(3<<3 | proto.WireVarint) b.EncodeVarint(uint64(x.IntValue)) case *OptionValue_DoubleValue: b.EncodeVarint(4<<3 | proto.WireFixed64) b.EncodeFixed64(math.Float64bits(x.DoubleValue)) case *OptionValue_StringValue: b.EncodeVarint(5<<3 | proto.WireBytes) b.EncodeStringBytes(x.StringValue) case *OptionValue_DurationValue: b.EncodeVarint(6<<3 | proto.WireVarint) b.EncodeVarint(uint64(x.DurationValue)) case nil: default: return fmt.Errorf("OptionValue.Value has unexpected type %T", x) } return nil } func _OptionValue_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { m := msg.(*OptionValue) switch tag { case 2: // value.boolValue if wire != proto.WireVarint { return true, proto.ErrInternalBadWireType } x, err := b.DecodeVarint() m.Value = &OptionValue_BoolValue{x != 0} return true, err case 3: // value.intValue if wire != proto.WireVarint { return true, proto.ErrInternalBadWireType } x, err := b.DecodeVarint() m.Value = &OptionValue_IntValue{int64(x)} return true, err case 4: // value.doubleValue if wire != proto.WireFixed64 { return true, proto.ErrInternalBadWireType } x, err := b.DecodeFixed64() m.Value = &OptionValue_DoubleValue{math.Float64frombits(x)} return true, err case 5: // value.stringValue if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } x, err := b.DecodeStringBytes() m.Value = &OptionValue_StringValue{x} return true, err case 6: // value.durationValue if wire != proto.WireVarint { return true, proto.ErrInternalBadWireType } x, err := b.DecodeVarint() m.Value = &OptionValue_DurationValue{int64(x)} return true, err default: return false, nil } } func _OptionValue_OneofSizer(msg proto.Message) (n int) { m := msg.(*OptionValue) // value switch x := m.Value.(type) { case *OptionValue_BoolValue: n += proto.SizeVarint(2<<3 | proto.WireVarint) n += 1 case *OptionValue_IntValue: n += proto.SizeVarint(3<<3 | proto.WireVarint) n += proto.SizeVarint(uint64(x.IntValue)) case *OptionValue_DoubleValue: n += proto.SizeVarint(4<<3 | proto.WireFixed64) n += 8 case *OptionValue_StringValue: n += proto.SizeVarint(5<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(len(x.StringValue))) n += len(x.StringValue) case *OptionValue_DurationValue: n += proto.SizeVarint(6<<3 | proto.WireVarint) n += proto.SizeVarint(uint64(x.DurationValue)) case nil: default: panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) } return n } // Respond to Kapacitor whether initialization was successful. type InitResponse struct { Success bool `protobuf:"varint,1,opt,name=success" json:"success,omitempty"` Error string `protobuf:"bytes,2,opt,name=error" json:"error,omitempty"` } func (m *InitResponse) Reset() { *m = InitResponse{} } func (m *InitResponse) String() string { return proto.CompactTextString(m) } func (*InitResponse) ProtoMessage() {} func (*InitResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } // Request that the process provide a snapshot of its state. type SnapshotRequest struct { } func (m *SnapshotRequest) Reset() { *m = SnapshotRequest{} } func (m *SnapshotRequest) String() string { return proto.CompactTextString(m) } func (*SnapshotRequest) ProtoMessage() {} func (*SnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } // Respond to Kapacitor with a serialized snapshot of the running state. type SnapshotResponse struct { Snapshot []byte `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` } func (m *SnapshotResponse) Reset() { *m = SnapshotResponse{} } func (m *SnapshotResponse) String() string { return proto.CompactTextString(m) } func (*SnapshotResponse) ProtoMessage() {} func (*SnapshotResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } // Request that the process restore its state from a snapshot. type RestoreRequest struct { Snapshot []byte `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` } func (m *RestoreRequest) Reset() { *m = RestoreRequest{} } func (m *RestoreRequest) String() string { return proto.CompactTextString(m) } func (*RestoreRequest) ProtoMessage() {} func (*RestoreRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } // Respond with success or failure to a RestoreRequest type RestoreResponse struct { Success bool `protobuf:"varint,1,opt,name=success" json:"success,omitempty"` Error string `protobuf:"bytes,2,opt,name=error" json:"error,omitempty"` } func (m *RestoreResponse) Reset() { *m = RestoreResponse{} } func (m *RestoreResponse) String() string { return proto.CompactTextString(m) } func (*RestoreResponse) ProtoMessage() {} func (*RestoreResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } // Request that the process respond with a Keepalive to verify it is responding. type KeepaliveRequest struct { // The number of nanoseconds since the epoch. // Used only for debugging keepalive requests. Time int64 `protobuf:"varint,1,opt,name=time" json:"time,omitempty"` } func (m *KeepaliveRequest) Reset() { *m = KeepaliveRequest{} } func (m *KeepaliveRequest) String() string { return proto.CompactTextString(m) } func (*KeepaliveRequest) ProtoMessage() {} func (*KeepaliveRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } // Respond to KeepaliveRequest type KeepaliveResponse struct { // The number of nanoseconds since the epoch. // Used only for debugging keepalive requests. Time int64 `protobuf:"varint,1,opt,name=time" json:"time,omitempty"` } func (m *KeepaliveResponse) Reset() { *m = KeepaliveResponse{} } func (m *KeepaliveResponse) String() string { return proto.CompactTextString(m) } func (*KeepaliveResponse) ProtoMessage() {} func (*KeepaliveResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } // Sent from the process to Kapacitor indicating an error has occurred. // If an ErrorResponse is received, Kapacitor will terminate the process. type ErrorResponse struct { Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"` } func (m *ErrorResponse) Reset() { *m = ErrorResponse{} } func (m *ErrorResponse) String() string { return proto.CompactTextString(m) } func (*ErrorResponse) ProtoMessage() {} func (*ErrorResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } // Indicates the beginning of a batch. // All subsequent points should be considered // part of the batch until EndBatch arrives. // This includes grouping. Batches of // differing groups may not be interleaved. // // All the meta data but tmax is provided, // since tmax may not be known at // the beginning of a batch. // // Size is the number of points in the batch. // If size is 0 then the batch has an undetermined size. type BeginBatch struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` Group string `protobuf:"bytes,2,opt,name=group" json:"group,omitempty"` Tags map[string]string `protobuf:"bytes,3,rep,name=tags" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` Size int64 `protobuf:"varint,4,opt,name=size" json:"size,omitempty"` ByName bool `protobuf:"varint,5,opt,name=byName" json:"byName,omitempty"` } func (m *BeginBatch) Reset() { *m = BeginBatch{} } func (m *BeginBatch) String() string { return proto.CompactTextString(m) } func (*BeginBatch) ProtoMessage() {} func (*BeginBatch) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } func (m *BeginBatch) GetTags() map[string]string { if m != nil { return m.Tags } return nil } // Message containing information about a single data point. // Can be sent on it's own or bookended by BeginBatch and EndBatch messages. type Point struct { Time int64 `protobuf:"varint,1,opt,name=time" json:"time,omitempty"` Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` Database string `protobuf:"bytes,3,opt,name=database" json:"database,omitempty"` RetentionPolicy string `protobuf:"bytes,4,opt,name=retentionPolicy" json:"retentionPolicy,omitempty"` Group string `protobuf:"bytes,5,opt,name=group" json:"group,omitempty"` Dimensions []string `protobuf:"bytes,6,rep,name=dimensions" json:"dimensions,omitempty"` Tags map[string]string `protobuf:"bytes,7,rep,name=tags" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` FieldsDouble map[string]float64 `protobuf:"bytes,8,rep,name=fieldsDouble" json:"fieldsDouble,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"` FieldsInt map[string]int64 `protobuf:"bytes,9,rep,name=fieldsInt" json:"fieldsInt,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` FieldsString map[string]string `protobuf:"bytes,10,rep,name=fieldsString" json:"fieldsString,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` FieldsBool map[string]bool `protobuf:"bytes,12,rep,name=fieldsBool" json:"fieldsBool,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` ByName bool `protobuf:"varint,11,opt,name=byName" json:"byName,omitempty"` } func (m *Point) Reset() { *m = Point{} } func (m *Point) String() string { return proto.CompactTextString(m) } func (*Point) ProtoMessage() {} func (*Point) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } func (m *Point) GetTags() map[string]string { if m != nil { return m.Tags } return nil } func (m *Point) GetFieldsDouble() map[string]float64 { if m != nil { return m.FieldsDouble } return nil } func (m *Point) GetFieldsInt() map[string]int64 { if m != nil { return m.FieldsInt } return nil } func (m *Point) GetFieldsString() map[string]string { if m != nil { return m.FieldsString } return nil } func (m *Point) GetFieldsBool() map[string]bool { if m != nil { return m.FieldsBool } return nil } // Indicates the end of a batch and contains // all meta data associated with the batch. // The same meta information is provided for // ease of use with the addition of tmax since it // may not be know at BeginBatch. type EndBatch struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` Group string `protobuf:"bytes,2,opt,name=group" json:"group,omitempty"` Tmax int64 `protobuf:"varint,3,opt,name=tmax" json:"tmax,omitempty"` Tags map[string]string `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` ByName bool `protobuf:"varint,5,opt,name=byName" json:"byName,omitempty"` } func (m *EndBatch) Reset() { *m = EndBatch{} } func (m *EndBatch) String() string { return proto.CompactTextString(m) } func (*EndBatch) ProtoMessage() {} func (*EndBatch) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } func (m *EndBatch) GetTags() map[string]string { if m != nil { return m.Tags } return nil } // Request message wrapper -- sent from Kapacitor to process type Request struct { // Types that are valid to be assigned to Message: // *Request_Info // *Request_Init // *Request_Keepalive // *Request_Snapshot // *Request_Restore // *Request_Begin // *Request_Point // *Request_End Message isRequest_Message `protobuf_oneof:"message"` } func (m *Request) Reset() { *m = Request{} } func (m *Request) String() string { return proto.CompactTextString(m) } func (*Request) ProtoMessage() {} func (*Request) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } type isRequest_Message interface { isRequest_Message() } type Request_Info struct { Info *InfoRequest `protobuf:"bytes,1,opt,name=info,oneof"` } type Request_Init struct { Init *InitRequest `protobuf:"bytes,2,opt,name=init,oneof"` } type Request_Keepalive struct { Keepalive *KeepaliveRequest `protobuf:"bytes,3,opt,name=keepalive,oneof"` } type Request_Snapshot struct { Snapshot *SnapshotRequest `protobuf:"bytes,4,opt,name=snapshot,oneof"` } type Request_Restore struct { Restore *RestoreRequest `protobuf:"bytes,5,opt,name=restore,oneof"` } type Request_Begin struct { Begin *BeginBatch `protobuf:"bytes,16,opt,name=begin,oneof"` } type Request_Point struct { Point *Point `protobuf:"bytes,17,opt,name=point,oneof"` } type Request_End struct { End *EndBatch `protobuf:"bytes,18,opt,name=end,oneof"` } func (*Request_Info) isRequest_Message() {} func (*Request_Init) isRequest_Message() {} func (*Request_Keepalive) isRequest_Message() {} func (*Request_Snapshot) isRequest_Message() {} func (*Request_Restore) isRequest_Message() {} func (*Request_Begin) isRequest_Message() {} func (*Request_Point) isRequest_Message() {} func (*Request_End) isRequest_Message() {} func (m *Request) GetMessage() isRequest_Message { if m != nil { return m.Message } return nil } func (m *Request) GetInfo() *InfoRequest { if x, ok := m.GetMessage().(*Request_Info); ok { return x.Info } return nil } func (m *Request) GetInit() *InitRequest { if x, ok := m.GetMessage().(*Request_Init); ok { return x.Init } return nil } func (m *Request) GetKeepalive() *KeepaliveRequest { if x, ok := m.GetMessage().(*Request_Keepalive); ok { return x.Keepalive } return nil } func (m *Request) GetSnapshot() *SnapshotRequest { if x, ok := m.GetMessage().(*Request_Snapshot); ok { return x.Snapshot } return nil } func (m *Request) GetRestore() *RestoreRequest { if x, ok := m.GetMessage().(*Request_Restore); ok { return x.Restore } return nil } func (m *Request) GetBegin() *BeginBatch { if x, ok := m.GetMessage().(*Request_Begin); ok { return x.Begin } return nil } func (m *Request) GetPoint() *Point { if x, ok := m.GetMessage().(*Request_Point); ok { return x.Point } return nil } func (m *Request) GetEnd() *EndBatch { if x, ok := m.GetMessage().(*Request_End); ok { return x.End } return nil } // XXX_OneofFuncs is for the internal use of the proto package. func (*Request) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { return _Request_OneofMarshaler, _Request_OneofUnmarshaler, _Request_OneofSizer, []interface{}{ (*Request_Info)(nil), (*Request_Init)(nil), (*Request_Keepalive)(nil), (*Request_Snapshot)(nil), (*Request_Restore)(nil), (*Request_Begin)(nil), (*Request_Point)(nil), (*Request_End)(nil), } } func _Request_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { m := msg.(*Request) // message switch x := m.Message.(type) { case *Request_Info: b.EncodeVarint(1<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Info); err != nil { return err } case *Request_Init: b.EncodeVarint(2<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Init); err != nil { return err } case *Request_Keepalive: b.EncodeVarint(3<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Keepalive); err != nil { return err } case *Request_Snapshot: b.EncodeVarint(4<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Snapshot); err != nil { return err } case *Request_Restore: b.EncodeVarint(5<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Restore); err != nil { return err } case *Request_Begin: b.EncodeVarint(16<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Begin); err != nil { return err } case *Request_Point: b.EncodeVarint(17<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Point); err != nil { return err } case *Request_End: b.EncodeVarint(18<<3 | proto.WireBytes) if err := b.EncodeMessage(x.End); err != nil { return err } case nil: default: return fmt.Errorf("Request.Message has unexpected type %T", x) } return nil } func _Request_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { m := msg.(*Request) switch tag { case 1: // message.info if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(InfoRequest) err := b.DecodeMessage(msg) m.Message = &Request_Info{msg} return true, err case 2: // message.init if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(InitRequest) err := b.DecodeMessage(msg) m.Message = &Request_Init{msg} return true, err case 3: // message.keepalive if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(KeepaliveRequest) err := b.DecodeMessage(msg) m.Message = &Request_Keepalive{msg} return true, err case 4: // message.snapshot if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(SnapshotRequest) err := b.DecodeMessage(msg) m.Message = &Request_Snapshot{msg} return true, err case 5: // message.restore if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(RestoreRequest) err := b.DecodeMessage(msg) m.Message = &Request_Restore{msg} return true, err case 16: // message.begin if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(BeginBatch) err := b.DecodeMessage(msg) m.Message = &Request_Begin{msg} return true, err case 17: // message.point if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(Point) err := b.DecodeMessage(msg) m.Message = &Request_Point{msg} return true, err case 18: // message.end if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(EndBatch) err := b.DecodeMessage(msg) m.Message = &Request_End{msg} return true, err default: return false, nil } } func _Request_OneofSizer(msg proto.Message) (n int) { m := msg.(*Request) // message switch x := m.Message.(type) { case *Request_Info: s := proto.Size(x.Info) n += proto.SizeVarint(1<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_Init: s := proto.Size(x.Init) n += proto.SizeVarint(2<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_Keepalive: s := proto.Size(x.Keepalive) n += proto.SizeVarint(3<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_Snapshot: s := proto.Size(x.Snapshot) n += proto.SizeVarint(4<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_Restore: s := proto.Size(x.Restore) n += proto.SizeVarint(5<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_Begin: s := proto.Size(x.Begin) n += proto.SizeVarint(16<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_Point: s := proto.Size(x.Point) n += proto.SizeVarint(17<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Request_End: s := proto.Size(x.End) n += proto.SizeVarint(18<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case nil: default: panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) } return n } // Response message wrapper -- sent from process to Kapacitor type Response struct { // Types that are valid to be assigned to Message: // *Response_Info // *Response_Init // *Response_Keepalive // *Response_Snapshot // *Response_Restore // *Response_Error // *Response_Begin // *Response_Point // *Response_End Message isResponse_Message `protobuf_oneof:"message"` } func (m *Response) Reset() { *m = Response{} } func (m *Response) String() string { return proto.CompactTextString(m) } func (*Response) ProtoMessage() {} func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } type isResponse_Message interface { isResponse_Message() } type Response_Info struct { Info *InfoResponse `protobuf:"bytes,1,opt,name=info,oneof"` } type Response_Init struct { Init *InitResponse `protobuf:"bytes,2,opt,name=init,oneof"` } type Response_Keepalive struct { Keepalive *KeepaliveResponse `protobuf:"bytes,3,opt,name=keepalive,oneof"` } type Response_Snapshot struct { Snapshot *SnapshotResponse `protobuf:"bytes,4,opt,name=snapshot,oneof"` } type Response_Restore struct { Restore *RestoreResponse `protobuf:"bytes,5,opt,name=restore,oneof"` } type Response_Error struct { Error *ErrorResponse `protobuf:"bytes,6,opt,name=error,oneof"` } type Response_Begin struct { Begin *BeginBatch `protobuf:"bytes,16,opt,name=begin,oneof"` } type Response_Point struct { Point *Point `protobuf:"bytes,17,opt,name=point,oneof"` } type Response_End struct { End *EndBatch `protobuf:"bytes,18,opt,name=end,oneof"` } func (*Response_Info) isResponse_Message() {} func (*Response_Init) isResponse_Message() {} func (*Response_Keepalive) isResponse_Message() {} func (*Response_Snapshot) isResponse_Message() {} func (*Response_Restore) isResponse_Message() {} func (*Response_Error) isResponse_Message() {} func (*Response_Begin) isResponse_Message() {} func (*Response_Point) isResponse_Message() {} func (*Response_End) isResponse_Message() {} func (m *Response) GetMessage() isResponse_Message { if m != nil { return m.Message } return nil } func (m *Response) GetInfo() *InfoResponse { if x, ok := m.GetMessage().(*Response_Info); ok { return x.Info } return nil } func (m *Response) GetInit() *InitResponse { if x, ok := m.GetMessage().(*Response_Init); ok { return x.Init } return nil } func (m *Response) GetKeepalive() *KeepaliveResponse { if x, ok := m.GetMessage().(*Response_Keepalive); ok { return x.Keepalive } return nil } func (m *Response) GetSnapshot() *SnapshotResponse { if x, ok := m.GetMessage().(*Response_Snapshot); ok { return x.Snapshot } return nil } func (m *Response) GetRestore() *RestoreResponse { if x, ok := m.GetMessage().(*Response_Restore); ok { return x.Restore } return nil } func (m *Response) GetError() *ErrorResponse { if x, ok := m.GetMessage().(*Response_Error); ok { return x.Error } return nil } func (m *Response) GetBegin() *BeginBatch { if x, ok := m.GetMessage().(*Response_Begin); ok { return x.Begin } return nil } func (m *Response) GetPoint() *Point { if x, ok := m.GetMessage().(*Response_Point); ok { return x.Point } return nil } func (m *Response) GetEnd() *EndBatch { if x, ok := m.GetMessage().(*Response_End); ok { return x.End } return nil } // XXX_OneofFuncs is for the internal use of the proto package. func (*Response) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { return _Response_OneofMarshaler, _Response_OneofUnmarshaler, _Response_OneofSizer, []interface{}{ (*Response_Info)(nil), (*Response_Init)(nil), (*Response_Keepalive)(nil), (*Response_Snapshot)(nil), (*Response_Restore)(nil), (*Response_Error)(nil), (*Response_Begin)(nil), (*Response_Point)(nil), (*Response_End)(nil), } } func _Response_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { m := msg.(*Response) // message switch x := m.Message.(type) { case *Response_Info: b.EncodeVarint(1<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Info); err != nil { return err } case *Response_Init: b.EncodeVarint(2<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Init); err != nil { return err } case *Response_Keepalive: b.EncodeVarint(3<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Keepalive); err != nil { return err } case *Response_Snapshot: b.EncodeVarint(4<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Snapshot); err != nil { return err } case *Response_Restore: b.EncodeVarint(5<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Restore); err != nil { return err } case *Response_Error: b.EncodeVarint(6<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Error); err != nil { return err } case *Response_Begin: b.EncodeVarint(16<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Begin); err != nil { return err } case *Response_Point: b.EncodeVarint(17<<3 | proto.WireBytes) if err := b.EncodeMessage(x.Point); err != nil { return err } case *Response_End: b.EncodeVarint(18<<3 | proto.WireBytes) if err := b.EncodeMessage(x.End); err != nil { return err } case nil: default: return fmt.Errorf("Response.Message has unexpected type %T", x) } return nil } func _Response_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { m := msg.(*Response) switch tag { case 1: // message.info if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(InfoResponse) err := b.DecodeMessage(msg) m.Message = &Response_Info{msg} return true, err case 2: // message.init if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(InitResponse) err := b.DecodeMessage(msg) m.Message = &Response_Init{msg} return true, err case 3: // message.keepalive if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(KeepaliveResponse) err := b.DecodeMessage(msg) m.Message = &Response_Keepalive{msg} return true, err case 4: // message.snapshot if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(SnapshotResponse) err := b.DecodeMessage(msg) m.Message = &Response_Snapshot{msg} return true, err case 5: // message.restore if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(RestoreResponse) err := b.DecodeMessage(msg) m.Message = &Response_Restore{msg} return true, err case 6: // message.error if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(ErrorResponse) err := b.DecodeMessage(msg) m.Message = &Response_Error{msg} return true, err case 16: // message.begin if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(BeginBatch) err := b.DecodeMessage(msg) m.Message = &Response_Begin{msg} return true, err case 17: // message.point if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(Point) err := b.DecodeMessage(msg) m.Message = &Response_Point{msg} return true, err case 18: // message.end if wire != proto.WireBytes { return true, proto.ErrInternalBadWireType } msg := new(EndBatch) err := b.DecodeMessage(msg) m.Message = &Response_End{msg} return true, err default: return false, nil } } func _Response_OneofSizer(msg proto.Message) (n int) { m := msg.(*Response) // message switch x := m.Message.(type) { case *Response_Info: s := proto.Size(x.Info) n += proto.SizeVarint(1<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Init: s := proto.Size(x.Init) n += proto.SizeVarint(2<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Keepalive: s := proto.Size(x.Keepalive) n += proto.SizeVarint(3<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Snapshot: s := proto.Size(x.Snapshot) n += proto.SizeVarint(4<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Restore: s := proto.Size(x.Restore) n += proto.SizeVarint(5<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Error: s := proto.Size(x.Error) n += proto.SizeVarint(6<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Begin: s := proto.Size(x.Begin) n += proto.SizeVarint(16<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_Point: s := proto.Size(x.Point) n += proto.SizeVarint(17<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case *Response_End: s := proto.Size(x.End) n += proto.SizeVarint(18<<3 | proto.WireBytes) n += proto.SizeVarint(uint64(s)) n += s case nil: default: panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) } return n } func init() { proto.RegisterType((*InfoRequest)(nil), "agent.InfoRequest") proto.RegisterType((*InfoResponse)(nil), "agent.InfoResponse") proto.RegisterType((*OptionInfo)(nil), "agent.OptionInfo") proto.RegisterType((*InitRequest)(nil), "agent.InitRequest") proto.RegisterType((*Option)(nil), "agent.Option") proto.RegisterType((*OptionValue)(nil), "agent.OptionValue") proto.RegisterType((*InitResponse)(nil), "agent.InitResponse") proto.RegisterType((*SnapshotRequest)(nil), "agent.SnapshotRequest") proto.RegisterType((*SnapshotResponse)(nil), "agent.SnapshotResponse") proto.RegisterType((*RestoreRequest)(nil), "agent.RestoreRequest") proto.RegisterType((*RestoreResponse)(nil), "agent.RestoreResponse") proto.RegisterType((*KeepaliveRequest)(nil), "agent.KeepaliveRequest") proto.RegisterType((*KeepaliveResponse)(nil), "agent.KeepaliveResponse") proto.RegisterType((*ErrorResponse)(nil), "agent.ErrorResponse") proto.RegisterType((*BeginBatch)(nil), "agent.BeginBatch") proto.RegisterType((*Point)(nil), "agent.Point") proto.RegisterType((*EndBatch)(nil), "agent.EndBatch") proto.RegisterType((*Request)(nil), "agent.Request") proto.RegisterType((*Response)(nil), "agent.Response") proto.RegisterEnum("agent.EdgeType", EdgeType_name, EdgeType_value) proto.RegisterEnum("agent.ValueType", ValueType_name, ValueType_value) } func init() { proto.RegisterFile("udf.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ // 1151 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x57, 0xdd, 0x72, 0xdb, 0xc4, 0x17, 0xaf, 0x22, 0xcb, 0x96, 0x8e, 0x9d, 0x58, 0xde, 0xe6, 0x9f, 0xea, 0x1f, 0x3a, 0x99, 0x20, 0x68, 0x9b, 0x84, 0x62, 0xc0, 0xc0, 0x50, 0x3a, 0x05, 0x26, 0xc6, 0x86, 0x78, 0x68, 0xe3, 0x8e, 0xe2, 0xf6, 0x5e, 0x8e, 0x36, 0xae, 0x26, 0x8e, 0x64, 0x24, 0x39, 0x60, 0xae, 0x78, 0x1c, 0x1e, 0x80, 0x87, 0xe0, 0x82, 0x27, 0x61, 0x86, 0x77, 0x60, 0xbf, 0xb4, 0x5a, 0xd9, 0x86, 0x4c, 0x99, 0xce, 0x70, 0xa7, 0x3d, 0xe7, 0x77, 0xbe, 0xcf, 0x9e, 0xb3, 0x02, 0x6b, 0x1e, 0x5c, 0xb4, 0x67, 0x49, 0x9c, 0xc5, 0xc8, 0xf0, 0x27, 0x38, 0xca, 0xdc, 0x4d, 0xa8, 0x0f, 0xa2, 0x8b, 0xd8, 0xc3, 0xdf, 0xcf, 0x71, 0x9a, 0xb9, 0x7f, 0x6a, 0xd0, 0xe0, 0xe7, 0x74, 0x16, 0x47, 0x29, 0x46, 0xf7, 0xc0, 0xf8, 0xc1, 0x8f, 0xb2, 0xd4, 0xd1, 0xf6, 0xb5, 0x83, 0xad, 0x4e, 0xb3, 0xcd, 0xc4, 0xda, 0xfd, 0x60, 0x82, 0x47, 0x8b, 0x19, 0xf6, 0x38, 0x17, 0xbd, 0x07, 0x26, 0x51, 0x7b, 0x1d, 0x06, 0x38, 0x75, 0x36, 0xd6, 0x23, 0x25, 0x00, 0x3d, 0x86, 0x5a, 0x3c, 0xcb, 0x42, 0xa2, 0xdf, 0xd1, 0xf7, 0xf5, 0x83, 0x7a, 0x67, 0x5f, 0x60, 0x55, 0xcb, 0xed, 0x21, 0x87, 0xf4, 0xa3, 0x2c, 0x59, 0x78, 0xb9, 0xc0, 0xee, 0x33, 0x68, 0xa8, 0x0c, 0x64, 0x83, 0x7e, 0x89, 0x17, 0xcc, 0x3b, 0xcb, 0xa3, 0x9f, 0xe8, 0x01, 0x18, 0xd7, 0xfe, 0x74, 0x8e, 0x99, 0x1f, 0xf5, 0x4e, 0x4b, 0xe8, 0xe6, 0x52, 0xcc, 0x02, 0xe7, 0x3f, 0xde, 0x78, 0xa4, 0xb9, 0x5f, 0x02, 0x14, 0x0c, 0xf4, 0x21, 0x00, 0x63, 0x51, 0x7f, 0x69, 0xc4, 0x3a, 0x89, 0xc3, 0x16, 0xf2, 0x2f, 0x73, 0x86, 0xa7, 0x60, 0xdc, 0x0b, 0x9a, 0xbe, 0x30, 0x13, 0xe9, 0x23, 0xb6, 0x65, 0x64, 0x1a, 0x8b, 0x6c, 0xb3, 0x64, 0x5d, 0x86, 0x81, 0x76, 0xa0, 0x9a, 0xf9, 0xe9, 0xe5, 0xa0, 0xc7, 0xbc, 0xb4, 0x3c, 0x71, 0xa2, 0xf4, 0x28, 0x0e, 0x30, 0xa1, 0xeb, 0x9c, 0xce, 0x4f, 0xee, 0x09, 0x54, 0xb9, 0x0a, 0x84, 0xa0, 0x12, 0xf9, 0x57, 0x58, 0x44, 0xcc, 0xbe, 0xd1, 0x11, 0x54, 0x99, 0x4f, 0x34, 0xf7, 0xd4, 0x2a, 0x2a, 0x59, 0x65, 0x9e, 0x7b, 0x02, 0xe1, 0xfe, 0xa1, 0x41, 0x5d, 0xa1, 0xa3, 0x77, 0xa1, 0x92, 0x91, 0x50, 0x44, 0x7d, 0x57, 0xa3, 0x65, 0x5c, 0xb4, 0x07, 0xd6, 0x38, 0x8e, 0xa7, 0x2f, 0x65, 0x62, 0xcd, 0x93, 0x5b, 0x5e, 0x41, 0x42, 0x77, 0xc1, 0x0c, 0xa3, 0x8c, 0xb3, 0xa9, 0xe7, 0x3a, 0x61, 0x4b, 0x0a, 0x72, 0xa1, 0x1e, 0xc4, 0xf3, 0xf1, 0x14, 0x73, 0x40, 0x85, 0x00, 0x34, 0x02, 0x50, 0x89, 0x14, 0x93, 0x66, 0x49, 0x18, 0x4d, 0x38, 0xc6, 0xa0, 0xe1, 0x51, 0x8c, 0x42, 0x44, 0xf7, 0x61, 0x33, 0x98, 0x27, 0xbe, 0x74, 0xde, 0xa9, 0x0a, 0x53, 0x65, 0x72, 0xb7, 0x26, 0x5a, 0x80, 0x94, 0xb7, 0xc1, 0xcb, 0x23, 0xba, 0xd9, 0x81, 0x5a, 0x3a, 0x3f, 0x3f, 0xc7, 0x29, 0xef, 0x67, 0xd3, 0xcb, 0x8f, 0x68, 0x1b, 0x0c, 0x9c, 0x24, 0x71, 0x22, 0xea, 0xc1, 0x0f, 0x6e, 0x0b, 0x9a, 0x67, 0x91, 0x3f, 0x4b, 0x5f, 0xc5, 0x79, 0x89, 0xdd, 0x36, 0xd8, 0x05, 0x49, 0xa8, 0xdd, 0x05, 0x33, 0x15, 0x34, 0xa6, 0xb7, 0xe1, 0xc9, 0xb3, 0xfb, 0x10, 0xb6, 0x08, 0x2e, 0x8b, 0x13, 0x9c, 0x37, 0xc9, 0x3f, 0xa1, 0x8f, 0xa1, 0x29, 0xd1, 0xff, 0xd2, 0xe7, 0xfb, 0x60, 0x7f, 0x87, 0xf1, 0xcc, 0x9f, 0x86, 0xd7, 0xd2, 0x24, 0x69, 0x9a, 0x2c, 0x14, 0x4d, 0xa3, 0x7b, 0xec, 0xdb, 0x7d, 0x00, 0x2d, 0x05, 0x27, 0x8c, 0xad, 0x03, 0xde, 0x83, 0xcd, 0x3e, 0xd5, 0x2c, 0x41, 0xd2, 0xae, 0xa6, 0xda, 0xfd, 0x5d, 0x03, 0xe8, 0xe2, 0x49, 0x18, 0x75, 0xfd, 0xec, 0xfc, 0xd5, 0xda, 0x3e, 0x25, 0x82, 0x93, 0x24, 0x9e, 0xcf, 0x72, 0x87, 0xd9, 0x01, 0x7d, 0x40, 0x6c, 0xfa, 0x93, 0x7c, 0x16, 0xbc, 0x25, 0x3a, 0xb0, 0x50, 0xd5, 0x1e, 0x11, 0x2e, 0x1f, 0x03, 0x0c, 0x48, 0x55, 0xa7, 0xe1, 0x4f, 0xbc, 0x8f, 0x88, 0x93, 0xf4, 0x9b, 0x5e, 0x9c, 0xf1, 0xe2, 0x94, 0x1a, 0x34, 0x58, 0x92, 0xc4, 0x69, 0xf7, 0x33, 0xb0, 0xa4, 0xf8, 0x9a, 0x61, 0xb1, 0xad, 0x0e, 0x0b, 0x4b, 0x9d, 0x0c, 0xbf, 0x54, 0xc1, 0x78, 0x1e, 0x93, 0x16, 0x5e, 0x97, 0x13, 0x19, 0xdd, 0x86, 0x12, 0x1d, 0xa9, 0x6b, 0xe0, 0x67, 0xfe, 0xd8, 0x4f, 0xb1, 0xb8, 0xbd, 0xf2, 0x8c, 0x0e, 0xa0, 0x99, 0xe0, 0x8c, 0xc4, 0x45, 0x7a, 0xf4, 0x79, 0x3c, 0x0d, 0xcf, 0x17, 0xcc, 0x7b, 0xcb, 0x5b, 0x26, 0x17, 0x39, 0x32, 0xd4, 0x1c, 0xed, 0x01, 0x04, 0xc4, 0x6e, 0x94, 0xb2, 0xd9, 0x52, 0x25, 0x99, 0xb2, 0x3c, 0x85, 0x42, 0x26, 0x00, 0xcf, 0x61, 0x8d, 0xe5, 0x70, 0x47, 0xe4, 0x90, 0xf9, 0xbf, 0x92, 0xbe, 0x2e, 0x34, 0x2e, 0x42, 0x3c, 0x0d, 0xd2, 0x1e, 0xbb, 0x7e, 0x8e, 0xc9, 0x64, 0xf6, 0x4a, 0x32, 0xdf, 0x28, 0x00, 0x2e, 0x5b, 0x92, 0x41, 0x9f, 0x83, 0xc5, 0xcf, 0x83, 0x28, 0x73, 0xac, 0x52, 0xe1, 0x54, 0x05, 0x84, 0xcb, 0xa5, 0x0b, 0x74, 0x61, 0xfe, 0x8c, 0xdd, 0x6c, 0x07, 0xfe, 0xd6, 0x3c, 0x07, 0x94, 0xcc, 0x73, 0x12, 0x7a, 0x02, 0xc0, 0xcf, 0x5d, 0x32, 0x81, 0x9c, 0x06, 0xd3, 0x70, 0x77, 0x8d, 0x06, 0xca, 0xe6, 0xf2, 0x0a, 0x5e, 0xe9, 0x95, 0xfa, 0x1b, 0xe9, 0x95, 0xdd, 0xaf, 0xa0, 0xb5, 0x92, 0xb0, 0x9b, 0x14, 0x68, 0xaa, 0x82, 0x27, 0xb0, 0x55, 0x4e, 0xd8, 0x4d, 0xd2, 0xfa, 0x5a, 0xf3, 0x4a, 0xc2, 0x5e, 0xcb, 0xff, 0x2f, 0xa0, 0xb9, 0x94, 0xaf, 0x9b, 0xc4, 0x4d, 0xf5, 0xaa, 0xfc, 0xa6, 0x81, 0xd9, 0x8f, 0x82, 0xd7, 0xbd, 0xf7, 0xf4, 0x5e, 0x5d, 0xf9, 0x3f, 0xf2, 0x7d, 0xe1, 0xb1, 0x6f, 0xf4, 0xbe, 0xe8, 0xe3, 0x0a, 0x2b, 0xe9, 0xff, 0xf3, 0x37, 0x84, 0x50, 0xbe, 0xd2, 0xca, 0x6f, 0xfc, 0xd6, 0xff, 0xac, 0x43, 0x2d, 0x1f, 0x9a, 0x07, 0x50, 0x09, 0xc9, 0xab, 0x80, 0x09, 0x16, 0x3b, 0x55, 0x79, 0x2d, 0x91, 0xc5, 0xc3, 0x10, 0x1c, 0x19, 0x66, 0xe2, 0xc5, 0x51, 0x20, 0xe5, 0xc3, 0x80, 0x23, 0xc3, 0x0c, 0x11, 0xc7, 0x2e, 0xf3, 0xa1, 0xcb, 0x02, 0xaf, 0x77, 0xee, 0x08, 0xf8, 0xf2, 0xd0, 0xa6, 0x0b, 0x56, 0x62, 0xd1, 0x27, 0xca, 0xd2, 0xa8, 0x30, 0xb9, 0xfc, 0x92, 0x2f, 0x2d, 0x28, 0xba, 0x78, 0x73, 0x24, 0xfa, 0x08, 0x6a, 0x09, 0x5f, 0x27, 0x2c, 0x41, 0xf5, 0xce, 0xff, 0x84, 0x50, 0x79, 0x25, 0x11, 0x99, 0x1c, 0x87, 0x0e, 0xc1, 0x18, 0xd3, 0xd1, 0xeb, 0xd8, 0xa5, 0xe7, 0x53, 0x31, 0x8e, 0x09, 0x98, 0x23, 0xc8, 0xd3, 0xc1, 0x98, 0xd1, 0xcb, 0xe6, 0xb4, 0x18, 0xb4, 0xa1, 0x5e, 0x40, 0x8a, 0x62, 0x4c, 0xf4, 0x0e, 0xe8, 0x38, 0x0a, 0x1c, 0xc4, 0x30, 0xcd, 0xa5, 0x8a, 0x12, 0x18, 0xe5, 0x76, 0x2d, 0xa8, 0x5d, 0x91, 0x95, 0x46, 0x98, 0xee, 0xaf, 0x3a, 0x98, 0x72, 0xd5, 0x1c, 0x96, 0x6a, 0x70, 0x7b, 0xcd, 0x3b, 0x51, 0x16, 0xe1, 0xb0, 0x54, 0x84, 0xdb, 0xa5, 0x22, 0xa8, 0x50, 0x52, 0x85, 0x47, 0xab, 0x55, 0x70, 0x56, 0xab, 0x20, 0x85, 0x94, 0x32, 0x7c, 0xba, 0x52, 0x86, 0x3b, 0x2b, 0x65, 0x90, 0x72, 0x45, 0x1d, 0x3a, 0xcb, 0x75, 0xd8, 0x59, 0xae, 0x83, 0x14, 0x92, 0x85, 0x78, 0x98, 0x6f, 0xd9, 0x2a, 0x93, 0xd8, 0xce, 0x33, 0xa7, 0xae, 0x62, 0x9a, 0x65, 0x06, 0xfa, 0xcf, 0xcb, 0x76, 0xf4, 0x36, 0x99, 0x01, 0xe2, 0xa9, 0x8f, 0x00, 0xaa, 0x67, 0x23, 0xaf, 0x7f, 0xfc, 0xcc, 0xbe, 0x85, 0x2c, 0x30, 0xba, 0xc7, 0xa3, 0xaf, 0x4f, 0x6c, 0xed, 0xa8, 0x07, 0x96, 0x7c, 0x57, 0x22, 0x13, 0x2a, 0xdd, 0xe1, 0xf0, 0x29, 0x41, 0xd4, 0x40, 0x1f, 0x9c, 0x8e, 0x6c, 0x8d, 0x8a, 0xf5, 0x86, 0x2f, 0xba, 0x4f, 0xfb, 0xf6, 0x86, 0x50, 0x31, 0x38, 0xfd, 0xd6, 0xd6, 0x51, 0x03, 0xcc, 0xde, 0x0b, 0xef, 0x78, 0x34, 0x18, 0x9e, 0xda, 0x95, 0x71, 0x95, 0xfd, 0xbf, 0x7c, 0xfc, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x71, 0xb7, 0xac, 0x48, 0xcc, 0x0c, 0x00, 0x00, } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/agent/udf.proto ================================================ syntax = "proto3"; package agent; //------------------------------------------------------ // RPC Messages for Kapacitor to communicate with // a child process or socket for data processing. // // Messages are streamed by writing a varint header // that contains the length of the following message. // // To decode the stream read a varint, then read // the determined size and decode as a protobuf message. // There is not footer so the next varint if any begins // right after the previous message. // //------------------------------------------------------ // Management messages // // *Request messages are sent to the UDF from Kapacitor. // *Response messages are sent to Kapacitor from the UDF. // // While there is an obvious request/response structure for communicating, // there is a loose coupling between request and response. // Meaning that ordering or synchronizing STDIN and STDOUT in anyway // is not necessary. // For example if Kapacitor requests a snapshot and the // UDF is in the middle of writing a previous response or // data points those can continue. Eventually Kapacitor will receive // the snapshot response and act accordingly. // // A KeepaliveRequest/KeepaliveResponse system is used to ensure that // the process is responsive. Every time that a KeepaliveRequest is sent // a KeepaliveResponse must be returned within a timeout. // If the timeout is reached than the process is considered dead and will be terminated/restarted. // // It is recommend to disable buffering on the input and output sockets. // Some languages like python will automatically buffer the STDIN and STDOUT sockets. // To disable this behavior use the -u flag on the python interpreter. // Request that the process return information about available Options. message InfoRequest { } enum EdgeType { STREAM = 0; BATCH = 1; } message InfoResponse { EdgeType wants = 1; EdgeType provides = 2; map options = 3; } enum ValueType { BOOL = 0; INT = 1; DOUBLE = 2; STRING = 3; DURATION = 4; } message OptionInfo { repeated ValueType valueTypes = 1; } // Request that the process initialize itself with the provided options. message InitRequest { repeated Option options = 1; string taskID = 2; string nodeID = 3; } message Option { string name = 1; repeated OptionValue values = 2; } message OptionValue { ValueType type = 1; oneof value { bool boolValue = 2; int64 intValue = 3; double doubleValue = 4; string stringValue = 5; int64 durationValue = 6; } } // Respond to Kapacitor whether initialization was successful. message InitResponse { bool success = 1; string error = 2; } // Request that the process provide a snapshot of its state. message SnapshotRequest { } // Respond to Kapacitor with a serialized snapshot of the running state. message SnapshotResponse { bytes snapshot = 1; } // Request that the process restore its state from a snapshot. message RestoreRequest { bytes snapshot = 1; } // Respond with success or failure to a RestoreRequest message RestoreResponse { bool success = 1; string error = 2; } // Request that the process respond with a Keepalive to verify it is responding. message KeepaliveRequest { // The number of nanoseconds since the epoch. // Used only for debugging keepalive requests. int64 time = 1; } // Respond to KeepaliveRequest message KeepaliveResponse { // The number of nanoseconds since the epoch. // Used only for debugging keepalive requests. int64 time = 1; } // Sent from the process to Kapacitor indicating an error has occurred. // If an ErrorResponse is received, Kapacitor will terminate the process. message ErrorResponse { string error = 1; } //------------------------------------------------------ // Data flow messages // // Sent and received by both the process and Kapacitor // Indicates the beginning of a batch. // All subsequent points should be considered // part of the batch until EndBatch arrives. // This includes grouping. Batches of // differing groups may not be interleaved. // // All the meta data but tmax is provided, // since tmax may not be known at // the beginning of a batch. // // Size is the number of points in the batch. // If size is 0 then the batch has an undetermined size. message BeginBatch { string name = 1; string group = 2; map tags = 3; int64 size = 4; bool byName = 5; } // Message containing information about a single data point. // Can be sent on it's own or bookended by BeginBatch and EndBatch messages. message Point { int64 time = 1; string name = 2; string database = 3; string retentionPolicy = 4; string group = 5; repeated string dimensions = 6; map tags = 7; map fieldsDouble = 8; map fieldsInt = 9; map fieldsString = 10; map fieldsBool = 12; bool byName = 11; } // Indicates the end of a batch and contains // all meta data associated with the batch. // The same meta information is provided for // ease of use with the addition of tmax since it // may not be know at BeginBatch. message EndBatch { string name = 1; string group = 2; int64 tmax = 3; map tags = 4; bool byName = 5; } //----------------------------------------------------------- // Wrapper messages // // All messages sent over STDIN will be Request messages. // All messages sent over STDOUT must be Response messages. // Request message wrapper -- sent from Kapacitor to process message Request { oneof message { // Management requests InfoRequest info = 1; InitRequest init = 2; KeepaliveRequest keepalive = 3; SnapshotRequest snapshot = 4; RestoreRequest restore = 5; // Data flow responses BeginBatch begin = 16; Point point = 17; EndBatch end = 18; } } // Response message wrapper -- sent from process to Kapacitor message Response { oneof message { // Management responses InfoResponse info = 1; InitResponse init = 2; KeepaliveResponse keepalive = 3; SnapshotResponse snapshot = 4; RestoreResponse restore = 5; ErrorResponse error = 6; // Data flow responses BeginBatch begin = 16; Point point = 17; EndBatch end = 18; } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/server.go ================================================ package udf import ( "errors" "fmt" "io" "log" "sync" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/udf/agent" ) var ErrServerStopped = errors.New("server already stopped") // Server provides an implementation for the core communication with UDFs. // The Server provides only a partial implementation of udf.Interface as // it is expected that setup and teardown will be necessary to create a Server. // As such the Open and Close methods are not implemented. // // Once a Server is created and started the owner can send points or batches // to the UDF by writing them to the PointIn or BatchIn channels respectively, // and according to the type of UDF created. // // The Server may be Aborted at anytime for various reasons. It is the owner's responsibility // via the abortCallback to stop writing to the *In channels since no more selects on the channels // will be performed. // // Calling Stop on the Server should only be done once the owner has stopped writing to the *In channel, // at which point the remaining data will be processed and the UDF will be allowed to clean up. // // Callling Info returns information about available options the UDF has. // // Calling Init is required to process data. // The behavior is undefined if you send points/batches to the Server without calling Init. type Server struct { // If the processes is Aborted (via Keepalive timeout, etc.) // then no more data will be read off the *In channels. // // Optional callback if the process aborts. // It is the owners response abortCallback func() abortOnce sync.Once // If abort fails after sometime this will be called killCallback func() inMsg chan edge.Message outMsg chan edge.Message stopped bool stopping chan struct{} aborted bool aborting chan struct{} // The first error that occurred or nil err error errMu sync.Mutex requests chan *agent.Request requestsGroup sync.WaitGroup keepalive chan int64 keepaliveTimeout time.Duration taskID string nodeID string in agent.ByteReadReader out io.WriteCloser // Group for waiting on read/write goroutines ioGroup sync.WaitGroup mu sync.Mutex logger *log.Logger responseBuf []byte infoResponse chan *agent.Response initResponse chan *agent.Response snapshotResponse chan *agent.Response restoreResponse chan *agent.Response // Buffer up batch messages begin *agent.BeginBatch points []edge.BatchPointMessage } func NewServer( taskID, nodeID string, in agent.ByteReadReader, out io.WriteCloser, l *log.Logger, timeout time.Duration, abortCallback func(), killCallback func(), ) *Server { s := &Server{ taskID: taskID, nodeID: nodeID, in: in, out: out, logger: l, requests: make(chan *agent.Request), keepalive: make(chan int64, 1), keepaliveTimeout: timeout, abortCallback: abortCallback, killCallback: killCallback, inMsg: make(chan edge.Message), outMsg: make(chan edge.Message), infoResponse: make(chan *agent.Response, 1), initResponse: make(chan *agent.Response, 1), snapshotResponse: make(chan *agent.Response, 1), restoreResponse: make(chan *agent.Response, 1), } return s } func (s *Server) In() chan<- edge.Message { return s.inMsg } func (s *Server) Out() <-chan edge.Message { return s.outMsg } func (s *Server) setError(err error) { s.errMu.Lock() defer s.errMu.Unlock() if s.err == nil { s.err = err } } // Start the Server func (s *Server) Start() error { s.mu.Lock() defer s.mu.Unlock() s.stopped = false s.stopping = make(chan struct{}) s.aborted = false s.aborting = make(chan struct{}) s.ioGroup.Add(1) go func() { err := s.writeData() if err != nil { s.setError(err) defer s.abort() } s.ioGroup.Done() }() s.ioGroup.Add(1) go func() { err := s.readData() if err != nil { s.setError(err) defer s.abort() } s.ioGroup.Done() }() s.requestsGroup.Add(2) go s.runKeepalive() go s.watchKeepalive() return nil } // Abort the server. // Data in-flight will not be processed. // Give a reason for aborting via the err parameter. func (s *Server) Abort(err error) { s.setError(err) s.abort() } func (s *Server) abort() { s.mu.Lock() defer s.mu.Unlock() if s.aborted { return } s.aborted = true close(s.aborting) if s.abortCallback != nil { s.abortOnce.Do(s.abortCallback) } _ = s.stop() } // Wait for all IO to stop on the in/out objects. func (s *Server) WaitIO() { s.ioGroup.Wait() } // Stop the Server cleanly. // // Calling Stop should only be done once the owner has stopped writing to the *In channel, // at which point the remaining data will be processed and the subprocess will be allowed to exit cleanly. func (s *Server) Stop() error { s.mu.Lock() defer s.mu.Unlock() return s.stop() } // internal stop function you must acquire the lock before calling func (s *Server) stop() error { if s.stopped { return s.err } s.stopped = true close(s.stopping) s.requestsGroup.Wait() close(s.requests) close(s.inMsg) s.ioGroup.Wait() // Return the error that occurred first s.errMu.Lock() defer s.errMu.Unlock() return s.err } type Info struct { Wants agent.EdgeType Provides agent.EdgeType Options map[string]*agent.OptionInfo } // Get information about the process, available options etc. // Info need not be called every time a process is started. func (s *Server) Info() (Info, error) { info := Info{} req := &agent.Request{Message: &agent.Request_Info{ Info: &agent.InfoRequest{}, }} resp, err := s.doRequestResponse(req, s.infoResponse) if err != nil { return info, err } ri := resp.Message.(*agent.Response_Info).Info info.Options = ri.Options info.Wants = ri.Wants info.Provides = ri.Provides return info, nil } // Initialize the process with a set of Options. // Calling Init is required even if you do not have any specific Options, just pass nil func (s *Server) Init(options []*agent.Option) error { req := &agent.Request{Message: &agent.Request_Init{ Init: &agent.InitRequest{ Options: options, TaskID: s.taskID, NodeID: s.nodeID, }, }} resp, err := s.doRequestResponse(req, s.initResponse) if err != nil { return err } init := resp.Message.(*agent.Response_Init).Init if !init.Success { return fmt.Errorf("failed to initialize processes %s", init.Error) } return nil } // Request a snapshot from the process. func (s *Server) Snapshot() ([]byte, error) { req := &agent.Request{Message: &agent.Request_Snapshot{ Snapshot: &agent.SnapshotRequest{}, }} resp, err := s.doRequestResponse(req, s.snapshotResponse) if err != nil { return nil, err } snapshot := resp.Message.(*agent.Response_Snapshot).Snapshot.Snapshot return snapshot, nil } // Request to restore a snapshot. func (s *Server) Restore(snapshot []byte) error { req := &agent.Request{Message: &agent.Request_Restore{ Restore: &agent.RestoreRequest{snapshot}, }} resp, err := s.doRequestResponse(req, s.restoreResponse) if err != nil { return err } restore := resp.Message.(*agent.Response_Restore).Restore if !restore.Success { return fmt.Errorf("error restoring snapshot: %s", restore.Error) } return nil } func (s *Server) doRequestResponse(req *agent.Request, respC chan *agent.Response) (*agent.Response, error) { err := func() error { s.mu.Lock() defer s.mu.Unlock() if s.stopped { if s.err != nil { return s.err } return ErrServerStopped } s.requestsGroup.Add(1) return nil }() if err != nil { return nil, err } defer s.requestsGroup.Done() select { case <-s.aborting: return nil, s.err case s.requests <- req: } select { case <-s.aborting: return nil, s.err case res := <-respC: return res, nil } } func (s *Server) doResponse(response *agent.Response, respC chan *agent.Response) { select { case respC <- response: default: s.logger.Printf("E! received %T without requesting it", response.Message) } } // send KeepaliveRequest on the specified interval func (s *Server) runKeepalive() { defer s.requestsGroup.Done() if s.keepaliveTimeout <= 0 { return } ticker := time.NewTicker(s.keepaliveTimeout / 2) defer ticker.Stop() for { select { case <-ticker.C: req := &agent.Request{Message: &agent.Request_Keepalive{ Keepalive: &agent.KeepaliveRequest{ Time: time.Now().UnixNano(), }, }} select { case s.requests <- req: case <-s.aborting: } case <-s.stopping: return } } } // Abort the process if a keepalive timeout is reached. func (s *Server) watchKeepalive() { // Defer functions are called LIFO. // We need to call p.abort after p.requestsGroup.Done so we just set a flag. var err error defer func() { if err != nil { s.setError(err) aborted := make(chan struct{}) go func() { timeout := s.keepaliveTimeout * 2 if timeout <= 0 { timeout = time.Second } time.Sleep(timeout) select { case <-aborted: // We cleanly aborted process is stopped default: // We failed to abort just kill it. if s.killCallback != nil { s.logger.Println("E! process not responding! killing") s.killCallback() } } }() s.abort() close(aborted) } }() defer s.requestsGroup.Done() // If timeout is <= 0 then we don't ever timeout from keepalive, // but we need to receive from p.keepalive or handleResponse will block. // So we set a long timeout and then ignore it if its reached. timeout := s.keepaliveTimeout if timeout <= 0 { timeout = time.Hour } last := time.Now().UnixNano() for { select { case last = <-s.keepalive: case <-time.After(timeout): // Ignore invalid timeout if s.keepaliveTimeout <= 0 { break } err = fmt.Errorf("keepalive timedout, last keepalive received was: %s", time.Unix(0, last)) s.logger.Println("E!", err) return case <-s.stopping: return } } } // Write Requests func (s *Server) writeData() error { defer s.out.Close() var begin edge.BeginBatchMessage for { select { case m, ok := <-s.inMsg: if !ok { s.inMsg = nil } switch msg := m.(type) { case edge.PointMessage: err := s.writePoint(msg) if err != nil { return err } case edge.BeginBatchMessage: begin = msg err := s.writeBeginBatch(msg) if err != nil { return err } case edge.BatchPointMessage: err := s.writeBatchPoint(begin.GroupID(), msg) if err != nil { return err } case edge.EndBatchMessage: err := s.writeEndBatch(begin.Name(), begin.Time(), begin.GroupInfo(), msg) if err != nil { return err } case edge.BufferedBatchMessage: err := s.writeBufferedBatch(msg) if err != nil { return err } } case req, ok := <-s.requests: if ok { err := s.writeRequest(req) if err != nil { return err } } else { s.requests = nil } case <-s.aborting: return s.err } if s.inMsg == nil && s.requests == nil { break } } return nil } func (s *Server) writePoint(p edge.PointMessage) error { strs, floats, ints, bools := s.fieldsToTypedMaps(p.Fields()) udfPoint := &agent.Point{ Time: p.Time().UnixNano(), Name: p.Name(), Database: p.Database(), RetentionPolicy: p.RetentionPolicy(), Group: string(p.GroupID()), Dimensions: p.Dimensions().TagNames, ByName: p.Dimensions().ByName, Tags: p.Tags(), FieldsDouble: floats, FieldsInt: ints, FieldsString: strs, FieldsBool: bools, } req := &agent.Request{ Message: &agent.Request_Point{Point: udfPoint}, } return s.writeRequest(req) } func (s *Server) fieldsToTypedMaps(fields models.Fields) ( strs map[string]string, floats map[string]float64, ints map[string]int64, bools map[string]bool, ) { for k, v := range fields { switch value := v.(type) { case string: if strs == nil { strs = make(map[string]string) } strs[k] = value case float64: if floats == nil { floats = make(map[string]float64) } floats[k] = value case int64: if ints == nil { ints = make(map[string]int64) } ints[k] = value case bool: if bools == nil { bools = make(map[string]bool) } bools[k] = value default: panic("unsupported field value type") } } return } func (s *Server) typeMapsToFields( strs map[string]string, floats map[string]float64, ints map[string]int64, bools map[string]bool, ) models.Fields { fields := make(models.Fields) for k, v := range strs { fields[k] = v } for k, v := range ints { fields[k] = v } for k, v := range floats { fields[k] = v } for k, v := range bools { fields[k] = v } return fields } func (s *Server) writeBeginBatch(begin edge.BeginBatchMessage) error { req := &agent.Request{ Message: &agent.Request_Begin{ Begin: &agent.BeginBatch{ Name: begin.Name(), Group: string(begin.GroupID()), Tags: begin.Tags(), Size: int64(begin.SizeHint()), ByName: begin.Dimensions().ByName, }}, } return s.writeRequest(req) } func (s *Server) writeBatchPoint(group models.GroupID, bp edge.BatchPointMessage) error { strs, floats, ints, bools := s.fieldsToTypedMaps(bp.Fields()) req := &agent.Request{ Message: &agent.Request_Point{ Point: &agent.Point{ Time: bp.Time().UnixNano(), Group: string(group), Tags: bp.Tags(), FieldsDouble: floats, FieldsInt: ints, FieldsString: strs, FieldsBool: bools, }, }, } return s.writeRequest(req) } func (s *Server) writeEndBatch(name string, tmax time.Time, groupInfo edge.GroupInfo, end edge.EndBatchMessage) error { req := &agent.Request{ Message: &agent.Request_End{ End: &agent.EndBatch{ Name: name, Group: string(groupInfo.ID), Tmax: tmax.UnixNano(), Tags: groupInfo.Tags, }, }, } return s.writeRequest(req) } func (s *Server) writeBufferedBatch(batch edge.BufferedBatchMessage) error { if err := s.writeBeginBatch(batch.Begin()); err != nil { return err } for _, bp := range batch.Points() { if err := s.writeBatchPoint(batch.GroupID(), bp); err != nil { return err } } return s.writeEndBatch(batch.Name(), batch.Time(), batch.GroupInfo(), batch.End()) } func (s *Server) writeRequest(req *agent.Request) error { err := agent.WriteMessage(req, s.out) if err != nil { err = fmt.Errorf("write error: %s", err) } return err } // Read Responses from STDOUT of the process. func (s *Server) readData() error { defer func() { close(s.outMsg) }() for { response, err := s.readResponse() if err == io.EOF { return nil } if err != nil { err = fmt.Errorf("read error: %s", err) return err } err = s.handleResponse(response) if err != nil { return err } } } func (s *Server) readResponse() (*agent.Response, error) { response := new(agent.Response) err := agent.ReadMessage(&s.responseBuf, s.in, response) if err != nil { return nil, err } return response, nil } func (s *Server) handleResponse(response *agent.Response) error { // Always reset the keepalive timer since we received a response select { case s.keepalive <- time.Now().UnixNano(): case <-s.stopping: // No one is watching the keepalive anymore so we don't need to feed it, // but we still want to handle the response case <-s.aborting: return s.err } // handle response switch msg := response.Message.(type) { case *agent.Response_Keepalive: // Noop we already reset the keepalive timer case *agent.Response_Info: s.doResponse(response, s.infoResponse) case *agent.Response_Init: s.doResponse(response, s.initResponse) case *agent.Response_Snapshot: s.doResponse(response, s.snapshotResponse) case *agent.Response_Restore: s.doResponse(response, s.restoreResponse) case *agent.Response_Error: s.logger.Println("E!", msg.Error.Error) return errors.New(msg.Error.Error) case *agent.Response_Begin: s.begin = msg.Begin s.points = make([]edge.BatchPointMessage, 0, msg.Begin.Size) case *agent.Response_Point: if s.points != nil { bp := edge.NewBatchPointMessage( s.typeMapsToFields( msg.Point.FieldsString, msg.Point.FieldsDouble, msg.Point.FieldsInt, msg.Point.FieldsBool, ), msg.Point.Tags, time.Unix(0, msg.Point.Time).UTC(), ) s.points = append(s.points, bp) } else { p := edge.NewPointMessage( msg.Point.Name, msg.Point.Database, msg.Point.RetentionPolicy, models.Dimensions{ByName: msg.Point.ByName, TagNames: msg.Point.Dimensions}, s.typeMapsToFields( msg.Point.FieldsString, msg.Point.FieldsDouble, msg.Point.FieldsInt, msg.Point.FieldsBool, ), msg.Point.Tags, time.Unix(0, msg.Point.Time).UTC(), ) select { case s.outMsg <- p: case <-s.aborting: return s.err } } case *agent.Response_End: begin := edge.NewBeginBatchMessage( msg.End.Name, msg.End.Tags, s.begin.ByName, time.Unix(0, msg.End.Tmax).UTC(), len(s.points), ) bufferedBatch := edge.NewBufferedBatchMessage( begin, s.points, edge.NewEndBatchMessage(), ) select { case s.outMsg <- bufferedBatch: case <-s.aborting: return s.err } s.begin = nil s.points = nil default: panic(fmt.Sprintf("unexpected response message %T", msg)) } return nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/server_test.go ================================================ package udf_test import ( "errors" "log" "os" "reflect" "testing" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/udf" "github.com/influxdata/kapacitor/udf/agent" udf_test "github.com/influxdata/kapacitor/udf/test" ) func TestUDF_StartStop(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartStop] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) s.Start() close(u.Responses) s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_StartInitStop(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartStop] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) go func() { req := <-u.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Errorf("expected init message got %T", req.Message) } res := &agent.Response{ Message: &agent.Response_Init{ Init: &agent.InitResponse{ Success: true, }, }, } u.Responses <- res close(u.Responses) }() s.Start() err := s.Init(nil) if err != nil { t.Fatal(err) } s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_StartInitAbort(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartInfoAbort] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) s.Start() expErr := errors.New("explicit abort") go func() { req := <-u.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Error("expected init message") } s.Abort(expErr) close(u.Responses) }() err := s.Init(nil) if err != expErr { t.Fatal("expected explicit abort error") } } func TestUDF_StartInfoStop(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartInfoStop] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) go func() { req := <-u.Requests _, ok := req.Message.(*agent.Request_Info) if !ok { t.Errorf("expected info message got %T", req.Message) } res := &agent.Response{ Message: &agent.Response_Info{ Info: &agent.InfoResponse{ Wants: agent.EdgeType_STREAM, Provides: agent.EdgeType_BATCH, }, }, } u.Responses <- res close(u.Responses) }() s.Start() info, err := s.Info() if err != nil { t.Fatal(err) } if exp, got := agent.EdgeType_STREAM, info.Wants; got != exp { t.Errorf("unexpected info.Wants got %v exp %v", got, exp) } if exp, got := agent.EdgeType_BATCH, info.Provides; got != exp { t.Errorf("unexpected info.Provides got %v exp %v", got, exp) } s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_StartInfoAbort(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartInfoAbort] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) s.Start() expErr := errors.New("explicit abort") go func() { req := <-u.Requests _, ok := req.Message.(*agent.Request_Info) if !ok { t.Error("expected info message") } s.Abort(expErr) close(u.Responses) }() _, err := s.Info() if err != expErr { t.Fatal("expected ErrUDFProcessAborted") } } func TestUDF_Keepalive(t *testing.T) { t.Parallel() u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_Keepalive] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, time.Millisecond*100, nil, nil) s.Start() s.Init(nil) req := <-u.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Error("expected init message") } select { case req = <-u.Requests: case <-time.After(time.Second): t.Fatal("expected keepalive message") } if req == nil { t.Fatal("expected keepalive message got nil, u was killed.") } _, ok = req.Message.(*agent.Request_Keepalive) if !ok { t.Errorf("expected keepalive message got %T", req.Message) } close(u.Responses) s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_MissedKeepalive(t *testing.T) { t.Parallel() abortCalled := make(chan struct{}) aborted := func() { close(abortCalled) } u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_MissedKeepalive] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, time.Millisecond*100, aborted, nil) s.Start() // Since the keepalive is missed, the process should abort on its own. for range u.Requests { } select { case <-abortCalled: case <-time.After(time.Second): t.Error("expected abort callback to be called") } close(u.Responses) if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_KillCallBack(t *testing.T) { t.Parallel() timeout := time.Millisecond * 100 abortCalled := make(chan struct{}) killCalled := make(chan struct{}) aborted := func() { time.Sleep(timeout * 3) close(abortCalled) } kill := func() { close(killCalled) } u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_MissedKeepalive] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, timeout, aborted, kill) s.Start() // Since the keepalive is missed, the process should abort on its own. for range u.Requests { } // Since abort takes a long time killCallback should be called select { case <-killCalled: case <-time.After(time.Second): t.Error("expected kill callback to be called") } close(u.Responses) if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_MissedKeepaliveInit(t *testing.T) { t.Parallel() abortCalled := make(chan struct{}) aborted := func() { close(abortCalled) } u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_MissedKeepaliveInit] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, time.Millisecond*100, aborted, nil) s.Start() s.Init(nil) // Since the keepalive is missed, the process should abort on its own. for range u.Requests { } select { case <-abortCalled: case <-time.After(time.Second): t.Error("expected abort callback to be called") } close(u.Responses) if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_MissedKeepaliveInfo(t *testing.T) { t.Parallel() abortCalled := make(chan struct{}) aborted := func() { close(abortCalled) } u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_MissedKeepaliveInfo] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, time.Millisecond*100, aborted, nil) s.Start() s.Info() // Since the keepalive is missed, the process should abort on its own. for range u.Requests { } select { case <-abortCalled: case <-time.After(time.Second): t.Error("expected abort callback to be called") } close(u.Responses) if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_SnapshotRestore(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_SnapshotRestore] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) go func() { // Init req := <-u.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Error("expected init message") } u.Responses <- &agent.Response{ Message: &agent.Response_Init{ Init: &agent.InitResponse{Success: true}, }, } // Snapshot req = <-u.Requests if req == nil { t.Fatal("expected snapshot message got nil") } _, ok = req.Message.(*agent.Request_Snapshot) if !ok { t.Errorf("expected snapshot message got %T", req.Message) } data := []byte{42} u.Responses <- &agent.Response{ Message: &agent.Response_Snapshot{ Snapshot: &agent.SnapshotResponse{Snapshot: data}, }, } // Restore req = <-u.Requests if req == nil { t.Fatal("expected restore message got nil") } restore, ok := req.Message.(*agent.Request_Restore) if !ok { t.Errorf("expected restore message got %T", req.Message) } if !reflect.DeepEqual(data, restore.Restore.Snapshot) { t.Errorf("unexpected restore snapshot got %v exp %v", restore.Restore.Snapshot, data) } u.Responses <- &agent.Response{ Message: &agent.Response_Restore{ Restore: &agent.RestoreResponse{Success: true}, }, } close(u.Responses) }() s.Start() s.Init(nil) snapshot, err := s.Snapshot() if err != nil { t.Fatal(err) } err = s.Restore(snapshot) if err != nil { t.Fatal(err) } s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_StartInitPointStop(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartPointStop] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) go func() { req := <-u.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Errorf("expected init message got %T", req.Message) } res := &agent.Response{ Message: &agent.Response_Init{ Init: &agent.InitResponse{ Success: true, }, }, } u.Responses <- res req = <-u.Requests pt, ok := req.Message.(*agent.Request_Point) if !ok { t.Errorf("expected point message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_Point{ Point: pt.Point, }, } u.Responses <- res close(u.Responses) }() s.Start() err := s.Init(nil) if err != nil { t.Fatal(err) } // Write point to server p := edge.NewPointMessage( "test", "db", "rp", models.Dimensions{}, models.Fields{"f1": 1.0, "f2": 2.0}, models.Tags{"t1": "v1", "t2": "v2"}, time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), ) s.In() <- p rp := <-s.Out() if !reflect.DeepEqual(rp, p) { t.Errorf("unexpected returned point got: %v exp %v", rp, p) } s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } func TestUDF_StartInitBatchStop(t *testing.T) { u := udf_test.NewIO() l := log.New(os.Stderr, "[TestUDF_StartPointStop] ", log.LstdFlags) s := udf.NewServer("testTask", "testNode", u.Out(), u.In(), l, 0, nil, nil) go func() { req := <-u.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Errorf("expected init message got %T", req.Message) } res := &agent.Response{ Message: &agent.Response_Init{ Init: &agent.InitResponse{ Success: true, }, }, } u.Responses <- res // Begin batch req = <-u.Requests bb, ok := req.Message.(*agent.Request_Begin) if !ok { t.Errorf("expected begin message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_Begin{ Begin: bb.Begin, }, } u.Responses <- res // Point req = <-u.Requests pt, ok := req.Message.(*agent.Request_Point) if !ok { t.Errorf("expected point message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_Point{ Point: pt.Point, }, } u.Responses <- res // End batch req = <-u.Requests eb, ok := req.Message.(*agent.Request_End) if !ok { t.Errorf("expected end message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_End{ End: eb.End, }, } u.Responses <- res close(u.Responses) }() s.Start() err := s.Init(nil) if err != nil { t.Fatal(err) } // Write point to server b := edge.NewBufferedBatchMessage( edge.NewBeginBatchMessage( "test", models.Tags{"t1": "v1"}, false, time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), 1, ), []edge.BatchPointMessage{ edge.NewBatchPointMessage( models.Fields{"f1": 1.0, "f2": 2.0, "f3": int64(1), "f4": "str"}, models.Tags{"t1": "v1", "t2": "v2"}, time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), ), }, edge.NewEndBatchMessage(), ) s.In() <- b rb := <-s.Out() if !reflect.DeepEqual(b, rb) { t.Errorf("unexpected returned batch got: %v exp %v", rb, b) } s.Stop() // read all requests and wait till the chan is closed for range u.Requests { } if err := <-u.ErrC; err != nil { t.Error(err) } } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf/udf.go ================================================ package udf import ( "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/udf/agent" ) // Interface for communicating with a UDF type Interface interface { Open() error Info() (Info, error) Init(options []*agent.Option) error Abort(err error) Close() error Snapshot() ([]byte, error) Restore(snapshot []byte) error In() chan<- edge.Message Out() <-chan edge.Message } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf.go ================================================ package kapacitor import ( "bufio" "fmt" "io" "log" "net" "sync" "time" "github.com/cenkalti/backoff" "github.com/influxdata/kapacitor/command" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/udf" "github.com/influxdata/kapacitor/udf/agent" "github.com/pkg/errors" ) // User defined function type UDFNode struct { node u *pipeline.UDFNode udf udf.Interface aborted chan struct{} wg sync.WaitGroup mu sync.Mutex stopped bool } // Create a new UDFNode that sends incoming data to child udf func newUDFNode(et *ExecutingTask, n *pipeline.UDFNode, l *log.Logger) (*UDFNode, error) { un := &UDFNode{ node: node{Node: n, et: et, logger: l}, u: n, aborted: make(chan struct{}), } // Create the UDF f, err := et.tm.UDFService.Create( n.UDFName, et.Task.ID, n.Name(), l, un.abortedCallback, ) if err != nil { return nil, err } un.udf = f un.node.runF = un.runUDF un.node.stopF = un.stopUDF return un, nil } var errNodeAborted = errors.New("node aborted") func (n *UDFNode) stopUDF() { n.mu.Lock() defer n.mu.Unlock() if !n.stopped { n.stopped = true if n.udf != nil { n.udf.Abort(errNodeAborted) } } } func (n *UDFNode) runUDF(snapshot []byte) (err error) { defer func() { n.mu.Lock() defer n.mu.Unlock() //Ignore stopped errors if the udf was stopped externally if n.stopped && (err == udf.ErrServerStopped || err == errNodeAborted) { err = nil } n.stopped = true }() if err := n.udf.Open(); err != nil { return err } if err := n.udf.Init(n.u.Options); err != nil { return err } if snapshot != nil { if err := n.udf.Restore(snapshot); err != nil { return err } } forwardErr := make(chan error, 1) go func() { out := n.udf.Out() for m := range out { if err := edge.Forward(n.outs, m); err != nil { forwardErr <- err return } } forwardErr <- nil }() // The abort callback needs to know when we are done writing // so we wrap in a wait group. n.wg.Add(1) go func() { defer n.wg.Done() in := n.udf.In() for m, ok := n.ins[0].Emit(); ok; m, ok = n.ins[0].Emit() { n.timer.Start() select { case in <- m: case <-n.aborted: return } n.timer.Stop() } }() // wait till we are done writing n.wg.Wait() // Close the udf if err := n.udf.Close(); err != nil { return err } // Wait/Return any error from the forwarding goroutine return <-forwardErr } func (n *UDFNode) abortedCallback() { close(n.aborted) // wait till we are done writing n.wg.Wait() } func (n *UDFNode) snapshot() ([]byte, error) { return n.udf.Snapshot() } // UDFProcess wraps an external process and sends and receives data // over STDIN and STDOUT. Lines received over STDERR are logged // via normal Kapacitor logging. type UDFProcess struct { taskName string nodeName string server *udf.Server commander command.Commander cmdSpec command.Spec cmd command.Command stderr io.Reader // Group for waiting on the process itself processGroup sync.WaitGroup logStdErrGroup sync.WaitGroup mu sync.Mutex logger *log.Logger timeout time.Duration abortCallback func() } func NewUDFProcess( taskName, nodeName string, commander command.Commander, cmdSpec command.Spec, l *log.Logger, timeout time.Duration, abortCallback func(), ) *UDFProcess { return &UDFProcess{ taskName: taskName, nodeName: nodeName, commander: commander, cmdSpec: cmdSpec, logger: l, timeout: timeout, abortCallback: abortCallback, } } // Open the UDFProcess func (p *UDFProcess) Open() error { p.mu.Lock() defer p.mu.Unlock() cmd := p.commander.NewCommand(p.cmdSpec) stdin, err := cmd.StdinPipe() if err != nil { return err } stdout, err := cmd.StdoutPipe() if err != nil { return err } stderr, err := cmd.StderrPipe() if err != nil { return err } p.stderr = stderr err = cmd.Start() if err != nil { return err } p.cmd = cmd outBuf := bufio.NewReader(stdout) p.server = udf.NewServer( p.taskName, p.nodeName, outBuf, stdin, p.logger, p.timeout, p.abortCallback, cmd.Kill, ) if err := p.server.Start(); err != nil { return err } p.logStdErrGroup.Add(1) go p.logStdErr() // Wait for process to terminate p.processGroup.Add(1) go func() { // First wait for the pipe read writes to finish p.logStdErrGroup.Wait() p.server.WaitIO() err := cmd.Wait() if err != nil { err = fmt.Errorf("process exited unexpectedly: %v", err) defer p.server.Abort(err) } p.processGroup.Done() }() return nil } // Stop the UDFProcess cleanly. // // Calling Close should only be done once the owner has stopped writing to the *In channel, // at which point the remaining data will be processed and the subprocess will be allowed to exit cleanly. func (p *UDFProcess) Close() error { p.mu.Lock() defer p.mu.Unlock() err := p.server.Stop() p.processGroup.Wait() return err } // Replay any lines from STDERR of the process to the Kapacitor log. func (p *UDFProcess) logStdErr() { defer p.logStdErrGroup.Done() scanner := bufio.NewScanner(p.stderr) for scanner.Scan() { p.logger.Println("I!P", scanner.Text()) } } func (p *UDFProcess) Abort(err error) { p.server.Abort(err) } func (p *UDFProcess) Init(options []*agent.Option) error { return p.server.Init(options) } func (p *UDFProcess) Snapshot() ([]byte, error) { return p.server.Snapshot() } func (p *UDFProcess) Restore(snapshot []byte) error { return p.server.Restore(snapshot) } func (p *UDFProcess) In() chan<- edge.Message { return p.server.In() } func (p *UDFProcess) Out() <-chan edge.Message { return p.server.Out() } func (p *UDFProcess) Info() (udf.Info, error) { return p.server.Info() } type UDFSocket struct { taskName string nodeName string server *udf.Server socket Socket logger *log.Logger timeout time.Duration abortCallback func() } type Socket interface { Open() error Close() error In() io.WriteCloser Out() io.Reader } func NewUDFSocket( taskName, nodeName string, socket Socket, l *log.Logger, timeout time.Duration, abortCallback func(), ) *UDFSocket { return &UDFSocket{ taskName: taskName, nodeName: nodeName, socket: socket, logger: l, timeout: timeout, abortCallback: abortCallback, } } func (s *UDFSocket) Open() error { err := s.socket.Open() if err != nil { return err } in := s.socket.In() out := s.socket.Out() outBuf := bufio.NewReader(out) s.server = udf.NewServer( s.taskName, s.nodeName, outBuf, in, s.logger, s.timeout, s.abortCallback, func() { s.socket.Close() }, ) return s.server.Start() } func (s *UDFSocket) Close() error { if err := s.server.Stop(); err != nil { // Always close the socket s.socket.Close() return errors.Wrap(err, "stopping UDF server") } if err := s.socket.Close(); err != nil { return errors.Wrap(err, "closing UDF socket connection") } return nil } func (s *UDFSocket) Abort(err error) { s.server.Abort(err) } func (s *UDFSocket) Init(options []*agent.Option) error { return s.server.Init(options) } func (s *UDFSocket) Snapshot() ([]byte, error) { return s.server.Snapshot() } func (s *UDFSocket) Restore(snapshot []byte) error { return s.server.Restore(snapshot) } func (s *UDFSocket) In() chan<- edge.Message { return s.server.In() } func (s *UDFSocket) Out() <-chan edge.Message { return s.server.Out() } func (s *UDFSocket) Info() (udf.Info, error) { return s.server.Info() } type socket struct { path string conn *net.UnixConn } func NewSocketConn(path string) Socket { return &socket{ path: path, } } func (s *socket) Open() error { addr, err := net.ResolveUnixAddr("unix", s.path) if err != nil { return err } // Connect to socket b := backoff.NewExponentialBackOff() b.MaxElapsedTime = time.Minute * 5 err = backoff.Retry(func() error { conn, err := net.DialUnix("unix", nil, addr) if err != nil { return err } s.conn = conn return nil }, b, ) return err } func (s *socket) Close() error { return s.conn.Close() } type unixCloser struct { *net.UnixConn } func (u unixCloser) Close() error { // Only close the write end of the socket connection. // The socket connection as a whole will be closed later. return u.CloseWrite() } func (s *socket) In() io.WriteCloser { return unixCloser{s.conn} } func (s *socket) Out() io.Reader { return s.conn } ================================================ FILE: vendor/github.com/influxdata/kapacitor/udf_test.go ================================================ package kapacitor_test import ( "bytes" "fmt" "io" "log" "os" "reflect" "testing" "time" "github.com/influxdata/kapacitor" "github.com/influxdata/kapacitor/command" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/udf" "github.com/influxdata/kapacitor/udf/agent" udf_test "github.com/influxdata/kapacitor/udf/test" ) func newUDFSocket(name string) (*kapacitor.UDFSocket, *udf_test.IO) { uio := udf_test.NewIO() l := log.New(os.Stderr, fmt.Sprintf("[%s] ", name), log.LstdFlags) u := kapacitor.NewUDFSocket(name, "testNode", newTestSocket(uio), l, 0, nil) return u, uio } func newUDFProcess(name string) (*kapacitor.UDFProcess, *udf_test.IO) { uio := udf_test.NewIO() cmd := newTestCommander(uio) l := log.New(os.Stderr, fmt.Sprintf("[%s] ", name), log.LstdFlags) u := kapacitor.NewUDFProcess(name, "testNode", cmd, command.Spec{}, l, 0, nil) return u, uio } func TestUDFSocket_OpenClose(t *testing.T) { u, uio := newUDFSocket("OpenClose") testUDF_OpenClose(u, uio, t) } func TestUDFProcess_OpenClose(t *testing.T) { u, uio := newUDFProcess("OpenClose") testUDF_OpenClose(u, uio, t) } func testUDF_OpenClose(u udf.Interface, uio *udf_test.IO, t *testing.T) { u.Open() close(uio.Responses) u.Close() // read all requests and wait till the chan is closed for range uio.Requests { } if err := <-uio.ErrC; err != nil { t.Error(err) } } func TestUDFSocket_WritePoint(t *testing.T) { u, uio := newUDFSocket("WritePoint") testUDF_WritePoint(u, uio, t) } func TestUDFProcess_WritePoint(t *testing.T) { u, uio := newUDFProcess("WritePoint") testUDF_WritePoint(u, uio, t) } func testUDF_WritePoint(u udf.Interface, uio *udf_test.IO, t *testing.T) { go func() { req := <-uio.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Errorf("expected init message got %T", req.Message) } res := &agent.Response{ Message: &agent.Response_Init{ Init: &agent.InitResponse{ Success: true, }, }, } uio.Responses <- res req = <-uio.Requests pt, ok := req.Message.(*agent.Request_Point) if !ok { t.Errorf("expected point message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_Point{ Point: pt.Point, }, } uio.Responses <- res close(uio.Responses) }() err := u.Open() if err != nil { t.Fatal(err) } err = u.Init(nil) if err != nil { t.Fatal(err) } // Write point to server p := edge.NewPointMessage( "test", "db", "rp", models.Dimensions{}, models.Fields{"f1": 1.0, "f2": 2.0}, models.Tags{"t1": "v1", "t2": "v2"}, time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), ) u.In() <- p rp := <-u.Out() if !reflect.DeepEqual(rp, p) { t.Errorf("unexpected returned point got: %v exp %v", rp, p) } u.Close() // read all requests and wait till the chan is closed for range uio.Requests { } if err := <-uio.ErrC; err != nil { t.Error(err) } } func TestUDFSocket_WriteBatch(t *testing.T) { u, uio := newUDFSocket("WriteBatch") testUDF_WriteBatch(u, uio, t) } func TestUDFProcess_WriteBatch(t *testing.T) { u, uio := newUDFProcess("WriteBatch") testUDF_WriteBatch(u, uio, t) } func testUDF_WriteBatch(u udf.Interface, uio *udf_test.IO, t *testing.T) { go func() { req := <-uio.Requests _, ok := req.Message.(*agent.Request_Init) if !ok { t.Errorf("expected init message got %T", req.Message) } res := &agent.Response{ Message: &agent.Response_Init{ Init: &agent.InitResponse{ Success: true, }, }, } uio.Responses <- res // Begin batch req = <-uio.Requests bb, ok := req.Message.(*agent.Request_Begin) if !ok { t.Errorf("expected begin message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_Begin{ Begin: bb.Begin, }, } uio.Responses <- res // Point req = <-uio.Requests pt, ok := req.Message.(*agent.Request_Point) if !ok { t.Errorf("expected point message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_Point{ Point: pt.Point, }, } uio.Responses <- res // End batch req = <-uio.Requests eb, ok := req.Message.(*agent.Request_End) if !ok { t.Errorf("expected end message got %T", req.Message) } res = &agent.Response{ Message: &agent.Response_End{ End: eb.End, }, } uio.Responses <- res close(uio.Responses) }() err := u.Open() if err != nil { t.Fatal(err) } err = u.Init(nil) if err != nil { t.Fatal(err) } // Write point to server b := edge.NewBufferedBatchMessage( edge.NewBeginBatchMessage( "test", models.Tags{"t1": "v1"}, false, time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), 1, ), []edge.BatchPointMessage{ edge.NewBatchPointMessage( models.Fields{"f1": 1.0, "f2": 2.0, "f3": int64(1), "f4": "str"}, models.Tags{"t1": "v1", "t2": "v2"}, time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC), ), }, edge.NewEndBatchMessage(), ) u.In() <- b rb := <-u.Out() if !reflect.DeepEqual(b, rb) { t.Errorf("unexpected returned batch got: %v exp %v", rb, b) } u.Close() // read all requests and wait till the chan is closed for range uio.Requests { } if err := <-uio.ErrC; err != nil { t.Error(err) } } type testCommander struct { uio *udf_test.IO } func newTestCommander(uio *udf_test.IO) command.Commander { return &testCommander{ uio: uio, } } func (c *testCommander) NewCommand(command.Spec) command.Command { return c } func (c *testCommander) Start() error { return nil } func (c *testCommander) Wait() error { return nil } func (c *testCommander) Stdin(io.Reader) {} func (c *testCommander) Stdout(io.Writer) {} func (c *testCommander) Stderr(io.Writer) {} func (c *testCommander) StdinPipe() (io.WriteCloser, error) { return c.uio.In(), nil } func (c *testCommander) StdoutPipe() (io.Reader, error) { return c.uio.Out(), nil } func (c *testCommander) StderrPipe() (io.Reader, error) { return &bytes.Buffer{}, nil } func (c *testCommander) Kill() { c.uio.Kill() } type testSocket struct { uio *udf_test.IO } func newTestSocket(uio *udf_test.IO) kapacitor.Socket { return &testSocket{ uio: uio, } } func (s *testSocket) Open() error { return nil } func (s *testSocket) Close() error { return nil } func (s *testSocket) In() io.WriteCloser { return s.uio.In() } func (s *testSocket) Out() io.Reader { return s.uio.Out() } ================================================ FILE: vendor/github.com/influxdata/kapacitor/union.go ================================================ package kapacitor import ( "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" ) type UnionNode struct { node u *pipeline.UnionNode // Buffer of points/batches from each source. sources [][]timeMessage // the low water marks for each source. lowMarks []time.Time rename string } type timeMessage interface { edge.Message edge.TimeGetter } // Create a new UnionNode which combines all parent data streams into a single stream. // No transformation of any kind is performed. func newUnionNode(et *ExecutingTask, n *pipeline.UnionNode, l *log.Logger) (*UnionNode, error) { un := &UnionNode{ u: n, node: node{Node: n, et: et, logger: l}, rename: n.Rename, } un.node.runF = un.runUnion return un, nil } func (n *UnionNode) runUnion([]byte) error { // Keep buffer of values from parents so they can be ordered. n.sources = make([][]timeMessage, len(n.ins)) n.lowMarks = make([]time.Time, len(n.ins)) consumer := edge.NewMultiConsumerWithStats(n.ins, n) return consumer.Consume() } func (n *UnionNode) BufferedBatch(src int, batch edge.BufferedBatchMessage) error { n.timer.Start() defer n.timer.Stop() if n.rename != "" { batch = batch.ShallowCopy() batch.SetBegin(batch.Begin().ShallowCopy()) batch.Begin().SetName(n.rename) } // Add newest point to buffer n.sources[src] = append(n.sources[src], batch) // Emit the next values return n.emitReady(false) } func (n *UnionNode) Point(src int, p edge.PointMessage) error { n.timer.Start() defer n.timer.Stop() if n.rename != "" { p = p.ShallowCopy() p.SetName(n.rename) } // Add newest point to buffer n.sources[src] = append(n.sources[src], p) // Emit the next values return n.emitReady(false) } func (n *UnionNode) Barrier(src int, b edge.BarrierMessage) error { n.timer.Start() defer n.timer.Stop() // Add newest point to buffer n.sources[src] = append(n.sources[src], b) // Emit the next values return n.emitReady(false) } func (n *UnionNode) Finish() error { // We are done, emit all buffered return n.emitReady(true) } func (n *UnionNode) emitReady(drain bool) error { emitted := true // Emit all points until nothing changes for emitted { emitted = false // Find low water mark var mark time.Time validSources := 0 for i, values := range n.sources { sourceMark := n.lowMarks[i] if len(values) > 0 { t := values[0].Time() if mark.IsZero() || t.Before(mark) { mark = t } sourceMark = t } n.lowMarks[i] = sourceMark if !sourceMark.IsZero() { validSources++ // Only consider the sourceMark if we are not draining if !drain && (mark.IsZero() || sourceMark.Before(mark)) { mark = sourceMark } } } if !drain && validSources != len(n.sources) { // We can't continue processing until we have // at least one value from each parent. // Unless we are draining the buffer than we can continue. return nil } // Emit all values that are at or below the mark. for i, values := range n.sources { var j int l := len(values) for j = 0; j < l; j++ { if !values[j].Time().After(mark) { err := n.emit(values[j]) if err != nil { return err } // Note that we emitted something emitted = true } else { break } } // Drop values that were emitted n.sources[i] = values[j:] } } return nil } func (n *UnionNode) emit(m edge.Message) error { n.timer.Pause() defer n.timer.Resume() return edge.Forward(n.outs, m) } ================================================ FILE: vendor/github.com/influxdata/kapacitor/update_tick_docs.sh ================================================ #!/bin/bash # To generate the tick docs we use a little utility similar # to godoc called tickdoc. It organizes the fields and method # of structs into property methods and chaining methods. dest=$1 # output path for the .md files if [ -z "$dest" ] then echo "Usage: ./update_tick_docs.sh output_path" exit 1 fi tickdoc -config tickdoc.conf ./pipeline $dest ================================================ FILE: vendor/github.com/influxdata/kapacitor/where.go ================================================ package kapacitor import ( "errors" "fmt" "log" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/pipeline" "github.com/influxdata/kapacitor/tick/ast" "github.com/influxdata/kapacitor/tick/stateful" ) type WhereNode struct { node w *pipeline.WhereNode endpoint string expression stateful.Expression scopePool stateful.ScopePool } // Create a new WhereNode which filters down the batch or stream by a condition func newWhereNode(et *ExecutingTask, n *pipeline.WhereNode, l *log.Logger) (wn *WhereNode, err error) { wn = &WhereNode{ node: node{Node: n, et: et, logger: l}, w: n, } expr, err := stateful.NewExpression(n.Lambda.Expression) if err != nil { return nil, fmt.Errorf("Failed to compile expression in where clause: %v", err) } wn.expression = expr wn.scopePool = stateful.NewScopePool(ast.FindReferenceVariables(n.Lambda.Expression)) wn.runF = wn.runWhere if n.Lambda == nil { return nil, errors.New("nil expression passed to WhereNode") } return } func (n *WhereNode) runWhere(snapshot []byte) error { consumer := edge.NewGroupedConsumer( n.ins[0], n, ) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *WhereNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, n.newGroup()), ), nil } func (n *WhereNode) newGroup() *whereGroup { return &whereGroup{ n: n, expr: n.expression.CopyReset(), } } type whereGroup struct { n *WhereNode expr stateful.Expression } func (g *whereGroup) BeginBatch(begin edge.BeginBatchMessage) (edge.Message, error) { begin = begin.ShallowCopy() begin.SetSizeHint(0) return begin, nil } func (g *whereGroup) BatchPoint(bp edge.BatchPointMessage) (edge.Message, error) { return g.doWhere(bp) } func (g *whereGroup) EndBatch(end edge.EndBatchMessage) (edge.Message, error) { return end, nil } func (g *whereGroup) Point(p edge.PointMessage) (edge.Message, error) { return g.doWhere(p) } func (g *whereGroup) doWhere(p edge.FieldsTagsTimeGetterMessage) (edge.Message, error) { pass, err := EvalPredicate(g.expr, g.n.scopePool, p) if err != nil { g.n.incrementErrorCount() g.n.logger.Println("E! error while evaluating expression:", err) return nil, nil } if pass { return p, nil } return nil, nil } func (g *whereGroup) Barrier(b edge.BarrierMessage) (edge.Message, error) { return b, nil } func (g *whereGroup) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } ================================================ FILE: vendor/github.com/influxdata/kapacitor/window.go ================================================ package kapacitor import ( "errors" "fmt" "log" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/influxdata/kapacitor/pipeline" ) type WindowNode struct { node w *pipeline.WindowNode } // Create a new WindowNode, which windows data for a period of time and emits the window. func newWindowNode(et *ExecutingTask, n *pipeline.WindowNode, l *log.Logger) (*WindowNode, error) { if n.Period == 0 && n.PeriodCount == 0 { return nil, errors.New("window node must have either a non zero period or non zero period count") } wn := &WindowNode{ w: n, node: node{Node: n, et: et, logger: l}, } wn.node.runF = wn.runWindow return wn, nil } func (n *WindowNode) runWindow([]byte) error { consumer := edge.NewGroupedConsumer(n.ins[0], n) n.statMap.Set(statCardinalityGauge, consumer.CardinalityVar()) return consumer.Consume() } func (n *WindowNode) NewGroup(group edge.GroupInfo, first edge.PointMeta) (edge.Receiver, error) { r, err := n.newWindow(group, first) if err != nil { return nil, err } return edge.NewReceiverFromForwardReceiverWithStats( n.outs, edge.NewTimedForwardReceiver(n.timer, r), ), nil } func (n *WindowNode) DeleteGroup(group models.GroupID) { // Nothing to do } func (n *WindowNode) newWindow(group edge.GroupInfo, first edge.PointMeta) (edge.ForwardReceiver, error) { switch { case n.w.Period != 0: return newWindowByTime( first.Name(), first.Time(), group, n.w.Period, n.w.Every, n.w.AlignFlag, n.w.FillPeriodFlag, n.logger, ), nil case n.w.PeriodCount != 0: return newWindowByCount( first.Name(), group, int(n.w.PeriodCount), int(n.w.EveryCount), n.w.FillPeriodFlag, n.logger, ), nil default: return nil, errors.New("unreachable code, window node should have a non-zero period or period count") } } type windowByTime struct { name string group edge.GroupInfo nextEmit time.Time buf *windowTimeBuffer align, fillPeriod bool period time.Duration every time.Duration logger *log.Logger } func newWindowByTime( name string, t time.Time, group edge.GroupInfo, period, every time.Duration, align, fillPeriod bool, logger *log.Logger, ) *windowByTime { // Determine nextEmit time. var nextEmit time.Time if fillPeriod { nextEmit = t.Add(period) if align { firstPeriod := nextEmit // Needs to be aligned with Every and be greater than now+Period nextEmit = nextEmit.Truncate(every) if !nextEmit.After(firstPeriod) { // This means we will drop the first few points nextEmit = nextEmit.Add(every) } } } else { nextEmit = t.Add(every) if align { nextEmit = nextEmit.Truncate(every) } } return &windowByTime{ name: name, group: group, nextEmit: nextEmit, buf: &windowTimeBuffer{logger: logger}, align: align, fillPeriod: fillPeriod, period: period, every: every, logger: logger, } } func (w *windowByTime) BeginBatch(edge.BeginBatchMessage) (edge.Message, error) { return nil, errors.New("window does not support batch data") } func (w *windowByTime) BatchPoint(edge.BatchPointMessage) (edge.Message, error) { return nil, errors.New("window does not support batch data") } func (w *windowByTime) EndBatch(edge.EndBatchMessage) (edge.Message, error) { return nil, errors.New("window does not support batch data") } func (w *windowByTime) Barrier(b edge.BarrierMessage) (edge.Message, error) { //TODO(nathanielc): Implement barrier messages to flush window return b, nil } func (w *windowByTime) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (w *windowByTime) Point(p edge.PointMessage) (msg edge.Message, err error) { if w.every == 0 { // Insert point before. w.buf.insert(p) // Since we are emitting every point we can use a right aligned window (oldest, now] if !p.Time().Before(w.nextEmit) { // purge old points oldest := p.Time().Add(-1 * w.period) w.buf.purge(oldest, false) // get current batch msg = w.batch(p.Time()) // Next emit time is now w.nextEmit = p.Time() } } else { // Since more points can arrive with the same time we need to use a left aligned window [oldest, now). if !p.Time().Before(w.nextEmit) { // purge old points oldest := w.nextEmit.Add(-1 * w.period) w.buf.purge(oldest, true) // get current batch msg = w.batch(w.nextEmit) // Determine next emit time. // This is dependent on the current time not the last time we emitted. w.nextEmit = p.Time().Add(w.every) if w.align { w.nextEmit = w.nextEmit.Truncate(w.every) } } // Insert point after. w.buf.insert(p) } return } // batch returns the current window buffer as a batch message. // TODO(nathanielc): A possible optimization could be to not buffer the data at all if we know that we do not have overlapping windows. func (w *windowByTime) batch(tmax time.Time) edge.BufferedBatchMessage { points := w.buf.points() return edge.NewBufferedBatchMessage( edge.NewBeginBatchMessage( w.name, w.group.Tags, w.group.Dimensions.ByName, tmax, len(points), ), points, edge.NewEndBatchMessage(), ) } // implements a purpose built ring buffer for the window of points type windowTimeBuffer struct { window []edge.PointMessage start int stop int size int logger *log.Logger } // Insert a single point into the buffer. func (b *windowTimeBuffer) insert(p edge.PointMessage) { if b.size == cap(b.window) { //Increase our buffer c := 2 * (b.size + 1) w := make([]edge.PointMessage, b.size+1, c) if b.size == 0 { //do nothing } else if b.stop > b.start { n := copy(w, b.window[b.start:b.stop]) if n != b.size { panic(fmt.Sprintf("did not copy all the data: copied: %d size: %d start: %d stop: %d\n", n, b.size, b.start, b.stop)) } } else { n := 0 n += copy(w, b.window[b.start:]) n += copy(w[b.size-b.start:], b.window[:b.stop]) if n != b.size { panic(fmt.Sprintf("did not copy all the data: copied: %d size: %d start: %d stop: %d\n", n, b.size, b.start, b.stop)) } } b.window = w b.start = 0 b.stop = b.size } // Check if we need to wrap around if len(b.window) == cap(b.window) && b.stop == len(b.window) { b.stop = 0 } // Insert point if b.stop == len(b.window) { b.window = append(b.window, p) } else { b.window[b.stop] = p } b.size++ b.stop++ } // Purge expired data from the window. func (b *windowTimeBuffer) purge(oldest time.Time, inclusive bool) { include := func(t time.Time) bool { if inclusive { return !t.Before(oldest) } return t.After(oldest) } l := len(b.window) if l == 0 { return } if b.start < b.stop { for ; b.start < b.stop; b.start++ { if include(b.window[b.start].Time()) { break } } b.size = b.stop - b.start } else { if include(b.window[l-1].Time()) { for ; b.start < l; b.start++ { if include(b.window[b.start].Time()) { break } } b.size = l - b.start + b.stop } else { for b.start = 0; b.start < b.stop; b.start++ { if include(b.window[b.start].Time()) { break } } b.size = b.stop - b.start } } } // Returns a copy of the current buffer. // TODO(nathanielc): Optimize this function use buffered vs unbuffered batch messages. func (b *windowTimeBuffer) points() []edge.BatchPointMessage { if b.size == 0 { return nil } points := make([]edge.BatchPointMessage, b.size) if b.stop > b.start { for i, p := range b.window[b.start:b.stop] { points[i] = edge.BatchPointFromPoint(p) } } else { j := 0 l := len(b.window) for i := b.start; i < l; i++ { p := b.window[i] points[j] = edge.BatchPointFromPoint(p) j++ } for i := 0; i < b.stop; i++ { p := b.window[i] points[j] = edge.BatchPointFromPoint(p) j++ } } return points } type windowByCount struct { name string group edge.GroupInfo buf []edge.BatchPointMessage start int stop int period int every int nextEmit int size int count int logger *log.Logger } func newWindowByCount( name string, group edge.GroupInfo, period, every int, fillPeriod bool, logger *log.Logger) *windowByCount { // Determine the first nextEmit index nextEmit := every if fillPeriod { nextEmit = period } return &windowByCount{ name: name, group: group, buf: make([]edge.BatchPointMessage, period), period: period, every: every, nextEmit: nextEmit, logger: logger, } } func (w *windowByCount) BeginBatch(edge.BeginBatchMessage) (edge.Message, error) { return nil, errors.New("window does not support batch data") } func (w *windowByCount) BatchPoint(edge.BatchPointMessage) (edge.Message, error) { return nil, errors.New("window does not support batch data") } func (w *windowByCount) EndBatch(edge.EndBatchMessage) (edge.Message, error) { return nil, errors.New("window does not support batch data") } func (w *windowByCount) Barrier(b edge.BarrierMessage) (edge.Message, error) { //TODO(nathanielc): Implement barrier messages to flush window return b, nil } func (w *windowByCount) DeleteGroup(d edge.DeleteGroupMessage) (edge.Message, error) { return d, nil } func (w *windowByCount) Point(p edge.PointMessage) (msg edge.Message, err error) { w.buf[w.stop] = edge.BatchPointFromPoint(p) w.stop = (w.stop + 1) % w.period if w.size == w.period { w.start = (w.start + 1) % w.period } else { w.size++ } w.count++ //Check if its time to emit if w.count == w.nextEmit { w.nextEmit += w.every msg = w.batch() } return } func (w *windowByCount) batch() edge.BufferedBatchMessage { points := w.points() return edge.NewBufferedBatchMessage( edge.NewBeginBatchMessage( w.name, w.group.Tags, w.group.Dimensions.ByName, points[len(points)-1].Time(), len(points), ), points, edge.NewEndBatchMessage(), ) } // Returns a copy of the current buffer. func (w *windowByCount) points() []edge.BatchPointMessage { if w.size == 0 { return nil } points := make([]edge.BatchPointMessage, w.size) if w.stop > w.start { copy(points, w.buf[w.start:w.stop]) } else { j := 0 l := len(w.buf) for i := w.start; i < l; i++ { points[j] = w.buf[i] j++ } for i := 0; i < w.stop; i++ { points[j] = w.buf[i] j++ } } return points } ================================================ FILE: vendor/github.com/influxdata/kapacitor/window_test.go ================================================ package kapacitor import ( "log" "os" "testing" "time" "github.com/influxdata/kapacitor/edge" "github.com/influxdata/kapacitor/models" "github.com/stretchr/testify/assert" ) var logger = log.New(os.Stderr, "[window] ", log.LstdFlags|log.Lshortfile) func TestWindowBufferByTime(t *testing.T) { assert := assert.New(t) buf := &windowTimeBuffer{logger: logger} size := 100 // fill buffer for i := 1; i <= size; i++ { t := time.Unix(int64(i), 0) p := edge.NewPointMessage( "name", "db", "rp", models.Dimensions{}, nil, nil, t, ) buf.insert(p) assert.Equal(i, buf.size) assert.Equal(0, buf.start) assert.Equal(i, buf.stop) } // purge entire buffer for i := 0; i <= size; i++ { oldest := time.Unix(int64(i+1), 0).UTC() buf.purge(oldest, true) assert.Equal(size-i, buf.size, "i: %d", i) assert.Equal(i, buf.start, "i: %d", i) assert.Equal(size, buf.stop, "i: %d", i) points := buf.points() if assert.Equal(size-i, len(points)) { for _, p := range points { assert.True(!p.Time().Before(oldest), "Point %s is not after oldest time %s", p.Time(), oldest) } } } assert.Equal(0, buf.size) // fill buffer again oldest := time.Unix(int64(size), 0).UTC() for i := 1; i <= size*2; i++ { t := time.Unix(int64(i+size), 0) p := edge.NewPointMessage( "name", "db", "rp", models.Dimensions{}, nil, nil, t, ) buf.insert(p) assert.Equal(i, buf.size) points := buf.points() if assert.Equal(i, len(points)) { for _, p := range points { if assert.NotNil(p, "i:%d", i) { assert.True(!p.Time().Before(oldest), "Point %s is not after oldest time %s", p.Time(), oldest) } } } } } func TestWindowBufferByCount(t *testing.T) { testCases := []struct { size int every int period int fillPeriod bool }{ { size: 100, every: 10, period: 10, }, { size: 100, every: 3, period: 10, }, { size: 100, every: 1, period: 2, }, { size: 100, every: 1, period: 1, }, { size: 100, every: 10, period: 5, }, { size: 100, every: 1, period: 5, }, } for _, tc := range testCases { t.Logf("Starting test size %d period %d every %d", tc.size, tc.period, tc.every) w := newWindowByCount( "test", edge.GroupInfo{}, tc.period, tc.every, tc.fillPeriod, logger, ) // fill buffer for i := 1; i <= tc.size; i++ { p := edge.NewPointMessage( "name", "db", "rp", models.Dimensions{}, nil, nil, time.Unix(int64(i), 0).UTC(), ) msg, err := w.Point(p) if err != nil { t.Fatal(err) } expEmit := tc.every == 0 || i%tc.every == 0 if tc.fillPeriod { expEmit = i > tc.period && expEmit } if expEmit && msg == nil { t.Errorf("%d unexpected nil forward message: got nil message, expected non nil message", i) } if !expEmit && msg != nil { t.Errorf("%d unexpected forward message: got non-nil message %v, expected nil message", i, msg) } size := i if size > tc.period { size = tc.period } if got, exp := w.size, size; got != exp { t.Errorf("%d unexpected size: got %d exp %d", i, got, exp) } start := (i - tc.period) % tc.period if start < 0 { start = 0 } if got, exp := w.start, start; got != exp { t.Errorf("%d unexpected start: got %d exp %d", i, got, exp) } if got, exp := w.stop, i%tc.period; got != exp { t.Errorf("%d unexpected stop: got %d exp %d", i, got, exp) } if msg != nil { if msg.Type() != edge.BufferedBatch { t.Fatalf("unexpected message type %v", msg.Type()) } b := msg.(edge.BufferedBatchMessage) l := i if l > tc.period { l = tc.period } points := b.Points() if got, exp := len(points), l; got != exp { t.Fatalf("%d unexpected number of points got %d exp %d", i, got, exp) } for j, p := range points { if got, exp := p.Time(), time.Unix(int64(i+j-len(points)+1), 0).UTC(); !got.Equal(exp) { t.Errorf("%d unexpected point[%d].Time: got %v exp %v", i, j, got, exp) } } } } } } ================================================ FILE: vendor/github.com/influxdata/wlog/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2016 InfluxData Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/influxdata/wlog/README.md ================================================ # wlog Simple log level based Go logger. Provides an io.Writer that filters log messages based on a log level prefix. Valid log levels are: DEBUG, INFO, WARN, ERROR, OFF. Log messages need to begin with a L! where L is one of D, I, W, or E. ## Usage Create a *log.Logger via wlog.New: ```go package main import ( "log" "os" "github.com/influxdata/wlog" ) func main() { var logger *log.Logger logger = wlog.New(os.Stderr, "prefix", log.LstdFlags) logger.Println("I! initialized logger") } ``` Create a *log.Logger explicitly using wlog.Writer: ```go package main import ( "log" "os" "github.com/influxdata/wlog" ) func main() { var logger *log.Logger logger = log.New(wlog.NewWriter(os.Stderr), "prefix", log.LstdFlags) logger.Println("I! initialized logger") } ``` Prefix log messages with a log level char and the `!` delimiter. ```go logger.Println("D! this is a debug log") logger.Println("I! this is an info log") logger.Println("W! this is a warn log") logger.Println("E! this is an error log") ``` The log level can be changed via the SetLevel or the SetLevelFromName functions. ```go package main import ( "log" "os" "github.com/influxdata/wlog" ) func main() { var logger *log.Logger logger = wlog.New(os.Stderr, "prefix", log.LstdFlags) wlog.SetLevel(wlog.DEBUG) logger.Println("D! initialized logger") wlog.SetLevelFromName("INFO") logger.Println("D! this message will be dropped") logger.Println("I! this message will be printed") } ``` ================================================ FILE: vendor/github.com/influxdata/wlog/writer.go ================================================ /* Provides an io.Writer that filters log messages based on a log level. Valid log levels are: DEBUG, INFO, WARN, ERROR. Log messages need to begin with a L! where L is one of D, I, W, or E. Examples: log.Println("D! this is a debug log") log.Println("I! this is an info log") log.Println("W! this is a warn log") log.Println("E! this is an error log") Simply pass a instance of wlog.Writer to log.New or use the helper wlog.New function. The log level can be changed via the SetLevel or the SetLevelFromName functions. */ package wlog import ( "fmt" "io" "log" "strings" "sync" ) type Level int const ( _ Level = iota DEBUG INFO WARN ERROR OFF ) const Delimiter = '!' var invalidMSG = []byte("log messages must have 'L!' prefix where L is one of 'D', 'I', 'W', 'E'") var Levels = map[byte]Level{ 'D': DEBUG, 'I': INFO, 'W': WARN, 'E': ERROR, } var ReverseLevels map[Level]byte func init() { ReverseLevels = make(map[Level]byte, len(Levels)) for k, l := range Levels { ReverseLevels[l] = k } } // The global and only log level. Log levels are not implemented per writer. var logLevel = INFO var mu sync.RWMutex // Set the current logging Level. func SetLevel(l Level) { mu.Lock() defer mu.Unlock() logLevel = l } // Retrieve the current logging Level. func LogLevel() Level { mu.RLock() defer mu.RUnlock() return logLevel } // name to Level mappings var StringToLevel = map[string]Level{ "DEBUG": DEBUG, "INFO": INFO, "WARN": WARN, "ERROR": ERROR, "OFF": OFF, } // Set the log level via a string name. To set it directly use 'logLevel'. func SetLevelFromName(level string) error { l := StringToLevel[strings.ToUpper(level)] if l > 0 { SetLevel(l) } else { return fmt.Errorf("invalid log level: %q", level) } return nil } // Implements io.Writer. Checks first byte of write for log level // and drops the log if necessary type Writer struct { start int w io.Writer } // Create a new *log.Logger wrapping w in a wlog.Writer func New(w io.Writer, prefix string, flag int) *log.Logger { return log.New(NewWriter(w), prefix, flag) } // Create a new wlog.Writer wrapping w. func NewWriter(w io.Writer) *Writer { return &Writer{-1, w} } // Implements the io.Writer method. func (w *Writer) Write(buf []byte) (int, error) { if len(buf) > 0 { if w.start == -1 { // Find start of message index for i, c := range buf { if c == Delimiter && i > 0 { l := buf[i-1] level := Levels[l] if level > 0 { w.start = i - 1 break } } } if w.start == -1 { buf = append(invalidMSG, buf...) return w.w.Write(buf) } } l := Levels[buf[w.start]] if l >= LogLevel() { return w.w.Write(buf) } else if l == 0 { buf = append(invalidMSG, buf...) return w.w.Write(buf) } } return 0, nil } // StaticLevelWriter prefixes all log messages // with a static log level. type StaticLevelWriter struct { levelPrefix []byte w io.Writer } // Create a writer that always append a static log prefix to all messages. // Usefult for supplying a *log.Logger to a package that doesn't // prefix log messages itself. func NewStaticLevelWriter(w io.Writer, level Level) *StaticLevelWriter { levelPrefix := []byte{ReverseLevels[level], '!', ' '} return &StaticLevelWriter{ levelPrefix: levelPrefix, w: w, } } func (w *StaticLevelWriter) Write(buf []byte) (int, error) { buf = append(w.levelPrefix, buf...) return w.w.Write(buf) } ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/.travis.yml ================================================ language: go ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/NOTICE ================================================ Copyright 2012 Matt T. Proud (matt.proud@gmail.com) ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/README.md ================================================ # Overview This repository provides various Protocol Buffer extensions for the Go language (golang), namely support for record length-delimited message streaming. | Java | Go | | ------------------------------ | --------------------- | | MessageLite#parseDelimitedFrom | pbutil.ReadDelimited | | MessageLite#writeDelimitedTo | pbutil.WriteDelimited | Because [Code Review 9102043](https://codereview.appspot.com/9102043/) is destined to never be merged into mainline (i.e., never be promoted to formal [goprotobuf features](https://github.com/golang/protobuf)), this repository will live here in the wild. # Documentation We have [generated Go Doc documentation](http://godoc.org/github.com/matttproud/golang_protobuf_extensions/pbutil) here. # Testing [![Build Status](https://travis-ci.org/matttproud/golang_protobuf_extensions.png?branch=master)](https://travis-ci.org/matttproud/golang_protobuf_extensions) ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/all_test.go ================================================ // Copyright 2013 Matt T. Proud // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pbutil import ( "bytes" "testing" . "github.com/golang/protobuf/proto" . "github.com/golang/protobuf/proto/testdata" ) func TestWriteDelimited(t *testing.T) { t.Parallel() for _, test := range []struct { msg Message buf []byte n int err error }{ { msg: &Empty{}, n: 1, buf: []byte{0}, }, { msg: &GoEnum{Foo: FOO_FOO1.Enum()}, n: 3, buf: []byte{2, 8, 1}, }, { msg: &Strings{ StringField: String(`This is my gigantic, unhappy string. It exceeds the encoding size of a single byte varint. We are using it to fuzz test the correctness of the header decoding mechanisms, which may prove problematic. I expect it may. Let's hope you enjoy testing as much as we do.`), }, n: 271, buf: []byte{141, 2, 10, 138, 2, 84, 104, 105, 115, 32, 105, 115, 32, 109, 121, 32, 103, 105, 103, 97, 110, 116, 105, 99, 44, 32, 117, 110, 104, 97, 112, 112, 121, 32, 115, 116, 114, 105, 110, 103, 46, 32, 32, 73, 116, 32, 101, 120, 99, 101, 101, 100, 115, 10, 116, 104, 101, 32, 101, 110, 99, 111, 100, 105, 110, 103, 32, 115, 105, 122, 101, 32, 111, 102, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 98, 121, 116, 101, 32, 118, 97, 114, 105, 110, 116, 46, 32, 32, 87, 101, 32, 97, 114, 101, 32, 117, 115, 105, 110, 103, 32, 105, 116, 32, 116, 111, 32, 102, 117, 122, 122, 32, 116, 101, 115, 116, 32, 116, 104, 101, 10, 99, 111, 114, 114, 101, 99, 116, 110, 101, 115, 115, 32, 111, 102, 32, 116, 104, 101, 32, 104, 101, 97, 100, 101, 114, 32, 100, 101, 99, 111, 100, 105, 110, 103, 32, 109, 101, 99, 104, 97, 110, 105, 115, 109, 115, 44, 32, 119, 104, 105, 99, 104, 32, 109, 97, 121, 32, 112, 114, 111, 118, 101, 32, 112, 114, 111, 98, 108, 101, 109, 97, 116, 105, 99, 46, 10, 73, 32, 101, 120, 112, 101, 99, 116, 32, 105, 116, 32, 109, 97, 121, 46, 32, 32, 76, 101, 116, 39, 115, 32, 104, 111, 112, 101, 32, 121, 111, 117, 32, 101, 110, 106, 111, 121, 32, 116, 101, 115, 116, 105, 110, 103, 32, 97, 115, 32, 109, 117, 99, 104, 32, 97, 115, 32, 119, 101, 32, 100, 111, 46}, }, } { var buf bytes.Buffer if n, err := WriteDelimited(&buf, test.msg); n != test.n || err != test.err { t.Fatalf("WriteDelimited(buf, %#v) = %v, %v; want %v, %v", test.msg, n, err, test.n, test.err) } if out := buf.Bytes(); !bytes.Equal(out, test.buf) { t.Fatalf("WriteDelimited(buf, %#v); buf = %v; want %v", test.msg, out, test.buf) } } } func TestReadDelimited(t *testing.T) { t.Parallel() for _, test := range []struct { buf []byte msg Message n int err error }{ { buf: []byte{0}, msg: &Empty{}, n: 1, }, { n: 3, buf: []byte{2, 8, 1}, msg: &GoEnum{Foo: FOO_FOO1.Enum()}, }, { buf: []byte{141, 2, 10, 138, 2, 84, 104, 105, 115, 32, 105, 115, 32, 109, 121, 32, 103, 105, 103, 97, 110, 116, 105, 99, 44, 32, 117, 110, 104, 97, 112, 112, 121, 32, 115, 116, 114, 105, 110, 103, 46, 32, 32, 73, 116, 32, 101, 120, 99, 101, 101, 100, 115, 10, 116, 104, 101, 32, 101, 110, 99, 111, 100, 105, 110, 103, 32, 115, 105, 122, 101, 32, 111, 102, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 98, 121, 116, 101, 32, 118, 97, 114, 105, 110, 116, 46, 32, 32, 87, 101, 32, 97, 114, 101, 32, 117, 115, 105, 110, 103, 32, 105, 116, 32, 116, 111, 32, 102, 117, 122, 122, 32, 116, 101, 115, 116, 32, 116, 104, 101, 10, 99, 111, 114, 114, 101, 99, 116, 110, 101, 115, 115, 32, 111, 102, 32, 116, 104, 101, 32, 104, 101, 97, 100, 101, 114, 32, 100, 101, 99, 111, 100, 105, 110, 103, 32, 109, 101, 99, 104, 97, 110, 105, 115, 109, 115, 44, 32, 119, 104, 105, 99, 104, 32, 109, 97, 121, 32, 112, 114, 111, 118, 101, 32, 112, 114, 111, 98, 108, 101, 109, 97, 116, 105, 99, 46, 10, 73, 32, 101, 120, 112, 101, 99, 116, 32, 105, 116, 32, 109, 97, 121, 46, 32, 32, 76, 101, 116, 39, 115, 32, 104, 111, 112, 101, 32, 121, 111, 117, 32, 101, 110, 106, 111, 121, 32, 116, 101, 115, 116, 105, 110, 103, 32, 97, 115, 32, 109, 117, 99, 104, 32, 97, 115, 32, 119, 101, 32, 100, 111, 46}, msg: &Strings{ StringField: String(`This is my gigantic, unhappy string. It exceeds the encoding size of a single byte varint. We are using it to fuzz test the correctness of the header decoding mechanisms, which may prove problematic. I expect it may. Let's hope you enjoy testing as much as we do.`), }, n: 271, }, } { msg := Clone(test.msg) msg.Reset() if n, err := ReadDelimited(bytes.NewBuffer(test.buf), msg); n != test.n || err != test.err { t.Fatalf("ReadDelimited(%v, msg) = %v, %v; want %v, %v", test.buf, n, err, test.n, test.err) } if !Equal(msg, test.msg) { t.Fatalf("ReadDelimited(%v, msg); msg = %v; want %v", test.buf, msg, test.msg) } } } func TestEndToEndValid(t *testing.T) { t.Parallel() for _, test := range [][]Message{ {&Empty{}}, {&GoEnum{Foo: FOO_FOO1.Enum()}, &Empty{}, &GoEnum{Foo: FOO_FOO1.Enum()}}, {&GoEnum{Foo: FOO_FOO1.Enum()}}, {&Strings{ StringField: String(`This is my gigantic, unhappy string. It exceeds the encoding size of a single byte varint. We are using it to fuzz test the correctness of the header decoding mechanisms, which may prove problematic. I expect it may. Let's hope you enjoy testing as much as we do.`), }}, } { var buf bytes.Buffer var written int for i, msg := range test { n, err := WriteDelimited(&buf, msg) if err != nil { // Assumption: TestReadDelimited and TestWriteDelimited are sufficient // and inputs for this test are explicitly exercised there. t.Fatalf("WriteDelimited(buf, %v[%d]) = ?, %v; wanted ?, nil", test, i, err) } written += n } var read int for i, msg := range test { out := Clone(msg) out.Reset() n, _ := ReadDelimited(&buf, out) // Decide to do EOF checking? read += n if !Equal(out, msg) { t.Fatalf("out = %v; want %v[%d] = %#v", out, test, i, msg) } } if read != written { t.Fatalf("%v read = %d; want %d", test, read, written) } } } ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go ================================================ // Copyright 2013 Matt T. Proud // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pbutil import ( "encoding/binary" "errors" "io" "github.com/golang/protobuf/proto" ) var errInvalidVarint = errors.New("invalid varint32 encountered") // ReadDelimited decodes a message from the provided length-delimited stream, // where the length is encoded as 32-bit varint prefix to the message body. // It returns the total number of bytes read and any applicable error. This is // roughly equivalent to the companion Java API's // MessageLite#parseDelimitedFrom. As per the reader contract, this function // calls r.Read repeatedly as required until exactly one message including its // prefix is read and decoded (or an error has occurred). The function never // reads more bytes from the stream than required. The function never returns // an error if a message has been read and decoded correctly, even if the end // of the stream has been reached in doing so. In that case, any subsequent // calls return (0, io.EOF). func ReadDelimited(r io.Reader, m proto.Message) (n int, err error) { // Per AbstractParser#parsePartialDelimitedFrom with // CodedInputStream#readRawVarint32. var headerBuf [binary.MaxVarintLen32]byte var bytesRead, varIntBytes int var messageLength uint64 for varIntBytes == 0 { // i.e. no varint has been decoded yet. if bytesRead >= len(headerBuf) { return bytesRead, errInvalidVarint } // We have to read byte by byte here to avoid reading more bytes // than required. Each read byte is appended to what we have // read before. newBytesRead, err := r.Read(headerBuf[bytesRead : bytesRead+1]) if newBytesRead == 0 { if err != nil { return bytesRead, err } // A Reader should not return (0, nil), but if it does, // it should be treated as no-op (according to the // Reader contract). So let's go on... continue } bytesRead += newBytesRead // Now present everything read so far to the varint decoder and // see if a varint can be decoded already. messageLength, varIntBytes = proto.DecodeVarint(headerBuf[:bytesRead]) } messageBuf := make([]byte, messageLength) newBytesRead, err := io.ReadFull(r, messageBuf) bytesRead += newBytesRead if err != nil { return bytesRead, err } return bytesRead, proto.Unmarshal(messageBuf, m) } ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode_test.go ================================================ // Copyright 2016 Matt T. Proud // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pbutil import ( "bytes" "io" "testing" "testing/iotest" ) func TestReadDelimitedIllegalVarint(t *testing.T) { t.Parallel() var tests = []struct { in []byte n int err error }{ { in: []byte{255, 255, 255, 255, 255}, n: 5, err: errInvalidVarint, }, { in: []byte{255, 255, 255, 255, 255, 255}, n: 5, err: errInvalidVarint, }, } for _, test := range tests { n, err := ReadDelimited(bytes.NewReader(test.in), nil) if got, want := n, test.n; got != want { t.Errorf("ReadDelimited(%#v, nil) = %#v, ?; want = %v#, ?", test.in, got, want) } if got, want := err, test.err; got != want { t.Errorf("ReadDelimited(%#v, nil) = ?, %#v; want = ?, %#v", test.in, got, want) } } } func TestReadDelimitedPrematureHeader(t *testing.T) { t.Parallel() var data = []byte{128, 5} // 256 + 256 + 128 n, err := ReadDelimited(bytes.NewReader(data[0:1]), nil) if got, want := n, 1; got != want { t.Errorf("ReadDelimited(%#v, nil) = %#v, ?; want = %v#, ?", data[0:1], got, want) } if got, want := err, io.EOF; got != want { t.Errorf("ReadDelimited(%#v, nil) = ?, %#v; want = ?, %#v", data[0:1], got, want) } } func TestReadDelimitedPrematureBody(t *testing.T) { t.Parallel() var data = []byte{128, 5, 0, 0, 0} // 256 + 256 + 128 n, err := ReadDelimited(bytes.NewReader(data[:]), nil) if got, want := n, 5; got != want { t.Errorf("ReadDelimited(%#v, nil) = %#v, ?; want = %v#, ?", data, got, want) } if got, want := err, io.ErrUnexpectedEOF; got != want { t.Errorf("ReadDelimited(%#v, nil) = ?, %#v; want = ?, %#v", data, got, want) } } func TestReadDelimitedPrematureHeaderIncremental(t *testing.T) { t.Parallel() var data = []byte{128, 5} // 256 + 256 + 128 n, err := ReadDelimited(iotest.OneByteReader(bytes.NewReader(data[0:1])), nil) if got, want := n, 1; got != want { t.Errorf("ReadDelimited(%#v, nil) = %#v, ?; want = %v#, ?", data[0:1], got, want) } if got, want := err, io.EOF; got != want { t.Errorf("ReadDelimited(%#v, nil) = ?, %#v; want = ?, %#v", data[0:1], got, want) } } func TestReadDelimitedPrematureBodyIncremental(t *testing.T) { t.Parallel() var data = []byte{128, 5, 0, 0, 0} // 256 + 256 + 128 n, err := ReadDelimited(iotest.OneByteReader(bytes.NewReader(data[:])), nil) if got, want := n, 5; got != want { t.Errorf("ReadDelimited(%#v, nil) = %#v, ?; want = %v#, ?", data, got, want) } if got, want := err, io.ErrUnexpectedEOF; got != want { t.Errorf("ReadDelimited(%#v, nil) = ?, %#v; want = ?, %#v", data, got, want) } } ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go ================================================ // Copyright 2013 Matt T. Proud // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package pbutil provides record length-delimited Protocol Buffer streaming. package pbutil ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go ================================================ // Copyright 2013 Matt T. Proud // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pbutil import ( "encoding/binary" "io" "github.com/golang/protobuf/proto" ) // WriteDelimited encodes and dumps a message to the provided writer prefixed // with a 32-bit varint indicating the length of the encoded message, producing // a length-delimited record stream, which can be used to chain together // encoded messages of the same type together in a file. It returns the total // number of bytes written and any applicable error. This is roughly // equivalent to the companion Java API's MessageLite#writeDelimitedTo. func WriteDelimited(w io.Writer, m proto.Message) (n int, err error) { buffer, err := proto.Marshal(m) if err != nil { return 0, err } var buf [binary.MaxVarintLen32]byte encodedLength := binary.PutUvarint(buf[:], uint64(len(buffer))) sync, err := w.Write(buf[:encodedLength]) if err != nil { return sync, err } n, err = w.Write(buffer) return n + sync, err } ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode_test.go ================================================ // Copyright 2016 Matt T. Proud // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pbutil import ( "bytes" "errors" "testing" "github.com/golang/protobuf/proto" ) var errMarshal = errors.New("pbutil: can't marshal") type cantMarshal struct{ proto.Message } func (cantMarshal) Marshal() ([]byte, error) { return nil, errMarshal } var _ proto.Message = cantMarshal{} func TestWriteDelimitedMarshalErr(t *testing.T) { t.Parallel() var data cantMarshal var buf bytes.Buffer n, err := WriteDelimited(&buf, data) if got, want := n, 0; got != want { t.Errorf("WriteDelimited(buf, %#v) = %#v, ?; want = %v#, ?", data, got, want) } if got, want := err, errMarshal; got != want { t.Errorf("WriteDelimited(buf, %#v) = ?, %#v; want = ?, %#v", data, got, want) } } type canMarshal struct{ proto.Message } func (canMarshal) Marshal() ([]byte, error) { return []byte{0, 1, 2, 3, 4, 5}, nil } var errWrite = errors.New("pbutil: can't write") type cantWrite struct{} func (cantWrite) Write([]byte) (int, error) { return 0, errWrite } func TestWriteDelimitedWriteErr(t *testing.T) { t.Parallel() var data canMarshal var buf cantWrite n, err := WriteDelimited(buf, data) if got, want := n, 0; got != want { t.Errorf("WriteDelimited(buf, %#v) = %#v, ?; want = %v#, ?", data, got, want) } if got, want := err, errWrite; got != want { t.Errorf("WriteDelimited(buf, %#v) = ?, %#v; want = ?, %#v", data, got, want) } } ================================================ FILE: vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/fixtures_test.go ================================================ // Copyright 2010 The Go Authors. All rights reserved. // http://github.com/golang/protobuf/ // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package pbutil import ( . "github.com/golang/protobuf/proto" . "github.com/golang/protobuf/proto/testdata" ) // FROM https://github.com/golang/protobuf/blob/master/proto/all_test.go. func initGoTestField() *GoTestField { f := new(GoTestField) f.Label = String("label") f.Type = String("type") return f } // These are all structurally equivalent but the tag numbers differ. // (It's remarkable that required, optional, and repeated all have // 8 letters.) func initGoTest_RequiredGroup() *GoTest_RequiredGroup { return &GoTest_RequiredGroup{ RequiredField: String("required"), } } func initGoTest_OptionalGroup() *GoTest_OptionalGroup { return &GoTest_OptionalGroup{ RequiredField: String("optional"), } } func initGoTest_RepeatedGroup() *GoTest_RepeatedGroup { return &GoTest_RepeatedGroup{ RequiredField: String("repeated"), } } func initGoTest(setdefaults bool) *GoTest { pb := new(GoTest) if setdefaults { pb.F_BoolDefaulted = Bool(Default_GoTest_F_BoolDefaulted) pb.F_Int32Defaulted = Int32(Default_GoTest_F_Int32Defaulted) pb.F_Int64Defaulted = Int64(Default_GoTest_F_Int64Defaulted) pb.F_Fixed32Defaulted = Uint32(Default_GoTest_F_Fixed32Defaulted) pb.F_Fixed64Defaulted = Uint64(Default_GoTest_F_Fixed64Defaulted) pb.F_Uint32Defaulted = Uint32(Default_GoTest_F_Uint32Defaulted) pb.F_Uint64Defaulted = Uint64(Default_GoTest_F_Uint64Defaulted) pb.F_FloatDefaulted = Float32(Default_GoTest_F_FloatDefaulted) pb.F_DoubleDefaulted = Float64(Default_GoTest_F_DoubleDefaulted) pb.F_StringDefaulted = String(Default_GoTest_F_StringDefaulted) pb.F_BytesDefaulted = Default_GoTest_F_BytesDefaulted pb.F_Sint32Defaulted = Int32(Default_GoTest_F_Sint32Defaulted) pb.F_Sint64Defaulted = Int64(Default_GoTest_F_Sint64Defaulted) } pb.Kind = GoTest_TIME.Enum() pb.RequiredField = initGoTestField() pb.F_BoolRequired = Bool(true) pb.F_Int32Required = Int32(3) pb.F_Int64Required = Int64(6) pb.F_Fixed32Required = Uint32(32) pb.F_Fixed64Required = Uint64(64) pb.F_Uint32Required = Uint32(3232) pb.F_Uint64Required = Uint64(6464) pb.F_FloatRequired = Float32(3232) pb.F_DoubleRequired = Float64(6464) pb.F_StringRequired = String("string") pb.F_BytesRequired = []byte("bytes") pb.F_Sint32Required = Int32(-32) pb.F_Sint64Required = Int64(-64) pb.Requiredgroup = initGoTest_RequiredGroup() return pb } ================================================ FILE: vendor/github.com/pkg/errors/.gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.test *.prof ================================================ FILE: vendor/github.com/pkg/errors/.travis.yml ================================================ language: go go_import_path: github.com/pkg/errors go: - 1.4.3 - 1.5.4 - 1.6.2 - 1.7.1 - tip script: - go test -v ./... ================================================ FILE: vendor/github.com/pkg/errors/LICENSE ================================================ Copyright (c) 2015, Dave Cheney All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: vendor/github.com/pkg/errors/README.md ================================================ # errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) Package errors provides simple error handling primitives. `go get github.com/pkg/errors` The traditional error handling idiom in Go is roughly akin to ```go if err != nil { return err } ``` which applied recursively up the call stack results in error reports without context or debugging information. The errors package allows programmers to add context to the failure path in their code in a way that does not destroy the original value of the error. ## Adding context to an error The errors.Wrap function returns a new error that adds context to the original error. For example ```go _, err := ioutil.ReadAll(r) if err != nil { return errors.Wrap(err, "read failed") } ``` ## Retrieving the cause of an error Using `errors.Wrap` constructs a stack of errors, adding context to the preceding error. Depending on the nature of the error it may be necessary to reverse the operation of errors.Wrap to retrieve the original error for inspection. Any error value which implements this interface can be inspected by `errors.Cause`. ```go type causer interface { Cause() error } ``` `errors.Cause` will recursively retrieve the topmost error which does not implement `causer`, which is assumed to be the original cause. For example: ```go switch err := errors.Cause(err).(type) { case *MyError: // handle specifically default: // unknown error } ``` [Read the package documentation for more information](https://godoc.org/github.com/pkg/errors). ## Contributing We welcome pull requests, bug fixes and issue reports. With that said, the bar for adding new symbols to this package is intentionally set high. Before proposing a change, please discuss your change by raising an issue. ## Licence BSD-2-Clause ================================================ FILE: vendor/github.com/pkg/errors/appveyor.yml ================================================ version: build-{build}.{branch} clone_folder: C:\gopath\src\github.com\pkg\errors shallow_clone: true # for startup speed environment: GOPATH: C:\gopath platform: - x64 # http://www.appveyor.com/docs/installed-software install: # some helpful output for debugging builds - go version - go env # pre-installed MinGW at C:\MinGW is 32bit only # but MSYS2 at C:\msys64 has mingw64 - set PATH=C:\msys64\mingw64\bin;%PATH% - gcc --version - g++ --version build_script: - go install -v ./... test_script: - set PATH=C:\gopath\bin;%PATH% - go test -v ./... #artifacts: # - path: '%GOPATH%\bin\*.exe' deploy: off ================================================ FILE: vendor/github.com/pkg/errors/bench_test.go ================================================ // +build go1.7 package errors import ( "fmt" "testing" stderrors "errors" ) func noErrors(at, depth int) error { if at >= depth { return stderrors.New("no error") } return noErrors(at+1, depth) } func yesErrors(at, depth int) error { if at >= depth { return New("ye error") } return yesErrors(at+1, depth) } func BenchmarkErrors(b *testing.B) { var toperr error type run struct { stack int std bool } runs := []run{ {10, false}, {10, true}, {100, false}, {100, true}, {1000, false}, {1000, true}, } for _, r := range runs { part := "pkg/errors" if r.std { part = "errors" } name := fmt.Sprintf("%s-stack-%d", part, r.stack) b.Run(name, func(b *testing.B) { var err error f := yesErrors if r.std { f = noErrors } b.ReportAllocs() for i := 0; i < b.N; i++ { err = f(0, r.stack) } b.StopTimer() toperr = err }) } } ================================================ FILE: vendor/github.com/pkg/errors/errors.go ================================================ // Package errors provides simple error handling primitives. // // The traditional error handling idiom in Go is roughly akin to // // if err != nil { // return err // } // // which applied recursively up the call stack results in error reports // without context or debugging information. The errors package allows // programmers to add context to the failure path in their code in a way // that does not destroy the original value of the error. // // Adding context to an error // // The errors.Wrap function returns a new error that adds context to the // original error by recording a stack trace at the point Wrap is called, // and the supplied message. For example // // _, err := ioutil.ReadAll(r) // if err != nil { // return errors.Wrap(err, "read failed") // } // // If additional control is required the errors.WithStack and errors.WithMessage // functions destructure errors.Wrap into its component operations of annotating // an error with a stack trace and an a message, respectively. // // Retrieving the cause of an error // // Using errors.Wrap constructs a stack of errors, adding context to the // preceding error. Depending on the nature of the error it may be necessary // to reverse the operation of errors.Wrap to retrieve the original error // for inspection. Any error value which implements this interface // // type causer interface { // Cause() error // } // // can be inspected by errors.Cause. errors.Cause will recursively retrieve // the topmost error which does not implement causer, which is assumed to be // the original cause. For example: // // switch err := errors.Cause(err).(type) { // case *MyError: // // handle specifically // default: // // unknown error // } // // causer interface is not exported by this package, but is considered a part // of stable public API. // // Formatted printing of errors // // All error values returned from this package implement fmt.Formatter and can // be formatted by the fmt package. The following verbs are supported // // %s print the error. If the error has a Cause it will be // printed recursively // %v see %s // %+v extended format. Each Frame of the error's StackTrace will // be printed in detail. // // Retrieving the stack trace of an error or wrapper // // New, Errorf, Wrap, and Wrapf record a stack trace at the point they are // invoked. This information can be retrieved with the following interface. // // type stackTracer interface { // StackTrace() errors.StackTrace // } // // Where errors.StackTrace is defined as // // type StackTrace []Frame // // The Frame type represents a call site in the stack trace. Frame supports // the fmt.Formatter interface that can be used for printing information about // the stack trace of this error. For example: // // if err, ok := err.(stackTracer); ok { // for _, f := range err.StackTrace() { // fmt.Printf("%+s:%d", f) // } // } // // stackTracer interface is not exported by this package, but is considered a part // of stable public API. // // See the documentation for Frame.Format for more details. package errors import ( "fmt" "io" ) // New returns an error with the supplied message. // New also records the stack trace at the point it was called. func New(message string) error { return &fundamental{ msg: message, stack: callers(), } } // Errorf formats according to a format specifier and returns the string // as a value that satisfies error. // Errorf also records the stack trace at the point it was called. func Errorf(format string, args ...interface{}) error { return &fundamental{ msg: fmt.Sprintf(format, args...), stack: callers(), } } // fundamental is an error that has a message and a stack, but no caller. type fundamental struct { msg string *stack } func (f *fundamental) Error() string { return f.msg } func (f *fundamental) Format(s fmt.State, verb rune) { switch verb { case 'v': if s.Flag('+') { io.WriteString(s, f.msg) f.stack.Format(s, verb) return } fallthrough case 's': io.WriteString(s, f.msg) case 'q': fmt.Fprintf(s, "%q", f.msg) } } // WithStack annotates err with a stack trace at the point WithStack was called. // If err is nil, WithStack returns nil. func WithStack(err error) error { if err == nil { return nil } return &withStack{ err, callers(), } } type withStack struct { error *stack } func (w *withStack) Cause() error { return w.error } func (w *withStack) Format(s fmt.State, verb rune) { switch verb { case 'v': if s.Flag('+') { fmt.Fprintf(s, "%+v", w.Cause()) w.stack.Format(s, verb) return } fallthrough case 's': io.WriteString(s, w.Error()) case 'q': fmt.Fprintf(s, "%q", w.Error()) } } // Wrap returns an error annotating err with a stack trace // at the point Wrap is called, and the supplied message. // If err is nil, Wrap returns nil. func Wrap(err error, message string) error { if err == nil { return nil } err = &withMessage{ cause: err, msg: message, } return &withStack{ err, callers(), } } // Wrapf returns an error annotating err with a stack trace // at the point Wrapf is call, and the format specifier. // If err is nil, Wrapf returns nil. func Wrapf(err error, format string, args ...interface{}) error { if err == nil { return nil } err = &withMessage{ cause: err, msg: fmt.Sprintf(format, args...), } return &withStack{ err, callers(), } } // WithMessage annotates err with a new message. // If err is nil, WithMessage returns nil. func WithMessage(err error, message string) error { if err == nil { return nil } return &withMessage{ cause: err, msg: message, } } type withMessage struct { cause error msg string } func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() } func (w *withMessage) Cause() error { return w.cause } func (w *withMessage) Format(s fmt.State, verb rune) { switch verb { case 'v': if s.Flag('+') { fmt.Fprintf(s, "%+v\n", w.Cause()) io.WriteString(s, w.msg) return } fallthrough case 's', 'q': io.WriteString(s, w.Error()) } } // Cause returns the underlying cause of the error, if possible. // An error value has a cause if it implements the following // interface: // // type causer interface { // Cause() error // } // // If the error does not implement Cause, the original error will // be returned. If the error is nil, nil will be returned without further // investigation. func Cause(err error) error { type causer interface { Cause() error } for err != nil { cause, ok := err.(causer) if !ok { break } err = cause.Cause() } return err } ================================================ FILE: vendor/github.com/pkg/errors/errors_test.go ================================================ package errors import ( "errors" "fmt" "io" "reflect" "testing" ) func TestNew(t *testing.T) { tests := []struct { err string want error }{ {"", fmt.Errorf("")}, {"foo", fmt.Errorf("foo")}, {"foo", New("foo")}, {"string with format specifiers: %v", errors.New("string with format specifiers: %v")}, } for _, tt := range tests { got := New(tt.err) if got.Error() != tt.want.Error() { t.Errorf("New.Error(): got: %q, want %q", got, tt.want) } } } func TestWrapNil(t *testing.T) { got := Wrap(nil, "no error") if got != nil { t.Errorf("Wrap(nil, \"no error\"): got %#v, expected nil", got) } } func TestWrap(t *testing.T) { tests := []struct { err error message string want string }{ {io.EOF, "read error", "read error: EOF"}, {Wrap(io.EOF, "read error"), "client error", "client error: read error: EOF"}, } for _, tt := range tests { got := Wrap(tt.err, tt.message).Error() if got != tt.want { t.Errorf("Wrap(%v, %q): got: %v, want %v", tt.err, tt.message, got, tt.want) } } } type nilError struct{} func (nilError) Error() string { return "nil error" } func TestCause(t *testing.T) { x := New("error") tests := []struct { err error want error }{{ // nil error is nil err: nil, want: nil, }, { // explicit nil error is nil err: (error)(nil), want: nil, }, { // typed nil is nil err: (*nilError)(nil), want: (*nilError)(nil), }, { // uncaused error is unaffected err: io.EOF, want: io.EOF, }, { // caused error returns cause err: Wrap(io.EOF, "ignored"), want: io.EOF, }, { err: x, // return from errors.New want: x, }, { WithMessage(nil, "whoops"), nil, }, { WithMessage(io.EOF, "whoops"), io.EOF, }, { WithStack(nil), nil, }, { WithStack(io.EOF), io.EOF, }} for i, tt := range tests { got := Cause(tt.err) if !reflect.DeepEqual(got, tt.want) { t.Errorf("test %d: got %#v, want %#v", i+1, got, tt.want) } } } func TestWrapfNil(t *testing.T) { got := Wrapf(nil, "no error") if got != nil { t.Errorf("Wrapf(nil, \"no error\"): got %#v, expected nil", got) } } func TestWrapf(t *testing.T) { tests := []struct { err error message string want string }{ {io.EOF, "read error", "read error: EOF"}, {Wrapf(io.EOF, "read error without format specifiers"), "client error", "client error: read error without format specifiers: EOF"}, {Wrapf(io.EOF, "read error with %d format specifier", 1), "client error", "client error: read error with 1 format specifier: EOF"}, } for _, tt := range tests { got := Wrapf(tt.err, tt.message).Error() if got != tt.want { t.Errorf("Wrapf(%v, %q): got: %v, want %v", tt.err, tt.message, got, tt.want) } } } func TestErrorf(t *testing.T) { tests := []struct { err error want string }{ {Errorf("read error without format specifiers"), "read error without format specifiers"}, {Errorf("read error with %d format specifier", 1), "read error with 1 format specifier"}, } for _, tt := range tests { got := tt.err.Error() if got != tt.want { t.Errorf("Errorf(%v): got: %q, want %q", tt.err, got, tt.want) } } } func TestWithStackNil(t *testing.T) { got := WithStack(nil) if got != nil { t.Errorf("WithStack(nil): got %#v, expected nil", got) } } func TestWithStack(t *testing.T) { tests := []struct { err error want string }{ {io.EOF, "EOF"}, {WithStack(io.EOF), "EOF"}, } for _, tt := range tests { got := WithStack(tt.err).Error() if got != tt.want { t.Errorf("WithStack(%v): got: %v, want %v", tt.err, got, tt.want) } } } func TestWithMessageNil(t *testing.T) { got := WithMessage(nil, "no error") if got != nil { t.Errorf("WithMessage(nil, \"no error\"): got %#v, expected nil", got) } } func TestWithMessage(t *testing.T) { tests := []struct { err error message string want string }{ {io.EOF, "read error", "read error: EOF"}, {WithMessage(io.EOF, "read error"), "client error", "client error: read error: EOF"}, } for _, tt := range tests { got := WithMessage(tt.err, tt.message).Error() if got != tt.want { t.Errorf("WithMessage(%v, %q): got: %q, want %q", tt.err, tt.message, got, tt.want) } } } // errors.New, etc values are not expected to be compared by value // but the change in errors#27 made them incomparable. Assert that // various kinds of errors have a functional equality operator, even // if the result of that equality is always false. func TestErrorEquality(t *testing.T) { vals := []error{ nil, io.EOF, errors.New("EOF"), New("EOF"), Errorf("EOF"), Wrap(io.EOF, "EOF"), Wrapf(io.EOF, "EOF%d", 2), WithMessage(nil, "whoops"), WithMessage(io.EOF, "whoops"), WithStack(io.EOF), WithStack(nil), } for i := range vals { for j := range vals { _ = vals[i] == vals[j] // mustn't panic } } } ================================================ FILE: vendor/github.com/pkg/errors/example_test.go ================================================ package errors_test import ( "fmt" "github.com/pkg/errors" ) func ExampleNew() { err := errors.New("whoops") fmt.Println(err) // Output: whoops } func ExampleNew_printf() { err := errors.New("whoops") fmt.Printf("%+v", err) // Example output: // whoops // github.com/pkg/errors_test.ExampleNew_printf // /home/dfc/src/github.com/pkg/errors/example_test.go:17 // testing.runExample // /home/dfc/go/src/testing/example.go:114 // testing.RunExamples // /home/dfc/go/src/testing/example.go:38 // testing.(*M).Run // /home/dfc/go/src/testing/testing.go:744 // main.main // /github.com/pkg/errors/_test/_testmain.go:106 // runtime.main // /home/dfc/go/src/runtime/proc.go:183 // runtime.goexit // /home/dfc/go/src/runtime/asm_amd64.s:2059 } func ExampleWithMessage() { cause := errors.New("whoops") err := errors.WithMessage(cause, "oh noes") fmt.Println(err) // Output: oh noes: whoops } func ExampleWithStack() { cause := errors.New("whoops") err := errors.WithStack(cause) fmt.Println(err) // Output: whoops } func ExampleWithStack_printf() { cause := errors.New("whoops") err := errors.WithStack(cause) fmt.Printf("%+v", err) // Example Output: // whoops // github.com/pkg/errors_test.ExampleWithStack_printf // /home/fabstu/go/src/github.com/pkg/errors/example_test.go:55 // testing.runExample // /usr/lib/go/src/testing/example.go:114 // testing.RunExamples // /usr/lib/go/src/testing/example.go:38 // testing.(*M).Run // /usr/lib/go/src/testing/testing.go:744 // main.main // github.com/pkg/errors/_test/_testmain.go:106 // runtime.main // /usr/lib/go/src/runtime/proc.go:183 // runtime.goexit // /usr/lib/go/src/runtime/asm_amd64.s:2086 // github.com/pkg/errors_test.ExampleWithStack_printf // /home/fabstu/go/src/github.com/pkg/errors/example_test.go:56 // testing.runExample // /usr/lib/go/src/testing/example.go:114 // testing.RunExamples // /usr/lib/go/src/testing/example.go:38 // testing.(*M).Run // /usr/lib/go/src/testing/testing.go:744 // main.main // github.com/pkg/errors/_test/_testmain.go:106 // runtime.main // /usr/lib/go/src/runtime/proc.go:183 // runtime.goexit // /usr/lib/go/src/runtime/asm_amd64.s:2086 } func ExampleWrap() { cause := errors.New("whoops") err := errors.Wrap(cause, "oh noes") fmt.Println(err) // Output: oh noes: whoops } func fn() error { e1 := errors.New("error") e2 := errors.Wrap(e1, "inner") e3 := errors.Wrap(e2, "middle") return errors.Wrap(e3, "outer") } func ExampleCause() { err := fn() fmt.Println(err) fmt.Println(errors.Cause(err)) // Output: outer: middle: inner: error // error } func ExampleWrap_extended() { err := fn() fmt.Printf("%+v\n", err) // Example output: // error // github.com/pkg/errors_test.fn // /home/dfc/src/github.com/pkg/errors/example_test.go:47 // github.com/pkg/errors_test.ExampleCause_printf // /home/dfc/src/github.com/pkg/errors/example_test.go:63 // testing.runExample // /home/dfc/go/src/testing/example.go:114 // testing.RunExamples // /home/dfc/go/src/testing/example.go:38 // testing.(*M).Run // /home/dfc/go/src/testing/testing.go:744 // main.main // /github.com/pkg/errors/_test/_testmain.go:104 // runtime.main // /home/dfc/go/src/runtime/proc.go:183 // runtime.goexit // /home/dfc/go/src/runtime/asm_amd64.s:2059 // github.com/pkg/errors_test.fn // /home/dfc/src/github.com/pkg/errors/example_test.go:48: inner // github.com/pkg/errors_test.fn // /home/dfc/src/github.com/pkg/errors/example_test.go:49: middle // github.com/pkg/errors_test.fn // /home/dfc/src/github.com/pkg/errors/example_test.go:50: outer } func ExampleWrapf() { cause := errors.New("whoops") err := errors.Wrapf(cause, "oh noes #%d", 2) fmt.Println(err) // Output: oh noes #2: whoops } func ExampleErrorf_extended() { err := errors.Errorf("whoops: %s", "foo") fmt.Printf("%+v", err) // Example output: // whoops: foo // github.com/pkg/errors_test.ExampleErrorf // /home/dfc/src/github.com/pkg/errors/example_test.go:101 // testing.runExample // /home/dfc/go/src/testing/example.go:114 // testing.RunExamples // /home/dfc/go/src/testing/example.go:38 // testing.(*M).Run // /home/dfc/go/src/testing/testing.go:744 // main.main // /github.com/pkg/errors/_test/_testmain.go:102 // runtime.main // /home/dfc/go/src/runtime/proc.go:183 // runtime.goexit // /home/dfc/go/src/runtime/asm_amd64.s:2059 } func Example_stackTrace() { type stackTracer interface { StackTrace() errors.StackTrace } err, ok := errors.Cause(fn()).(stackTracer) if !ok { panic("oops, err does not implement stackTracer") } st := err.StackTrace() fmt.Printf("%+v", st[0:2]) // top two frames // Example output: // github.com/pkg/errors_test.fn // /home/dfc/src/github.com/pkg/errors/example_test.go:47 // github.com/pkg/errors_test.Example_stackTrace // /home/dfc/src/github.com/pkg/errors/example_test.go:127 } func ExampleCause_printf() { err := errors.Wrap(func() error { return func() error { return errors.Errorf("hello %s", fmt.Sprintf("world")) }() }(), "failed") fmt.Printf("%v", err) // Output: failed: hello world } ================================================ FILE: vendor/github.com/pkg/errors/format_test.go ================================================ package errors import ( "errors" "fmt" "io" "regexp" "strings" "testing" ) func TestFormatNew(t *testing.T) { tests := []struct { error format string want string }{{ New("error"), "%s", "error", }, { New("error"), "%v", "error", }, { New("error"), "%+v", "error\n" + "github.com/pkg/errors.TestFormatNew\n" + "\t.+/github.com/pkg/errors/format_test.go:26", }, { New("error"), "%q", `"error"`, }} for i, tt := range tests { testFormatRegexp(t, i, tt.error, tt.format, tt.want) } } func TestFormatErrorf(t *testing.T) { tests := []struct { error format string want string }{{ Errorf("%s", "error"), "%s", "error", }, { Errorf("%s", "error"), "%v", "error", }, { Errorf("%s", "error"), "%+v", "error\n" + "github.com/pkg/errors.TestFormatErrorf\n" + "\t.+/github.com/pkg/errors/format_test.go:56", }} for i, tt := range tests { testFormatRegexp(t, i, tt.error, tt.format, tt.want) } } func TestFormatWrap(t *testing.T) { tests := []struct { error format string want string }{{ Wrap(New("error"), "error2"), "%s", "error2: error", }, { Wrap(New("error"), "error2"), "%v", "error2: error", }, { Wrap(New("error"), "error2"), "%+v", "error\n" + "github.com/pkg/errors.TestFormatWrap\n" + "\t.+/github.com/pkg/errors/format_test.go:82", }, { Wrap(io.EOF, "error"), "%s", "error: EOF", }, { Wrap(io.EOF, "error"), "%v", "error: EOF", }, { Wrap(io.EOF, "error"), "%+v", "EOF\n" + "error\n" + "github.com/pkg/errors.TestFormatWrap\n" + "\t.+/github.com/pkg/errors/format_test.go:96", }, { Wrap(Wrap(io.EOF, "error1"), "error2"), "%+v", "EOF\n" + "error1\n" + "github.com/pkg/errors.TestFormatWrap\n" + "\t.+/github.com/pkg/errors/format_test.go:103\n", }, { Wrap(New("error with space"), "context"), "%q", `"context: error with space"`, }} for i, tt := range tests { testFormatRegexp(t, i, tt.error, tt.format, tt.want) } } func TestFormatWrapf(t *testing.T) { tests := []struct { error format string want string }{{ Wrapf(io.EOF, "error%d", 2), "%s", "error2: EOF", }, { Wrapf(io.EOF, "error%d", 2), "%v", "error2: EOF", }, { Wrapf(io.EOF, "error%d", 2), "%+v", "EOF\n" + "error2\n" + "github.com/pkg/errors.TestFormatWrapf\n" + "\t.+/github.com/pkg/errors/format_test.go:134", }, { Wrapf(New("error"), "error%d", 2), "%s", "error2: error", }, { Wrapf(New("error"), "error%d", 2), "%v", "error2: error", }, { Wrapf(New("error"), "error%d", 2), "%+v", "error\n" + "github.com/pkg/errors.TestFormatWrapf\n" + "\t.+/github.com/pkg/errors/format_test.go:149", }} for i, tt := range tests { testFormatRegexp(t, i, tt.error, tt.format, tt.want) } } func TestFormatWithStack(t *testing.T) { tests := []struct { error format string want []string }{{ WithStack(io.EOF), "%s", []string{"EOF"}, }, { WithStack(io.EOF), "%v", []string{"EOF"}, }, { WithStack(io.EOF), "%+v", []string{"EOF", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:175"}, }, { WithStack(New("error")), "%s", []string{"error"}, }, { WithStack(New("error")), "%v", []string{"error"}, }, { WithStack(New("error")), "%+v", []string{"error", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:189", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:189"}, }, { WithStack(WithStack(io.EOF)), "%+v", []string{"EOF", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:197", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:197"}, }, { WithStack(WithStack(Wrapf(io.EOF, "message"))), "%+v", []string{"EOF", "message", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:205", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:205", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:205"}, }, { WithStack(Errorf("error%d", 1)), "%+v", []string{"error1", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:216", "github.com/pkg/errors.TestFormatWithStack\n" + "\t.+/github.com/pkg/errors/format_test.go:216"}, }} for i, tt := range tests { testFormatCompleteCompare(t, i, tt.error, tt.format, tt.want, true) } } func TestFormatWithMessage(t *testing.T) { tests := []struct { error format string want []string }{{ WithMessage(New("error"), "error2"), "%s", []string{"error2: error"}, }, { WithMessage(New("error"), "error2"), "%v", []string{"error2: error"}, }, { WithMessage(New("error"), "error2"), "%+v", []string{ "error", "github.com/pkg/errors.TestFormatWithMessage\n" + "\t.+/github.com/pkg/errors/format_test.go:244", "error2"}, }, { WithMessage(io.EOF, "addition1"), "%s", []string{"addition1: EOF"}, }, { WithMessage(io.EOF, "addition1"), "%v", []string{"addition1: EOF"}, }, { WithMessage(io.EOF, "addition1"), "%+v", []string{"EOF", "addition1"}, }, { WithMessage(WithMessage(io.EOF, "addition1"), "addition2"), "%v", []string{"addition2: addition1: EOF"}, }, { WithMessage(WithMessage(io.EOF, "addition1"), "addition2"), "%+v", []string{"EOF", "addition1", "addition2"}, }, { Wrap(WithMessage(io.EOF, "error1"), "error2"), "%+v", []string{"EOF", "error1", "error2", "github.com/pkg/errors.TestFormatWithMessage\n" + "\t.+/github.com/pkg/errors/format_test.go:272"}, }, { WithMessage(Errorf("error%d", 1), "error2"), "%+v", []string{"error1", "github.com/pkg/errors.TestFormatWithMessage\n" + "\t.+/github.com/pkg/errors/format_test.go:278", "error2"}, }, { WithMessage(WithStack(io.EOF), "error"), "%+v", []string{ "EOF", "github.com/pkg/errors.TestFormatWithMessage\n" + "\t.+/github.com/pkg/errors/format_test.go:285", "error"}, }, { WithMessage(Wrap(WithStack(io.EOF), "inside-error"), "outside-error"), "%+v", []string{ "EOF", "github.com/pkg/errors.TestFormatWithMessage\n" + "\t.+/github.com/pkg/errors/format_test.go:293", "inside-error", "github.com/pkg/errors.TestFormatWithMessage\n" + "\t.+/github.com/pkg/errors/format_test.go:293", "outside-error"}, }} for i, tt := range tests { testFormatCompleteCompare(t, i, tt.error, tt.format, tt.want, true) } } func TestFormatGeneric(t *testing.T) { starts := []struct { err error want []string }{ {New("new-error"), []string{ "new-error", "github.com/pkg/errors.TestFormatGeneric\n" + "\t.+/github.com/pkg/errors/format_test.go:315"}, }, {Errorf("errorf-error"), []string{ "errorf-error", "github.com/pkg/errors.TestFormatGeneric\n" + "\t.+/github.com/pkg/errors/format_test.go:319"}, }, {errors.New("errors-new-error"), []string{ "errors-new-error"}, }, } wrappers := []wrapper{ { func(err error) error { return WithMessage(err, "with-message") }, []string{"with-message"}, }, { func(err error) error { return WithStack(err) }, []string{ "github.com/pkg/errors.(func·002|TestFormatGeneric.func2)\n\t" + ".+/github.com/pkg/errors/format_test.go:333", }, }, { func(err error) error { return Wrap(err, "wrap-error") }, []string{ "wrap-error", "github.com/pkg/errors.(func·003|TestFormatGeneric.func3)\n\t" + ".+/github.com/pkg/errors/format_test.go:339", }, }, { func(err error) error { return Wrapf(err, "wrapf-error%d", 1) }, []string{ "wrapf-error1", "github.com/pkg/errors.(func·004|TestFormatGeneric.func4)\n\t" + ".+/github.com/pkg/errors/format_test.go:346", }, }, } for s := range starts { err := starts[s].err want := starts[s].want testFormatCompleteCompare(t, s, err, "%+v", want, false) testGenericRecursive(t, err, want, wrappers, 3) } } func testFormatRegexp(t *testing.T, n int, arg interface{}, format, want string) { got := fmt.Sprintf(format, arg) gotLines := strings.SplitN(got, "\n", -1) wantLines := strings.SplitN(want, "\n", -1) if len(wantLines) > len(gotLines) { t.Errorf("test %d: wantLines(%d) > gotLines(%d):\n got: %q\nwant: %q", n+1, len(wantLines), len(gotLines), got, want) return } for i, w := range wantLines { match, err := regexp.MatchString(w, gotLines[i]) if err != nil { t.Fatal(err) } if !match { t.Errorf("test %d: line %d: fmt.Sprintf(%q, err):\n got: %q\nwant: %q", n+1, i+1, format, got, want) } } } var stackLineR = regexp.MustCompile(`\.`) // parseBlocks parses input into a slice, where: // - incase entry contains a newline, its a stacktrace // - incase entry contains no newline, its a solo line. // // Detecting stack boundaries only works incase the WithStack-calls are // to be found on the same line, thats why it is optionally here. // // Example use: // // for _, e := range blocks { // if strings.ContainsAny(e, "\n") { // // Match as stack // } else { // // Match as line // } // } // func parseBlocks(input string, detectStackboundaries bool) ([]string, error) { var blocks []string stack := "" wasStack := false lines := map[string]bool{} // already found lines for _, l := range strings.Split(input, "\n") { isStackLine := stackLineR.MatchString(l) switch { case !isStackLine && wasStack: blocks = append(blocks, stack, l) stack = "" lines = map[string]bool{} case isStackLine: if wasStack { // Detecting two stacks after another, possible cause lines match in // our tests due to WithStack(WithStack(io.EOF)) on same line. if detectStackboundaries { if lines[l] { if len(stack) == 0 { return nil, errors.New("len of block must not be zero here") } blocks = append(blocks, stack) stack = l lines = map[string]bool{l: true} continue } } stack = stack + "\n" + l } else { stack = l } lines[l] = true case !isStackLine && !wasStack: blocks = append(blocks, l) default: return nil, errors.New("must not happen") } wasStack = isStackLine } // Use up stack if stack != "" { blocks = append(blocks, stack) } return blocks, nil } func testFormatCompleteCompare(t *testing.T, n int, arg interface{}, format string, want []string, detectStackBoundaries bool) { gotStr := fmt.Sprintf(format, arg) got, err := parseBlocks(gotStr, detectStackBoundaries) if err != nil { t.Fatal(err) } if len(got) != len(want) { t.Fatalf("test %d: fmt.Sprintf(%s, err) -> wrong number of blocks: got(%d) want(%d)\n got: %s\nwant: %s\ngotStr: %q", n+1, format, len(got), len(want), prettyBlocks(got), prettyBlocks(want), gotStr) } for i := range got { if strings.ContainsAny(want[i], "\n") { // Match as stack match, err := regexp.MatchString(want[i], got[i]) if err != nil { t.Fatal(err) } if !match { t.Fatalf("test %d: block %d: fmt.Sprintf(%q, err):\ngot:\n%q\nwant:\n%q\nall-got:\n%s\nall-want:\n%s\n", n+1, i+1, format, got[i], want[i], prettyBlocks(got), prettyBlocks(want)) } } else { // Match as message if got[i] != want[i] { t.Fatalf("test %d: fmt.Sprintf(%s, err) at block %d got != want:\n got: %q\nwant: %q", n+1, format, i+1, got[i], want[i]) } } } } type wrapper struct { wrap func(err error) error want []string } func prettyBlocks(blocks []string, prefix ...string) string { var out []string for _, b := range blocks { out = append(out, fmt.Sprintf("%v", b)) } return " " + strings.Join(out, "\n ") } func testGenericRecursive(t *testing.T, beforeErr error, beforeWant []string, list []wrapper, maxDepth int) { if len(beforeWant) == 0 { panic("beforeWant must not be empty") } for _, w := range list { if len(w.want) == 0 { panic("want must not be empty") } err := w.wrap(beforeErr) // Copy required cause append(beforeWant, ..) modified beforeWant subtly. beforeCopy := make([]string, len(beforeWant)) copy(beforeCopy, beforeWant) beforeWant := beforeCopy last := len(beforeWant) - 1 var want []string // Merge two stacks behind each other. if strings.ContainsAny(beforeWant[last], "\n") && strings.ContainsAny(w.want[0], "\n") { want = append(beforeWant[:last], append([]string{beforeWant[last] + "((?s).*)" + w.want[0]}, w.want[1:]...)...) } else { want = append(beforeWant, w.want...) } testFormatCompleteCompare(t, maxDepth, err, "%+v", want, false) if maxDepth > 0 { testGenericRecursive(t, err, want, list, maxDepth-1) } } } ================================================ FILE: vendor/github.com/pkg/errors/stack.go ================================================ package errors import ( "fmt" "io" "path" "runtime" "strings" ) // Frame represents a program counter inside a stack frame. type Frame uintptr // pc returns the program counter for this frame; // multiple frames may have the same PC value. func (f Frame) pc() uintptr { return uintptr(f) - 1 } // file returns the full path to the file that contains the // function for this Frame's pc. func (f Frame) file() string { fn := runtime.FuncForPC(f.pc()) if fn == nil { return "unknown" } file, _ := fn.FileLine(f.pc()) return file } // line returns the line number of source code of the // function for this Frame's pc. func (f Frame) line() int { fn := runtime.FuncForPC(f.pc()) if fn == nil { return 0 } _, line := fn.FileLine(f.pc()) return line } // Format formats the frame according to the fmt.Formatter interface. // // %s source file // %d source line // %n function name // %v equivalent to %s:%d // // Format accepts flags that alter the printing of some verbs, as follows: // // %+s path of source file relative to the compile time GOPATH // %+v equivalent to %+s:%d func (f Frame) Format(s fmt.State, verb rune) { switch verb { case 's': switch { case s.Flag('+'): pc := f.pc() fn := runtime.FuncForPC(pc) if fn == nil { io.WriteString(s, "unknown") } else { file, _ := fn.FileLine(pc) fmt.Fprintf(s, "%s\n\t%s", fn.Name(), file) } default: io.WriteString(s, path.Base(f.file())) } case 'd': fmt.Fprintf(s, "%d", f.line()) case 'n': name := runtime.FuncForPC(f.pc()).Name() io.WriteString(s, funcname(name)) case 'v': f.Format(s, 's') io.WriteString(s, ":") f.Format(s, 'd') } } // StackTrace is stack of Frames from innermost (newest) to outermost (oldest). type StackTrace []Frame func (st StackTrace) Format(s fmt.State, verb rune) { switch verb { case 'v': switch { case s.Flag('+'): for _, f := range st { fmt.Fprintf(s, "\n%+v", f) } case s.Flag('#'): fmt.Fprintf(s, "%#v", []Frame(st)) default: fmt.Fprintf(s, "%v", []Frame(st)) } case 's': fmt.Fprintf(s, "%s", []Frame(st)) } } // stack represents a stack of program counters. type stack []uintptr func (s *stack) Format(st fmt.State, verb rune) { switch verb { case 'v': switch { case st.Flag('+'): for _, pc := range *s { f := Frame(pc) fmt.Fprintf(st, "\n%+v", f) } } } } func (s *stack) StackTrace() StackTrace { f := make([]Frame, len(*s)) for i := 0; i < len(f); i++ { f[i] = Frame((*s)[i]) } return f } func callers() *stack { const depth = 32 var pcs [depth]uintptr n := runtime.Callers(3, pcs[:]) var st stack = pcs[0:n] return &st } // funcname removes the path prefix component of a function's name reported by func.Name(). func funcname(name string) string { i := strings.LastIndex(name, "/") name = name[i+1:] i = strings.Index(name, ".") return name[i+1:] } func trimGOPATH(name, file string) string { // Here we want to get the source file path relative to the compile time // GOPATH. As of Go 1.6.x there is no direct way to know the compiled // GOPATH at runtime, but we can infer the number of path segments in the // GOPATH. We note that fn.Name() returns the function name qualified by // the import path, which does not include the GOPATH. Thus we can trim // segments from the beginning of the file path until the number of path // separators remaining is one more than the number of path separators in // the function name. For example, given: // // GOPATH /home/user // file /home/user/src/pkg/sub/file.go // fn.Name() pkg/sub.Type.Method // // We want to produce: // // pkg/sub/file.go // // From this we can easily see that fn.Name() has one less path separator // than our desired output. We count separators from the end of the file // path until it finds two more than in the function name and then move // one character forward to preserve the initial path segment without a // leading separator. const sep = "/" goal := strings.Count(name, sep) + 2 i := len(file) for n := 0; n < goal; n++ { i = strings.LastIndex(file[:i], sep) if i == -1 { // not enough separators found, set i so that the slice expression // below leaves file unmodified i = -len(sep) break } } // get back to 0 or trim the leading separator file = file[i+len(sep):] return file } ================================================ FILE: vendor/github.com/pkg/errors/stack_test.go ================================================ package errors import ( "fmt" "runtime" "testing" ) var initpc, _, _, _ = runtime.Caller(0) func TestFrameLine(t *testing.T) { var tests = []struct { Frame want int }{{ Frame(initpc), 9, }, { func() Frame { var pc, _, _, _ = runtime.Caller(0) return Frame(pc) }(), 20, }, { func() Frame { var pc, _, _, _ = runtime.Caller(1) return Frame(pc) }(), 28, }, { Frame(0), // invalid PC 0, }} for _, tt := range tests { got := tt.Frame.line() want := tt.want if want != got { t.Errorf("Frame(%v): want: %v, got: %v", uintptr(tt.Frame), want, got) } } } type X struct{} func (x X) val() Frame { var pc, _, _, _ = runtime.Caller(0) return Frame(pc) } func (x *X) ptr() Frame { var pc, _, _, _ = runtime.Caller(0) return Frame(pc) } func TestFrameFormat(t *testing.T) { var tests = []struct { Frame format string want string }{{ Frame(initpc), "%s", "stack_test.go", }, { Frame(initpc), "%+s", "github.com/pkg/errors.init\n" + "\t.+/github.com/pkg/errors/stack_test.go", }, { Frame(0), "%s", "unknown", }, { Frame(0), "%+s", "unknown", }, { Frame(initpc), "%d", "9", }, { Frame(0), "%d", "0", }, { Frame(initpc), "%n", "init", }, { func() Frame { var x X return x.ptr() }(), "%n", `\(\*X\).ptr`, }, { func() Frame { var x X return x.val() }(), "%n", "X.val", }, { Frame(0), "%n", "", }, { Frame(initpc), "%v", "stack_test.go:9", }, { Frame(initpc), "%+v", "github.com/pkg/errors.init\n" + "\t.+/github.com/pkg/errors/stack_test.go:9", }, { Frame(0), "%v", "unknown:0", }} for i, tt := range tests { testFormatRegexp(t, i, tt.Frame, tt.format, tt.want) } } func TestFuncname(t *testing.T) { tests := []struct { name, want string }{ {"", ""}, {"runtime.main", "main"}, {"github.com/pkg/errors.funcname", "funcname"}, {"funcname", "funcname"}, {"io.copyBuffer", "copyBuffer"}, {"main.(*R).Write", "(*R).Write"}, } for _, tt := range tests { got := funcname(tt.name) want := tt.want if got != want { t.Errorf("funcname(%q): want: %q, got %q", tt.name, want, got) } } } func TestTrimGOPATH(t *testing.T) { var tests = []struct { Frame want string }{{ Frame(initpc), "github.com/pkg/errors/stack_test.go", }} for i, tt := range tests { pc := tt.Frame.pc() fn := runtime.FuncForPC(pc) file, _ := fn.FileLine(pc) got := trimGOPATH(fn.Name(), file) testFormatRegexp(t, i, got, "%s", tt.want) } } func TestStackTrace(t *testing.T) { tests := []struct { err error want []string }{{ New("ooh"), []string{ "github.com/pkg/errors.TestStackTrace\n" + "\t.+/github.com/pkg/errors/stack_test.go:172", }, }, { Wrap(New("ooh"), "ahh"), []string{ "github.com/pkg/errors.TestStackTrace\n" + "\t.+/github.com/pkg/errors/stack_test.go:177", // this is the stack of Wrap, not New }, }, { Cause(Wrap(New("ooh"), "ahh")), []string{ "github.com/pkg/errors.TestStackTrace\n" + "\t.+/github.com/pkg/errors/stack_test.go:182", // this is the stack of New }, }, { func() error { return New("ooh") }(), []string{ `github.com/pkg/errors.(func·009|TestStackTrace.func1)` + "\n\t.+/github.com/pkg/errors/stack_test.go:187", // this is the stack of New "github.com/pkg/errors.TestStackTrace\n" + "\t.+/github.com/pkg/errors/stack_test.go:187", // this is the stack of New's caller }, }, { Cause(func() error { return func() error { return Errorf("hello %s", fmt.Sprintf("world")) }() }()), []string{ `github.com/pkg/errors.(func·010|TestStackTrace.func2.1)` + "\n\t.+/github.com/pkg/errors/stack_test.go:196", // this is the stack of Errorf `github.com/pkg/errors.(func·011|TestStackTrace.func2)` + "\n\t.+/github.com/pkg/errors/stack_test.go:197", // this is the stack of Errorf's caller "github.com/pkg/errors.TestStackTrace\n" + "\t.+/github.com/pkg/errors/stack_test.go:198", // this is the stack of Errorf's caller's caller }, }} for i, tt := range tests { x, ok := tt.err.(interface { StackTrace() StackTrace }) if !ok { t.Errorf("expected %#v to implement StackTrace() StackTrace", tt.err) continue } st := x.StackTrace() for j, want := range tt.want { testFormatRegexp(t, i, st[j], "%+v", want) } } } func stackTrace() StackTrace { const depth = 8 var pcs [depth]uintptr n := runtime.Callers(1, pcs[:]) var st stack = pcs[0:n] return st.StackTrace() } func TestStackTraceFormat(t *testing.T) { tests := []struct { StackTrace format string want string }{{ nil, "%s", `\[\]`, }, { nil, "%v", `\[\]`, }, { nil, "%+v", "", }, { nil, "%#v", `\[\]errors.Frame\(nil\)`, }, { make(StackTrace, 0), "%s", `\[\]`, }, { make(StackTrace, 0), "%v", `\[\]`, }, { make(StackTrace, 0), "%+v", "", }, { make(StackTrace, 0), "%#v", `\[\]errors.Frame{}`, }, { stackTrace()[:2], "%s", `\[stack_test.go stack_test.go\]`, }, { stackTrace()[:2], "%v", `\[stack_test.go:225 stack_test.go:272\]`, }, { stackTrace()[:2], "%+v", "\n" + "github.com/pkg/errors.stackTrace\n" + "\t.+/github.com/pkg/errors/stack_test.go:225\n" + "github.com/pkg/errors.TestStackTraceFormat\n" + "\t.+/github.com/pkg/errors/stack_test.go:276", }, { stackTrace()[:2], "%#v", `\[\]errors.Frame{stack_test.go:225, stack_test.go:284}`, }} for i, tt := range tests { testFormatRegexp(t, i, tt.StackTrace, tt.format, tt.want) } } ================================================ FILE: vendor/github.com/pmezard/go-difflib/.travis.yml ================================================ language: go go: - 1.5 - tip ================================================ FILE: vendor/github.com/pmezard/go-difflib/LICENSE ================================================ Copyright (c) 2013, Patrick Mezard All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. The names of its contributors may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: vendor/github.com/pmezard/go-difflib/README.md ================================================ go-difflib ========== [![Build Status](https://travis-ci.org/pmezard/go-difflib.png?branch=master)](https://travis-ci.org/pmezard/go-difflib) [![GoDoc](https://godoc.org/github.com/pmezard/go-difflib/difflib?status.svg)](https://godoc.org/github.com/pmezard/go-difflib/difflib) Go-difflib is a partial port of python 3 difflib package. Its main goal was to make unified and context diff available in pure Go, mostly for testing purposes. The following class and functions (and related tests) have be ported: * `SequenceMatcher` * `unified_diff()` * `context_diff()` ## Installation ```bash $ go get github.com/pmezard/go-difflib/difflib ``` ### Quick Start Diffs are configured with Unified (or ContextDiff) structures, and can be output to an io.Writer or returned as a string. ```Go diff := UnifiedDiff{ A: difflib.SplitLines("foo\nbar\n"), B: difflib.SplitLines("foo\nbaz\n"), FromFile: "Original", ToFile: "Current", Context: 3, } text, _ := GetUnifiedDiffString(diff) fmt.Printf(text) ``` would output: ``` --- Original +++ Current @@ -1,3 +1,3 @@ foo -bar +baz ``` ================================================ FILE: vendor/github.com/pmezard/go-difflib/difflib/difflib.go ================================================ // Package difflib is a partial port of Python difflib module. // // It provides tools to compare sequences of strings and generate textual diffs. // // The following class and functions have been ported: // // - SequenceMatcher // // - unified_diff // // - context_diff // // Getting unified diffs was the main goal of the port. Keep in mind this code // is mostly suitable to output text differences in a human friendly way, there // are no guarantees generated diffs are consumable by patch(1). package difflib import ( "bufio" "bytes" "fmt" "io" "strings" ) func min(a, b int) int { if a < b { return a } return b } func max(a, b int) int { if a > b { return a } return b } func calculateRatio(matches, length int) float64 { if length > 0 { return 2.0 * float64(matches) / float64(length) } return 1.0 } type Match struct { A int B int Size int } type OpCode struct { Tag byte I1 int I2 int J1 int J2 int } // SequenceMatcher compares sequence of strings. The basic // algorithm predates, and is a little fancier than, an algorithm // published in the late 1980's by Ratcliff and Obershelp under the // hyperbolic name "gestalt pattern matching". The basic idea is to find // the longest contiguous matching subsequence that contains no "junk" // elements (R-O doesn't address junk). The same idea is then applied // recursively to the pieces of the sequences to the left and to the right // of the matching subsequence. This does not yield minimal edit // sequences, but does tend to yield matches that "look right" to people. // // SequenceMatcher tries to compute a "human-friendly diff" between two // sequences. Unlike e.g. UNIX(tm) diff, the fundamental notion is the // longest *contiguous* & junk-free matching subsequence. That's what // catches peoples' eyes. The Windows(tm) windiff has another interesting // notion, pairing up elements that appear uniquely in each sequence. // That, and the method here, appear to yield more intuitive difference // reports than does diff. This method appears to be the least vulnerable // to synching up on blocks of "junk lines", though (like blank lines in // ordinary text files, or maybe "

" lines in HTML files). That may be // because this is the only method of the 3 that has a *concept* of // "junk" . // // Timing: Basic R-O is cubic time worst case and quadratic time expected // case. SequenceMatcher is quadratic time for the worst case and has // expected-case behavior dependent in a complicated way on how many // elements the sequences have in common; best case time is linear. type SequenceMatcher struct { a []string b []string b2j map[string][]int IsJunk func(string) bool autoJunk bool bJunk map[string]struct{} matchingBlocks []Match fullBCount map[string]int bPopular map[string]struct{} opCodes []OpCode } func NewMatcher(a, b []string) *SequenceMatcher { m := SequenceMatcher{autoJunk: true} m.SetSeqs(a, b) return &m } func NewMatcherWithJunk(a, b []string, autoJunk bool, isJunk func(string) bool) *SequenceMatcher { m := SequenceMatcher{IsJunk: isJunk, autoJunk: autoJunk} m.SetSeqs(a, b) return &m } // Set two sequences to be compared. func (m *SequenceMatcher) SetSeqs(a, b []string) { m.SetSeq1(a) m.SetSeq2(b) } // Set the first sequence to be compared. The second sequence to be compared is // not changed. // // SequenceMatcher computes and caches detailed information about the second // sequence, so if you want to compare one sequence S against many sequences, // use .SetSeq2(s) once and call .SetSeq1(x) repeatedly for each of the other // sequences. // // See also SetSeqs() and SetSeq2(). func (m *SequenceMatcher) SetSeq1(a []string) { if &a == &m.a { return } m.a = a m.matchingBlocks = nil m.opCodes = nil } // Set the second sequence to be compared. The first sequence to be compared is // not changed. func (m *SequenceMatcher) SetSeq2(b []string) { if &b == &m.b { return } m.b = b m.matchingBlocks = nil m.opCodes = nil m.fullBCount = nil m.chainB() } func (m *SequenceMatcher) chainB() { // Populate line -> index mapping b2j := map[string][]int{} for i, s := range m.b { indices := b2j[s] indices = append(indices, i) b2j[s] = indices } // Purge junk elements m.bJunk = map[string]struct{}{} if m.IsJunk != nil { junk := m.bJunk for s, _ := range b2j { if m.IsJunk(s) { junk[s] = struct{}{} } } for s, _ := range junk { delete(b2j, s) } } // Purge remaining popular elements popular := map[string]struct{}{} n := len(m.b) if m.autoJunk && n >= 200 { ntest := n/100 + 1 for s, indices := range b2j { if len(indices) > ntest { popular[s] = struct{}{} } } for s, _ := range popular { delete(b2j, s) } } m.bPopular = popular m.b2j = b2j } func (m *SequenceMatcher) isBJunk(s string) bool { _, ok := m.bJunk[s] return ok } // Find longest matching block in a[alo:ahi] and b[blo:bhi]. // // If IsJunk is not defined: // // Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where // alo <= i <= i+k <= ahi // blo <= j <= j+k <= bhi // and for all (i',j',k') meeting those conditions, // k >= k' // i <= i' // and if i == i', j <= j' // // In other words, of all maximal matching blocks, return one that // starts earliest in a, and of all those maximal matching blocks that // start earliest in a, return the one that starts earliest in b. // // If IsJunk is defined, first the longest matching block is // determined as above, but with the additional restriction that no // junk element appears in the block. Then that block is extended as // far as possible by matching (only) junk elements on both sides. So // the resulting block never matches on junk except as identical junk // happens to be adjacent to an "interesting" match. // // If no blocks match, return (alo, blo, 0). func (m *SequenceMatcher) findLongestMatch(alo, ahi, blo, bhi int) Match { // CAUTION: stripping common prefix or suffix would be incorrect. // E.g., // ab // acab // Longest matching block is "ab", but if common prefix is // stripped, it's "a" (tied with "b"). UNIX(tm) diff does so // strip, so ends up claiming that ab is changed to acab by // inserting "ca" in the middle. That's minimal but unintuitive: // "it's obvious" that someone inserted "ac" at the front. // Windiff ends up at the same place as diff, but by pairing up // the unique 'b's and then matching the first two 'a's. besti, bestj, bestsize := alo, blo, 0 // find longest junk-free match // during an iteration of the loop, j2len[j] = length of longest // junk-free match ending with a[i-1] and b[j] j2len := map[int]int{} for i := alo; i != ahi; i++ { // look at all instances of a[i] in b; note that because // b2j has no junk keys, the loop is skipped if a[i] is junk newj2len := map[int]int{} for _, j := range m.b2j[m.a[i]] { // a[i] matches b[j] if j < blo { continue } if j >= bhi { break } k := j2len[j-1] + 1 newj2len[j] = k if k > bestsize { besti, bestj, bestsize = i-k+1, j-k+1, k } } j2len = newj2len } // Extend the best by non-junk elements on each end. In particular, // "popular" non-junk elements aren't in b2j, which greatly speeds // the inner loop above, but also means "the best" match so far // doesn't contain any junk *or* popular non-junk elements. for besti > alo && bestj > blo && !m.isBJunk(m.b[bestj-1]) && m.a[besti-1] == m.b[bestj-1] { besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 } for besti+bestsize < ahi && bestj+bestsize < bhi && !m.isBJunk(m.b[bestj+bestsize]) && m.a[besti+bestsize] == m.b[bestj+bestsize] { bestsize += 1 } // Now that we have a wholly interesting match (albeit possibly // empty!), we may as well suck up the matching junk on each // side of it too. Can't think of a good reason not to, and it // saves post-processing the (possibly considerable) expense of // figuring out what to do with it. In the case of an empty // interesting match, this is clearly the right thing to do, // because no other kind of match is possible in the regions. for besti > alo && bestj > blo && m.isBJunk(m.b[bestj-1]) && m.a[besti-1] == m.b[bestj-1] { besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 } for besti+bestsize < ahi && bestj+bestsize < bhi && m.isBJunk(m.b[bestj+bestsize]) && m.a[besti+bestsize] == m.b[bestj+bestsize] { bestsize += 1 } return Match{A: besti, B: bestj, Size: bestsize} } // Return list of triples describing matching subsequences. // // Each triple is of the form (i, j, n), and means that // a[i:i+n] == b[j:j+n]. The triples are monotonically increasing in // i and in j. It's also guaranteed that if (i, j, n) and (i', j', n') are // adjacent triples in the list, and the second is not the last triple in the // list, then i+n != i' or j+n != j'. IOW, adjacent triples never describe // adjacent equal blocks. // // The last triple is a dummy, (len(a), len(b), 0), and is the only // triple with n==0. func (m *SequenceMatcher) GetMatchingBlocks() []Match { if m.matchingBlocks != nil { return m.matchingBlocks } var matchBlocks func(alo, ahi, blo, bhi int, matched []Match) []Match matchBlocks = func(alo, ahi, blo, bhi int, matched []Match) []Match { match := m.findLongestMatch(alo, ahi, blo, bhi) i, j, k := match.A, match.B, match.Size if match.Size > 0 { if alo < i && blo < j { matched = matchBlocks(alo, i, blo, j, matched) } matched = append(matched, match) if i+k < ahi && j+k < bhi { matched = matchBlocks(i+k, ahi, j+k, bhi, matched) } } return matched } matched := matchBlocks(0, len(m.a), 0, len(m.b), nil) // It's possible that we have adjacent equal blocks in the // matching_blocks list now. nonAdjacent := []Match{} i1, j1, k1 := 0, 0, 0 for _, b := range matched { // Is this block adjacent to i1, j1, k1? i2, j2, k2 := b.A, b.B, b.Size if i1+k1 == i2 && j1+k1 == j2 { // Yes, so collapse them -- this just increases the length of // the first block by the length of the second, and the first // block so lengthened remains the block to compare against. k1 += k2 } else { // Not adjacent. Remember the first block (k1==0 means it's // the dummy we started with), and make the second block the // new block to compare against. if k1 > 0 { nonAdjacent = append(nonAdjacent, Match{i1, j1, k1}) } i1, j1, k1 = i2, j2, k2 } } if k1 > 0 { nonAdjacent = append(nonAdjacent, Match{i1, j1, k1}) } nonAdjacent = append(nonAdjacent, Match{len(m.a), len(m.b), 0}) m.matchingBlocks = nonAdjacent return m.matchingBlocks } // Return list of 5-tuples describing how to turn a into b. // // Each tuple is of the form (tag, i1, i2, j1, j2). The first tuple // has i1 == j1 == 0, and remaining tuples have i1 == the i2 from the // tuple preceding it, and likewise for j1 == the previous j2. // // The tags are characters, with these meanings: // // 'r' (replace): a[i1:i2] should be replaced by b[j1:j2] // // 'd' (delete): a[i1:i2] should be deleted, j1==j2 in this case. // // 'i' (insert): b[j1:j2] should be inserted at a[i1:i1], i1==i2 in this case. // // 'e' (equal): a[i1:i2] == b[j1:j2] func (m *SequenceMatcher) GetOpCodes() []OpCode { if m.opCodes != nil { return m.opCodes } i, j := 0, 0 matching := m.GetMatchingBlocks() opCodes := make([]OpCode, 0, len(matching)) for _, m := range matching { // invariant: we've pumped out correct diffs to change // a[:i] into b[:j], and the next matching block is // a[ai:ai+size] == b[bj:bj+size]. So we need to pump // out a diff to change a[i:ai] into b[j:bj], pump out // the matching block, and move (i,j) beyond the match ai, bj, size := m.A, m.B, m.Size tag := byte(0) if i < ai && j < bj { tag = 'r' } else if i < ai { tag = 'd' } else if j < bj { tag = 'i' } if tag > 0 { opCodes = append(opCodes, OpCode{tag, i, ai, j, bj}) } i, j = ai+size, bj+size // the list of matching blocks is terminated by a // sentinel with size 0 if size > 0 { opCodes = append(opCodes, OpCode{'e', ai, i, bj, j}) } } m.opCodes = opCodes return m.opCodes } // Isolate change clusters by eliminating ranges with no changes. // // Return a generator of groups with up to n lines of context. // Each group is in the same format as returned by GetOpCodes(). func (m *SequenceMatcher) GetGroupedOpCodes(n int) [][]OpCode { if n < 0 { n = 3 } codes := m.GetOpCodes() if len(codes) == 0 { codes = []OpCode{OpCode{'e', 0, 1, 0, 1}} } // Fixup leading and trailing groups if they show no changes. if codes[0].Tag == 'e' { c := codes[0] i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 codes[0] = OpCode{c.Tag, max(i1, i2-n), i2, max(j1, j2-n), j2} } if codes[len(codes)-1].Tag == 'e' { c := codes[len(codes)-1] i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 codes[len(codes)-1] = OpCode{c.Tag, i1, min(i2, i1+n), j1, min(j2, j1+n)} } nn := n + n groups := [][]OpCode{} group := []OpCode{} for _, c := range codes { i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 // End the current group and start a new one whenever // there is a large range with no changes. if c.Tag == 'e' && i2-i1 > nn { group = append(group, OpCode{c.Tag, i1, min(i2, i1+n), j1, min(j2, j1+n)}) groups = append(groups, group) group = []OpCode{} i1, j1 = max(i1, i2-n), max(j1, j2-n) } group = append(group, OpCode{c.Tag, i1, i2, j1, j2}) } if len(group) > 0 && !(len(group) == 1 && group[0].Tag == 'e') { groups = append(groups, group) } return groups } // Return a measure of the sequences' similarity (float in [0,1]). // // Where T is the total number of elements in both sequences, and // M is the number of matches, this is 2.0*M / T. // Note that this is 1 if the sequences are identical, and 0 if // they have nothing in common. // // .Ratio() is expensive to compute if you haven't already computed // .GetMatchingBlocks() or .GetOpCodes(), in which case you may // want to try .QuickRatio() or .RealQuickRation() first to get an // upper bound. func (m *SequenceMatcher) Ratio() float64 { matches := 0 for _, m := range m.GetMatchingBlocks() { matches += m.Size } return calculateRatio(matches, len(m.a)+len(m.b)) } // Return an upper bound on ratio() relatively quickly. // // This isn't defined beyond that it is an upper bound on .Ratio(), and // is faster to compute. func (m *SequenceMatcher) QuickRatio() float64 { // viewing a and b as multisets, set matches to the cardinality // of their intersection; this counts the number of matches // without regard to order, so is clearly an upper bound if m.fullBCount == nil { m.fullBCount = map[string]int{} for _, s := range m.b { m.fullBCount[s] = m.fullBCount[s] + 1 } } // avail[x] is the number of times x appears in 'b' less the // number of times we've seen it in 'a' so far ... kinda avail := map[string]int{} matches := 0 for _, s := range m.a { n, ok := avail[s] if !ok { n = m.fullBCount[s] } avail[s] = n - 1 if n > 0 { matches += 1 } } return calculateRatio(matches, len(m.a)+len(m.b)) } // Return an upper bound on ratio() very quickly. // // This isn't defined beyond that it is an upper bound on .Ratio(), and // is faster to compute than either .Ratio() or .QuickRatio(). func (m *SequenceMatcher) RealQuickRatio() float64 { la, lb := len(m.a), len(m.b) return calculateRatio(min(la, lb), la+lb) } // Convert range to the "ed" format func formatRangeUnified(start, stop int) string { // Per the diff spec at http://www.unix.org/single_unix_specification/ beginning := start + 1 // lines start numbering with one length := stop - start if length == 1 { return fmt.Sprintf("%d", beginning) } if length == 0 { beginning -= 1 // empty ranges begin at line just before the range } return fmt.Sprintf("%d,%d", beginning, length) } // Unified diff parameters type UnifiedDiff struct { A []string // First sequence lines FromFile string // First file name FromDate string // First file time B []string // Second sequence lines ToFile string // Second file name ToDate string // Second file time Eol string // Headers end of line, defaults to LF Context int // Number of context lines } // Compare two sequences of lines; generate the delta as a unified diff. // // Unified diffs are a compact way of showing line changes and a few // lines of context. The number of context lines is set by 'n' which // defaults to three. // // By default, the diff control lines (those with ---, +++, or @@) are // created with a trailing newline. This is helpful so that inputs // created from file.readlines() result in diffs that are suitable for // file.writelines() since both the inputs and outputs have trailing // newlines. // // For inputs that do not have trailing newlines, set the lineterm // argument to "" so that the output will be uniformly newline free. // // The unidiff format normally has a header for filenames and modification // times. Any or all of these may be specified using strings for // 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. // The modification times are normally expressed in the ISO 8601 format. func WriteUnifiedDiff(writer io.Writer, diff UnifiedDiff) error { buf := bufio.NewWriter(writer) defer buf.Flush() wf := func(format string, args ...interface{}) error { _, err := buf.WriteString(fmt.Sprintf(format, args...)) return err } ws := func(s string) error { _, err := buf.WriteString(s) return err } if len(diff.Eol) == 0 { diff.Eol = "\n" } started := false m := NewMatcher(diff.A, diff.B) for _, g := range m.GetGroupedOpCodes(diff.Context) { if !started { started = true fromDate := "" if len(diff.FromDate) > 0 { fromDate = "\t" + diff.FromDate } toDate := "" if len(diff.ToDate) > 0 { toDate = "\t" + diff.ToDate } if diff.FromFile != "" || diff.ToFile != "" { err := wf("--- %s%s%s", diff.FromFile, fromDate, diff.Eol) if err != nil { return err } err = wf("+++ %s%s%s", diff.ToFile, toDate, diff.Eol) if err != nil { return err } } } first, last := g[0], g[len(g)-1] range1 := formatRangeUnified(first.I1, last.I2) range2 := formatRangeUnified(first.J1, last.J2) if err := wf("@@ -%s +%s @@%s", range1, range2, diff.Eol); err != nil { return err } for _, c := range g { i1, i2, j1, j2 := c.I1, c.I2, c.J1, c.J2 if c.Tag == 'e' { for _, line := range diff.A[i1:i2] { if err := ws(" " + line); err != nil { return err } } continue } if c.Tag == 'r' || c.Tag == 'd' { for _, line := range diff.A[i1:i2] { if err := ws("-" + line); err != nil { return err } } } if c.Tag == 'r' || c.Tag == 'i' { for _, line := range diff.B[j1:j2] { if err := ws("+" + line); err != nil { return err } } } } } return nil } // Like WriteUnifiedDiff but returns the diff a string. func GetUnifiedDiffString(diff UnifiedDiff) (string, error) { w := &bytes.Buffer{} err := WriteUnifiedDiff(w, diff) return string(w.Bytes()), err } // Convert range to the "ed" format. func formatRangeContext(start, stop int) string { // Per the diff spec at http://www.unix.org/single_unix_specification/ beginning := start + 1 // lines start numbering with one length := stop - start if length == 0 { beginning -= 1 // empty ranges begin at line just before the range } if length <= 1 { return fmt.Sprintf("%d", beginning) } return fmt.Sprintf("%d,%d", beginning, beginning+length-1) } type ContextDiff UnifiedDiff // Compare two sequences of lines; generate the delta as a context diff. // // Context diffs are a compact way of showing line changes and a few // lines of context. The number of context lines is set by diff.Context // which defaults to three. // // By default, the diff control lines (those with *** or ---) are // created with a trailing newline. // // For inputs that do not have trailing newlines, set the diff.Eol // argument to "" so that the output will be uniformly newline free. // // The context diff format normally has a header for filenames and // modification times. Any or all of these may be specified using // strings for diff.FromFile, diff.ToFile, diff.FromDate, diff.ToDate. // The modification times are normally expressed in the ISO 8601 format. // If not specified, the strings default to blanks. func WriteContextDiff(writer io.Writer, diff ContextDiff) error { buf := bufio.NewWriter(writer) defer buf.Flush() var diffErr error wf := func(format string, args ...interface{}) { _, err := buf.WriteString(fmt.Sprintf(format, args...)) if diffErr == nil && err != nil { diffErr = err } } ws := func(s string) { _, err := buf.WriteString(s) if diffErr == nil && err != nil { diffErr = err } } if len(diff.Eol) == 0 { diff.Eol = "\n" } prefix := map[byte]string{ 'i': "+ ", 'd': "- ", 'r': "! ", 'e': " ", } started := false m := NewMatcher(diff.A, diff.B) for _, g := range m.GetGroupedOpCodes(diff.Context) { if !started { started = true fromDate := "" if len(diff.FromDate) > 0 { fromDate = "\t" + diff.FromDate } toDate := "" if len(diff.ToDate) > 0 { toDate = "\t" + diff.ToDate } if diff.FromFile != "" || diff.ToFile != "" { wf("*** %s%s%s", diff.FromFile, fromDate, diff.Eol) wf("--- %s%s%s", diff.ToFile, toDate, diff.Eol) } } first, last := g[0], g[len(g)-1] ws("***************" + diff.Eol) range1 := formatRangeContext(first.I1, last.I2) wf("*** %s ****%s", range1, diff.Eol) for _, c := range g { if c.Tag == 'r' || c.Tag == 'd' { for _, cc := range g { if cc.Tag == 'i' { continue } for _, line := range diff.A[cc.I1:cc.I2] { ws(prefix[cc.Tag] + line) } } break } } range2 := formatRangeContext(first.J1, last.J2) wf("--- %s ----%s", range2, diff.Eol) for _, c := range g { if c.Tag == 'r' || c.Tag == 'i' { for _, cc := range g { if cc.Tag == 'd' { continue } for _, line := range diff.B[cc.J1:cc.J2] { ws(prefix[cc.Tag] + line) } } break } } } return diffErr } // Like WriteContextDiff but returns the diff a string. func GetContextDiffString(diff ContextDiff) (string, error) { w := &bytes.Buffer{} err := WriteContextDiff(w, diff) return string(w.Bytes()), err } // Split a string on "\n" while preserving them. The output can be used // as input for UnifiedDiff and ContextDiff structures. func SplitLines(s string) []string { lines := strings.SplitAfter(s, "\n") lines[len(lines)-1] += "\n" return lines } ================================================ FILE: vendor/github.com/pmezard/go-difflib/difflib/difflib_test.go ================================================ package difflib import ( "bytes" "fmt" "math" "reflect" "strings" "testing" ) func assertAlmostEqual(t *testing.T, a, b float64, places int) { if math.Abs(a-b) > math.Pow10(-places) { t.Errorf("%.7f != %.7f", a, b) } } func assertEqual(t *testing.T, a, b interface{}) { if !reflect.DeepEqual(a, b) { t.Errorf("%v != %v", a, b) } } func splitChars(s string) []string { chars := make([]string, 0, len(s)) // Assume ASCII inputs for i := 0; i != len(s); i++ { chars = append(chars, string(s[i])) } return chars } func TestSequenceMatcherRatio(t *testing.T) { s := NewMatcher(splitChars("abcd"), splitChars("bcde")) assertEqual(t, s.Ratio(), 0.75) assertEqual(t, s.QuickRatio(), 0.75) assertEqual(t, s.RealQuickRatio(), 1.0) } func TestGetOptCodes(t *testing.T) { a := "qabxcd" b := "abycdf" s := NewMatcher(splitChars(a), splitChars(b)) w := &bytes.Buffer{} for _, op := range s.GetOpCodes() { fmt.Fprintf(w, "%s a[%d:%d], (%s) b[%d:%d] (%s)\n", string(op.Tag), op.I1, op.I2, a[op.I1:op.I2], op.J1, op.J2, b[op.J1:op.J2]) } result := string(w.Bytes()) expected := `d a[0:1], (q) b[0:0] () e a[1:3], (ab) b[0:2] (ab) r a[3:4], (x) b[2:3] (y) e a[4:6], (cd) b[3:5] (cd) i a[6:6], () b[5:6] (f) ` if expected != result { t.Errorf("unexpected op codes: \n%s", result) } } func TestGroupedOpCodes(t *testing.T) { a := []string{} for i := 0; i != 39; i++ { a = append(a, fmt.Sprintf("%02d", i)) } b := []string{} b = append(b, a[:8]...) b = append(b, " i") b = append(b, a[8:19]...) b = append(b, " x") b = append(b, a[20:22]...) b = append(b, a[27:34]...) b = append(b, " y") b = append(b, a[35:]...) s := NewMatcher(a, b) w := &bytes.Buffer{} for _, g := range s.GetGroupedOpCodes(-1) { fmt.Fprintf(w, "group\n") for _, op := range g { fmt.Fprintf(w, " %s, %d, %d, %d, %d\n", string(op.Tag), op.I1, op.I2, op.J1, op.J2) } } result := string(w.Bytes()) expected := `group e, 5, 8, 5, 8 i, 8, 8, 8, 9 e, 8, 11, 9, 12 group e, 16, 19, 17, 20 r, 19, 20, 20, 21 e, 20, 22, 21, 23 d, 22, 27, 23, 23 e, 27, 30, 23, 26 group e, 31, 34, 27, 30 r, 34, 35, 30, 31 e, 35, 38, 31, 34 ` if expected != result { t.Errorf("unexpected op codes: \n%s", result) } } func ExampleGetUnifiedDiffCode() { a := `one two three four fmt.Printf("%s,%T",a,b)` b := `zero one three four` diff := UnifiedDiff{ A: SplitLines(a), B: SplitLines(b), FromFile: "Original", FromDate: "2005-01-26 23:30:50", ToFile: "Current", ToDate: "2010-04-02 10:20:52", Context: 3, } result, _ := GetUnifiedDiffString(diff) fmt.Println(strings.Replace(result, "\t", " ", -1)) // Output: // --- Original 2005-01-26 23:30:50 // +++ Current 2010-04-02 10:20:52 // @@ -1,5 +1,4 @@ // +zero // one // -two // three // four // -fmt.Printf("%s,%T",a,b) } func ExampleGetContextDiffCode() { a := `one two three four fmt.Printf("%s,%T",a,b)` b := `zero one tree four` diff := ContextDiff{ A: SplitLines(a), B: SplitLines(b), FromFile: "Original", ToFile: "Current", Context: 3, Eol: "\n", } result, _ := GetContextDiffString(diff) fmt.Print(strings.Replace(result, "\t", " ", -1)) // Output: // *** Original // --- Current // *************** // *** 1,5 **** // one // ! two // ! three // four // - fmt.Printf("%s,%T",a,b) // --- 1,4 ---- // + zero // one // ! tree // four } func ExampleGetContextDiffString() { a := `one two three four` b := `zero one tree four` diff := ContextDiff{ A: SplitLines(a), B: SplitLines(b), FromFile: "Original", ToFile: "Current", Context: 3, Eol: "\n", } result, _ := GetContextDiffString(diff) fmt.Printf(strings.Replace(result, "\t", " ", -1)) // Output: // *** Original // --- Current // *************** // *** 1,4 **** // one // ! two // ! three // four // --- 1,4 ---- // + zero // one // ! tree // four } func rep(s string, count int) string { return strings.Repeat(s, count) } func TestWithAsciiOneInsert(t *testing.T) { sm := NewMatcher(splitChars(rep("b", 100)), splitChars("a"+rep("b", 100))) assertAlmostEqual(t, sm.Ratio(), 0.995, 3) assertEqual(t, sm.GetOpCodes(), []OpCode{{'i', 0, 0, 0, 1}, {'e', 0, 100, 1, 101}}) assertEqual(t, len(sm.bPopular), 0) sm = NewMatcher(splitChars(rep("b", 100)), splitChars(rep("b", 50)+"a"+rep("b", 50))) assertAlmostEqual(t, sm.Ratio(), 0.995, 3) assertEqual(t, sm.GetOpCodes(), []OpCode{{'e', 0, 50, 0, 50}, {'i', 50, 50, 50, 51}, {'e', 50, 100, 51, 101}}) assertEqual(t, len(sm.bPopular), 0) } func TestWithAsciiOnDelete(t *testing.T) { sm := NewMatcher(splitChars(rep("a", 40)+"c"+rep("b", 40)), splitChars(rep("a", 40)+rep("b", 40))) assertAlmostEqual(t, sm.Ratio(), 0.994, 3) assertEqual(t, sm.GetOpCodes(), []OpCode{{'e', 0, 40, 0, 40}, {'d', 40, 41, 40, 40}, {'e', 41, 81, 40, 80}}) } func TestWithAsciiBJunk(t *testing.T) { isJunk := func(s string) bool { return s == " " } sm := NewMatcherWithJunk(splitChars(rep("a", 40)+rep("b", 40)), splitChars(rep("a", 44)+rep("b", 40)), true, isJunk) assertEqual(t, sm.bJunk, map[string]struct{}{}) sm = NewMatcherWithJunk(splitChars(rep("a", 40)+rep("b", 40)), splitChars(rep("a", 44)+rep("b", 40)+rep(" ", 20)), false, isJunk) assertEqual(t, sm.bJunk, map[string]struct{}{" ": struct{}{}}) isJunk = func(s string) bool { return s == " " || s == "b" } sm = NewMatcherWithJunk(splitChars(rep("a", 40)+rep("b", 40)), splitChars(rep("a", 44)+rep("b", 40)+rep(" ", 20)), false, isJunk) assertEqual(t, sm.bJunk, map[string]struct{}{" ": struct{}{}, "b": struct{}{}}) } func TestSFBugsRatioForNullSeqn(t *testing.T) { sm := NewMatcher(nil, nil) assertEqual(t, sm.Ratio(), 1.0) assertEqual(t, sm.QuickRatio(), 1.0) assertEqual(t, sm.RealQuickRatio(), 1.0) } func TestSFBugsComparingEmptyLists(t *testing.T) { groups := NewMatcher(nil, nil).GetGroupedOpCodes(-1) assertEqual(t, len(groups), 0) diff := UnifiedDiff{ FromFile: "Original", ToFile: "Current", Context: 3, } result, err := GetUnifiedDiffString(diff) assertEqual(t, err, nil) assertEqual(t, result, "") } func TestOutputFormatRangeFormatUnified(t *testing.T) { // Per the diff spec at http://www.unix.org/single_unix_specification/ // // Each field shall be of the form: // %1d", if the range contains exactly one line, // and: // "%1d,%1d", , otherwise. // If a range is empty, its beginning line number shall be the number of // the line just before the range, or 0 if the empty range starts the file. fm := formatRangeUnified assertEqual(t, fm(3, 3), "3,0") assertEqual(t, fm(3, 4), "4") assertEqual(t, fm(3, 5), "4,2") assertEqual(t, fm(3, 6), "4,3") assertEqual(t, fm(0, 0), "0,0") } func TestOutputFormatRangeFormatContext(t *testing.T) { // Per the diff spec at http://www.unix.org/single_unix_specification/ // // The range of lines in file1 shall be written in the following format // if the range contains two or more lines: // "*** %d,%d ****\n", , // and the following format otherwise: // "*** %d ****\n", // The ending line number of an empty range shall be the number of the preceding line, // or 0 if the range is at the start of the file. // // Next, the range of lines in file2 shall be written in the following format // if the range contains two or more lines: // "--- %d,%d ----\n", , // and the following format otherwise: // "--- %d ----\n", fm := formatRangeContext assertEqual(t, fm(3, 3), "3") assertEqual(t, fm(3, 4), "4") assertEqual(t, fm(3, 5), "4,5") assertEqual(t, fm(3, 6), "4,6") assertEqual(t, fm(0, 0), "0") } func TestOutputFormatTabDelimiter(t *testing.T) { diff := UnifiedDiff{ A: splitChars("one"), B: splitChars("two"), FromFile: "Original", FromDate: "2005-01-26 23:30:50", ToFile: "Current", ToDate: "2010-04-12 10:20:52", Eol: "\n", } ud, err := GetUnifiedDiffString(diff) assertEqual(t, err, nil) assertEqual(t, SplitLines(ud)[:2], []string{ "--- Original\t2005-01-26 23:30:50\n", "+++ Current\t2010-04-12 10:20:52\n", }) cd, err := GetContextDiffString(ContextDiff(diff)) assertEqual(t, err, nil) assertEqual(t, SplitLines(cd)[:2], []string{ "*** Original\t2005-01-26 23:30:50\n", "--- Current\t2010-04-12 10:20:52\n", }) } func TestOutputFormatNoTrailingTabOnEmptyFiledate(t *testing.T) { diff := UnifiedDiff{ A: splitChars("one"), B: splitChars("two"), FromFile: "Original", ToFile: "Current", Eol: "\n", } ud, err := GetUnifiedDiffString(diff) assertEqual(t, err, nil) assertEqual(t, SplitLines(ud)[:2], []string{"--- Original\n", "+++ Current\n"}) cd, err := GetContextDiffString(ContextDiff(diff)) assertEqual(t, err, nil) assertEqual(t, SplitLines(cd)[:2], []string{"*** Original\n", "--- Current\n"}) } func TestOmitFilenames(t *testing.T) { diff := UnifiedDiff{ A: SplitLines("o\nn\ne\n"), B: SplitLines("t\nw\no\n"), Eol: "\n", } ud, err := GetUnifiedDiffString(diff) assertEqual(t, err, nil) assertEqual(t, SplitLines(ud), []string{ "@@ -0,0 +1,2 @@\n", "+t\n", "+w\n", "@@ -2,2 +3,0 @@\n", "-n\n", "-e\n", "\n", }) cd, err := GetContextDiffString(ContextDiff(diff)) assertEqual(t, err, nil) assertEqual(t, SplitLines(cd), []string{ "***************\n", "*** 0 ****\n", "--- 1,2 ----\n", "+ t\n", "+ w\n", "***************\n", "*** 2,3 ****\n", "- n\n", "- e\n", "--- 3 ----\n", "\n", }) } func TestSplitLines(t *testing.T) { allTests := []struct { input string want []string }{ {"foo", []string{"foo\n"}}, {"foo\nbar", []string{"foo\n", "bar\n"}}, {"foo\nbar\n", []string{"foo\n", "bar\n", "\n"}}, } for _, test := range allTests { assertEqual(t, SplitLines(test.input), test.want) } } func benchmarkSplitLines(b *testing.B, count int) { str := strings.Repeat("foo\n", count) b.ResetTimer() n := 0 for i := 0; i < b.N; i++ { n += len(SplitLines(str)) } } func BenchmarkSplitLines100(b *testing.B) { benchmarkSplitLines(b, 100) } func BenchmarkSplitLines10000(b *testing.B) { benchmarkSplitLines(b, 10000) } ================================================ FILE: vendor/github.com/prometheus/client_golang/.gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *~ *# .build ================================================ FILE: vendor/github.com/prometheus/client_golang/.travis.yml ================================================ sudo: false language: go go: - 1.6.3 - 1.7 - 1.8.1 script: - go test -short ./... ================================================ FILE: vendor/github.com/prometheus/client_golang/CHANGELOG.md ================================================ ## 0.8.0 / 2016-08-17 * [CHANGE] Registry is doing more consistency checks. This might break existing setups that used to export inconsistent metrics. * [CHANGE] Pushing to Pushgateway moved to package `push` and changed to allow arbitrary grouping. * [CHANGE] Removed `SelfCollector`. * [CHANGE] Removed `PanicOnCollectError` and `EnableCollectChecks` methods. * [CHANGE] Moved packages to the prometheus/common repo: `text`, `model`, `extraction`. * [CHANGE] Deprecated a number of functions. * [FEATURE] Allow custom registries. Added `Registerer` and `Gatherer` interfaces. * [FEATURE] Separated HTTP exposition, allowing custom HTTP handlers (package `promhttp`) and enabling the creation of other exposition mechanisms. * [FEATURE] `MustRegister` is variadic now, allowing registration of many collectors in one call. * [FEATURE] Added HTTP API v1 package. * [ENHANCEMENT] Numerous documentation improvements. * [ENHANCEMENT] Improved metric sorting. * [ENHANCEMENT] Inlined fnv64a hashing for improved performance. * [ENHANCEMENT] Several test improvements. * [BUGFIX] Handle collisions in MetricVec. ## 0.7.0 / 2015-07-27 * [CHANGE] Rename ExporterLabelPrefix to ExportedLabelPrefix. * [BUGFIX] Closed gaps in metric consistency check. * [BUGFIX] Validate LabelName/LabelSet on JSON unmarshaling. * [ENHANCEMENT] Document the possibility to create "empty" metrics in a metric vector. * [ENHANCEMENT] Fix and clarify various doc comments and the README.md. * [ENHANCEMENT] (Kind of) solve "The Proxy Problem" of http.InstrumentHandler. * [ENHANCEMENT] Change responseWriterDelegator.written to int64. ## 0.6.0 / 2015-06-01 * [CHANGE] Rename process_goroutines to go_goroutines. * [ENHANCEMENT] Validate label names during YAML decoding. * [ENHANCEMENT] Add LabelName regular expression. * [BUGFIX] Ensure alignment of struct members for 32-bit systems. ## 0.5.0 / 2015-05-06 * [BUGFIX] Removed a weakness in the fingerprinting aka signature code. This makes fingerprinting slower and more allocation-heavy, but the weakness was too severe to be tolerated. * [CHANGE] As a result of the above, Metric.Fingerprint is now returning a different fingerprint. To keep the same fingerprint, the new method Metric.FastFingerprint was introduced, which will be used by the Prometheus server for storage purposes (implying that a collision detection has to be added, too). * [ENHANCEMENT] The Metric.Equal and Metric.Before do not depend on fingerprinting anymore, removing the possibility of an undetected fingerprint collision. * [FEATURE] The Go collector in the exposition library includes garbage collection stats. * [FEATURE] The exposition library allows to create constant "throw-away" summaries and histograms. * [CHANGE] A number of new reserved labels and prefixes. ## 0.4.0 / 2015-04-08 * [CHANGE] Return NaN when Summaries have no observations yet. * [BUGFIX] Properly handle Summary decay upon Write(). * [BUGFIX] Fix the documentation link to the consumption library. * [FEATURE] Allow the metric family injection hook to merge with existing metric families. * [ENHANCEMENT] Removed cgo dependency and conditional compilation of procfs. * [MAINTENANCE] Adjusted to changes in matttproud/golang_protobuf_extensions. ## 0.3.2 / 2015-03-11 * [BUGFIX] Fixed the receiver type of COWMetric.Set(). This method is only used by the Prometheus server internally. * [CLEANUP] Added licenses of vendored code left out by godep. ## 0.3.1 / 2015-03-04 * [ENHANCEMENT] Switched fingerprinting functions from own free list to sync.Pool. * [CHANGE] Makefile uses Go 1.4.2 now (only relevant for examples and tests). ## 0.3.0 / 2015-03-03 * [CHANGE] Changed the fingerprinting for metrics. THIS WILL INVALIDATE ALL PERSISTED FINGERPRINTS. IF YOU COMPILE THE PROMETHEUS SERVER WITH THIS VERSION, YOU HAVE TO WIPE THE PREVIOUSLY CREATED STORAGE. * [CHANGE] LabelValuesToSignature removed. (Nobody had used it, and it was arguably broken.) * [CHANGE] Vendored dependencies. Those are only used by the Makefile. If client_golang is used as a library, the vendoring will stay out of your way. * [BUGFIX] Remove a weakness in the fingerprinting for metrics. (This made the fingerprinting change above necessary.) * [FEATURE] Added new fingerprinting functions SignatureForLabels and SignatureWithoutLabels to be used by the Prometheus server. These functions require fewer allocations than the ones currently used by the server. ## 0.2.0 / 2015-02-23 * [FEATURE] Introduce new Histagram metric type. * [CHANGE] Ignore process collector errors for now (better error handling pending). * [CHANGE] Use clear error interface for process pidFn. * [BUGFIX] Fix Go download links for several archs and OSes. * [ENHANCEMENT] Massively improve Gauge and Counter performance. * [ENHANCEMENT] Catch illegal label names for summaries in histograms. * [ENHANCEMENT] Reduce allocations during fingerprinting. * [ENHANCEMENT] Remove cgo dependency. procfs package will only be included if both cgo is available and the build is for an OS with procfs. * [CLEANUP] Clean up code style issues. * [CLEANUP] Mark slow test as such and exclude them from travis. * [CLEANUP] Update protobuf library package name. * [CLEANUP] Updated vendoring of beorn7/perks. ## 0.1.0 / 2015-02-02 * [CLEANUP] Introduced semantic versioning and changelog. From now on, changes will be reported in this file. ================================================ FILE: vendor/github.com/prometheus/client_golang/CONTRIBUTING.md ================================================ # Contributing Prometheus uses GitHub to manage reviews of pull requests. * If you have a trivial fix or improvement, go ahead and create a pull request, addressing (with `@...`) the maintainer of this repository (see [MAINTAINERS.md](MAINTAINERS.md)) in the description of the pull request. * If you plan to do something more involved, first discuss your ideas on our [mailing list](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). This will avoid unnecessary work and surely give you and us a good deal of inspiration. * Relevant coding style guidelines are the [Go Code Review Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) and the _Formatting and style_ section of Peter Bourgon's [Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/#formatting-and-style). ================================================ FILE: vendor/github.com/prometheus/client_golang/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: vendor/github.com/prometheus/client_golang/MAINTAINERS.md ================================================ * Björn Rabenstein ================================================ FILE: vendor/github.com/prometheus/client_golang/NOTICE ================================================ Prometheus instrumentation library for Go applications Copyright 2012-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). The following components are included in this product: perks - a fork of https://github.com/bmizerany/perks https://github.com/beorn7/perks Copyright 2013-2015 Blake Mizerany, Björn Rabenstein See https://github.com/beorn7/perks/blob/master/README.md for license details. Go support for Protocol Buffers - Google's data interchange format http://github.com/golang/protobuf/ Copyright 2010 The Go Authors See source code for license details. Support for streaming Protocol Buffer messages for the Go language (golang). https://github.com/matttproud/golang_protobuf_extensions Copyright 2013 Matt T. Proud Licensed under the Apache License, Version 2.0 ================================================ FILE: vendor/github.com/prometheus/client_golang/README.md ================================================ # Prometheus Go client library [![Build Status](https://travis-ci.org/prometheus/client_golang.svg?branch=master)](https://travis-ci.org/prometheus/client_golang) [![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/client_golang)](https://goreportcard.com/report/github.com/prometheus/client_golang) This is the [Go](http://golang.org) client library for [Prometheus](http://prometheus.io). It has two separate parts, one for instrumenting application code, and one for creating clients that talk to the Prometheus HTTP API. ## Instrumenting applications [![code-coverage](http://gocover.io/_badge/github.com/prometheus/client_golang/prometheus)](http://gocover.io/github.com/prometheus/client_golang/prometheus) [![go-doc](https://godoc.org/github.com/prometheus/client_golang/prometheus?status.svg)](https://godoc.org/github.com/prometheus/client_golang/prometheus) The [`prometheus` directory](https://github.com/prometheus/client_golang/tree/master/prometheus) contains the instrumentation library. See the [best practices section](http://prometheus.io/docs/practices/naming/) of the Prometheus documentation to learn more about instrumenting applications. The [`examples` directory](https://github.com/prometheus/client_golang/tree/master/examples) contains simple examples of instrumented code. ## Client for the Prometheus HTTP API [![code-coverage](http://gocover.io/_badge/github.com/prometheus/client_golang/api/prometheus)](http://gocover.io/github.com/prometheus/client_golang/api/prometheus) [![go-doc](https://godoc.org/github.com/prometheus/client_golang/api/prometheus?status.svg)](https://godoc.org/github.com/prometheus/client_golang/api/prometheus) The [`api/prometheus` directory](https://github.com/prometheus/client_golang/tree/master/api/prometheus) contains the client for the [Prometheus HTTP API](http://prometheus.io/docs/querying/api/). It allows you to write Go applications that query time series data from a Prometheus server. It is still in alpha stage. ## Where is `model`, `extraction`, and `text`? The `model` packages has been moved to [`prometheus/common/model`](https://github.com/prometheus/common/tree/master/model). The `extraction` and `text` packages are now contained in [`prometheus/common/expfmt`](https://github.com/prometheus/common/tree/master/expfmt). ## Contributing and community See the [contributing guidelines](CONTRIBUTING.md) and the [Community section](http://prometheus.io/community/) of the homepage. ================================================ FILE: vendor/github.com/prometheus/client_golang/VERSION ================================================ 0.8.0 ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/.gitignore ================================================ command-line-arguments.test ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/README.md ================================================ See [![go-doc](https://godoc.org/github.com/prometheus/client_golang/prometheus?status.svg)](https://godoc.org/github.com/prometheus/client_golang/prometheus). ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/benchmark_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "sync" "testing" ) func BenchmarkCounterWithLabelValues(b *testing.B) { m := NewCounterVec( CounterOpts{ Name: "benchmark_counter", Help: "A counter to benchmark it.", }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.WithLabelValues("eins", "zwei", "drei").Inc() } } func BenchmarkCounterWithLabelValuesConcurrent(b *testing.B) { m := NewCounterVec( CounterOpts{ Name: "benchmark_counter", Help: "A counter to benchmark it.", }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() wg := sync.WaitGroup{} for i := 0; i < 10; i++ { wg.Add(1) go func() { for j := 0; j < b.N/10; j++ { m.WithLabelValues("eins", "zwei", "drei").Inc() } wg.Done() }() } wg.Wait() } func BenchmarkCounterWithMappedLabels(b *testing.B) { m := NewCounterVec( CounterOpts{ Name: "benchmark_counter", Help: "A counter to benchmark it.", }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.With(Labels{"two": "zwei", "one": "eins", "three": "drei"}).Inc() } } func BenchmarkCounterWithPreparedMappedLabels(b *testing.B) { m := NewCounterVec( CounterOpts{ Name: "benchmark_counter", Help: "A counter to benchmark it.", }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() labels := Labels{"two": "zwei", "one": "eins", "three": "drei"} for i := 0; i < b.N; i++ { m.With(labels).Inc() } } func BenchmarkCounterNoLabels(b *testing.B) { m := NewCounter(CounterOpts{ Name: "benchmark_counter", Help: "A counter to benchmark it.", }) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.Inc() } } func BenchmarkGaugeWithLabelValues(b *testing.B) { m := NewGaugeVec( GaugeOpts{ Name: "benchmark_gauge", Help: "A gauge to benchmark it.", }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.WithLabelValues("eins", "zwei", "drei").Set(3.1415) } } func BenchmarkGaugeNoLabels(b *testing.B) { m := NewGauge(GaugeOpts{ Name: "benchmark_gauge", Help: "A gauge to benchmark it.", }) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.Set(3.1415) } } func BenchmarkSummaryWithLabelValues(b *testing.B) { m := NewSummaryVec( SummaryOpts{ Name: "benchmark_summary", Help: "A summary to benchmark it.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.WithLabelValues("eins", "zwei", "drei").Observe(3.1415) } } func BenchmarkSummaryNoLabels(b *testing.B) { m := NewSummary(SummaryOpts{ Name: "benchmark_summary", Help: "A summary to benchmark it.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.Observe(3.1415) } } func BenchmarkHistogramWithLabelValues(b *testing.B) { m := NewHistogramVec( HistogramOpts{ Name: "benchmark_histogram", Help: "A histogram to benchmark it.", }, []string{"one", "two", "three"}, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.WithLabelValues("eins", "zwei", "drei").Observe(3.1415) } } func BenchmarkHistogramNoLabels(b *testing.B) { m := NewHistogram(HistogramOpts{ Name: "benchmark_histogram", Help: "A histogram to benchmark it.", }, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { m.Observe(3.1415) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/collector.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus // Collector is the interface implemented by anything that can be used by // Prometheus to collect metrics. A Collector has to be registered for // collection. See Registerer.Register. // // The stock metrics provided by this package (Gauge, Counter, Summary, // Histogram, Untyped) are also Collectors (which only ever collect one metric, // namely itself). An implementer of Collector may, however, collect multiple // metrics in a coordinated fashion and/or create metrics on the fly. Examples // for collectors already implemented in this library are the metric vectors // (i.e. collection of multiple instances of the same Metric but with different // label values) like GaugeVec or SummaryVec, and the ExpvarCollector. type Collector interface { // Describe sends the super-set of all possible descriptors of metrics // collected by this Collector to the provided channel and returns once // the last descriptor has been sent. The sent descriptors fulfill the // consistency and uniqueness requirements described in the Desc // documentation. (It is valid if one and the same Collector sends // duplicate descriptors. Those duplicates are simply ignored. However, // two different Collectors must not send duplicate descriptors.) This // method idempotently sends the same descriptors throughout the // lifetime of the Collector. If a Collector encounters an error while // executing this method, it must send an invalid descriptor (created // with NewInvalidDesc) to signal the error to the registry. Describe(chan<- *Desc) // Collect is called by the Prometheus registry when collecting // metrics. The implementation sends each collected metric via the // provided channel and returns once the last metric has been sent. The // descriptor of each sent metric is one of those returned by // Describe. Returned metrics that share the same descriptor must differ // in their variable label values. This method may be called // concurrently and must therefore be implemented in a concurrency safe // way. Blocking occurs at the expense of total performance of rendering // all registered metrics. Ideally, Collector implementations support // concurrent readers. Collect(chan<- Metric) } // selfCollector implements Collector for a single Metric so that the Metric // collects itself. Add it as an anonymous field to a struct that implements // Metric, and call init with the Metric itself as an argument. type selfCollector struct { self Metric } // init provides the selfCollector with a reference to the metric it is supposed // to collect. It is usually called within the factory function to create a // metric. See example. func (c *selfCollector) init(self Metric) { c.self = self } // Describe implements Collector. func (c *selfCollector) Describe(ch chan<- *Desc) { ch <- c.self.Desc() } // Collect implements Collector. func (c *selfCollector) Collect(ch chan<- Metric) { ch <- c.self } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/counter.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "errors" ) // Counter is a Metric that represents a single numerical value that only ever // goes up. That implies that it cannot be used to count items whose number can // also go down, e.g. the number of currently running goroutines. Those // "counters" are represented by Gauges. // // A Counter is typically used to count requests served, tasks completed, errors // occurred, etc. // // To create Counter instances, use NewCounter. type Counter interface { Metric Collector // Inc increments the counter by 1. Use Add to increment it by arbitrary // non-negative values. Inc() // Add adds the given value to the counter. It panics if the value is < // 0. Add(float64) } // CounterOpts is an alias for Opts. See there for doc comments. type CounterOpts Opts // NewCounter creates a new Counter based on the provided CounterOpts. func NewCounter(opts CounterOpts) Counter { desc := NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ) result := &counter{value: value{desc: desc, valType: CounterValue, labelPairs: desc.constLabelPairs}} result.init(result) // Init self-collection. return result } type counter struct { value } func (c *counter) Add(v float64) { if v < 0 { panic(errors.New("counter cannot decrease in value")) } c.value.Add(v) } // CounterVec is a Collector that bundles a set of Counters that all share the // same Desc, but have different values for their variable labels. This is used // if you want to count the same thing partitioned by various dimensions // (e.g. number of HTTP requests, partitioned by response code and // method). Create instances with NewCounterVec. type CounterVec struct { *metricVec } // NewCounterVec creates a new CounterVec based on the provided CounterOpts and // partitioned by the given label names. func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec { desc := NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, labelNames, opts.ConstLabels, ) return &CounterVec{ metricVec: newMetricVec(desc, func(lvs ...string) Metric { result := &counter{value: value{ desc: desc, valType: CounterValue, labelPairs: makeLabelPairs(desc, lvs), }} result.init(result) // Init self-collection. return result }), } } // GetMetricWithLabelValues returns the Counter for the given slice of label // values (same order as the VariableLabels in Desc). If that combination of // label values is accessed for the first time, a new Counter is created. // // It is possible to call this method without using the returned Counter to only // create the new Counter but leave it at its starting value 0. See also the // SummaryVec example. // // Keeping the Counter for later use is possible (and should be considered if // performance is critical), but keep in mind that Reset, DeleteLabelValues and // Delete can be used to delete the Counter from the CounterVec. In that case, // the Counter will still exist, but it will not be exported anymore, even if a // Counter with the same label values is created later. // // An error is returned if the number of label values is not the same as the // number of VariableLabels in Desc. // // Note that for more than one label value, this method is prone to mistakes // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // an alternative to avoid that type of mistake. For higher label numbers, the // latter has a much more readable (albeit more verbose) syntax, but it comes // with a performance overhead (for creating and processing the Labels map). // See also the GaugeVec example. func (m *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) { metric, err := m.metricVec.getMetricWithLabelValues(lvs...) if metric != nil { return metric.(Counter), err } return nil, err } // GetMetricWith returns the Counter for the given Labels map (the label names // must match those of the VariableLabels in Desc). If that label map is // accessed for the first time, a new Counter is created. Implications of // creating a Counter without using it and keeping the Counter for later use are // the same as for GetMetricWithLabelValues. // // An error is returned if the number and names of the Labels are inconsistent // with those of the VariableLabels in Desc. // // This method is used for the same purpose as // GetMetricWithLabelValues(...string). See there for pros and cons of the two // methods. func (m *CounterVec) GetMetricWith(labels Labels) (Counter, error) { metric, err := m.metricVec.getMetricWith(labels) if metric != nil { return metric.(Counter), err } return nil, err } // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. By not returning an // error, WithLabelValues allows shortcuts like // myVec.WithLabelValues("404", "GET").Add(42) func (m *CounterVec) WithLabelValues(lvs ...string) Counter { return m.metricVec.withLabelValues(lvs...).(Counter) } // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. By not returning an error, With allows shortcuts like // myVec.With(Labels{"code": "404", "method": "GET"}).Add(42) func (m *CounterVec) With(labels Labels) Counter { return m.metricVec.with(labels).(Counter) } // CounterFunc is a Counter whose value is determined at collect time by calling a // provided function. // // To create CounterFunc instances, use NewCounterFunc. type CounterFunc interface { Metric Collector } // NewCounterFunc creates a new CounterFunc based on the provided // CounterOpts. The value reported is determined by calling the given function // from within the Write method. Take into account that metric collection may // happen concurrently. If that results in concurrent calls to Write, like in // the case where a CounterFunc is directly registered with Prometheus, the // provided function must be concurrency-safe. The function should also honor // the contract for a Counter (values only go up, not down), but compliance will // not be checked. func NewCounterFunc(opts CounterOpts, function func() float64) CounterFunc { return newValueFunc(NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ), CounterValue, function) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/counter_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "fmt" "math" "testing" dto "github.com/prometheus/client_model/go" ) func TestCounterAdd(t *testing.T) { counter := NewCounter(CounterOpts{ Name: "test", Help: "test help", ConstLabels: Labels{"a": "1", "b": "2"}, }).(*counter) counter.Inc() if expected, got := 1., math.Float64frombits(counter.valBits); expected != got { t.Errorf("Expected %f, got %f.", expected, got) } counter.Add(42) if expected, got := 43., math.Float64frombits(counter.valBits); expected != got { t.Errorf("Expected %f, got %f.", expected, got) } if expected, got := "counter cannot decrease in value", decreaseCounter(counter).Error(); expected != got { t.Errorf("Expected error %q, got %q.", expected, got) } m := &dto.Metric{} counter.Write(m) if expected, got := `label: label: counter: `, m.String(); expected != got { t.Errorf("expected %q, got %q", expected, got) } } func decreaseCounter(c *counter) (err error) { defer func() { if e := recover(); e != nil { err = e.(error) } }() c.Add(-1) return nil } func TestCounterVecGetMetricWithInvalidLabelValues(t *testing.T) { testCases := []struct { desc string labels Labels }{ { desc: "non utf8 label value", labels: Labels{"a": "\xFF"}, }, { desc: "not enough label values", labels: Labels{}, }, { desc: "too many label values", labels: Labels{"a": "1", "b": "2"}, }, } for _, test := range testCases { counterVec := NewCounterVec(CounterOpts{ Name: "test", }, []string{"a"}) labelValues := make([]string, len(test.labels)) for _, val := range test.labels { labelValues = append(labelValues, val) } expectPanic(t, func() { counterVec.WithLabelValues(labelValues...) }, fmt.Sprintf("WithLabelValues: expected panic because: %s", test.desc)) expectPanic(t, func() { counterVec.With(test.labels) }, fmt.Sprintf("WithLabelValues: expected panic because: %s", test.desc)) if _, err := counterVec.GetMetricWithLabelValues(labelValues...); err == nil { t.Errorf("GetMetricWithLabelValues: expected error because: %s", test.desc) } if _, err := counterVec.GetMetricWith(test.labels); err == nil { t.Errorf("GetMetricWith: expected error because: %s", test.desc) } } } func expectPanic(t *testing.T, op func(), errorMsg string) { defer func() { if err := recover(); err == nil { t.Error(errorMsg) } }() op() } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/desc.go ================================================ // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "errors" "fmt" "sort" "strings" "github.com/golang/protobuf/proto" "github.com/prometheus/common/model" dto "github.com/prometheus/client_model/go" ) // Desc is the descriptor used by every Prometheus Metric. It is essentially // the immutable meta-data of a Metric. The normal Metric implementations // included in this package manage their Desc under the hood. Users only have to // deal with Desc if they use advanced features like the ExpvarCollector or // custom Collectors and Metrics. // // Descriptors registered with the same registry have to fulfill certain // consistency and uniqueness criteria if they share the same fully-qualified // name: They must have the same help string and the same label names (aka label // dimensions) in each, constLabels and variableLabels, but they must differ in // the values of the constLabels. // // Descriptors that share the same fully-qualified names and the same label // values of their constLabels are considered equal. // // Use NewDesc to create new Desc instances. type Desc struct { // fqName has been built from Namespace, Subsystem, and Name. fqName string // help provides some helpful information about this metric. help string // constLabelPairs contains precalculated DTO label pairs based on // the constant labels. constLabelPairs []*dto.LabelPair // VariableLabels contains names of labels for which the metric // maintains variable values. variableLabels []string // id is a hash of the values of the ConstLabels and fqName. This // must be unique among all registered descriptors and can therefore be // used as an identifier of the descriptor. id uint64 // dimHash is a hash of the label names (preset and variable) and the // Help string. Each Desc with the same fqName must have the same // dimHash. dimHash uint64 // err is an error that occurred during construction. It is reported on // registration time. err error } // NewDesc allocates and initializes a new Desc. Errors are recorded in the Desc // and will be reported on registration time. variableLabels and constLabels can // be nil if no such labels should be set. fqName and help must not be empty. // // variableLabels only contain the label names. Their label values are variable // and therefore not part of the Desc. (They are managed within the Metric.) // // For constLabels, the label values are constant. Therefore, they are fully // specified in the Desc. See the Opts documentation for the implications of // constant labels. func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *Desc { d := &Desc{ fqName: fqName, help: help, variableLabels: variableLabels, } if help == "" { d.err = errors.New("empty help string") return d } if !model.IsValidMetricName(model.LabelValue(fqName)) { d.err = fmt.Errorf("%q is not a valid metric name", fqName) return d } // labelValues contains the label values of const labels (in order of // their sorted label names) plus the fqName (at position 0). labelValues := make([]string, 1, len(constLabels)+1) labelValues[0] = fqName labelNames := make([]string, 0, len(constLabels)+len(variableLabels)) labelNameSet := map[string]struct{}{} // First add only the const label names and sort them... for labelName := range constLabels { if !checkLabelName(labelName) { d.err = fmt.Errorf("%q is not a valid label name", labelName) return d } labelNames = append(labelNames, labelName) labelNameSet[labelName] = struct{}{} } sort.Strings(labelNames) // ... so that we can now add const label values in the order of their names. for _, labelName := range labelNames { labelValues = append(labelValues, constLabels[labelName]) } // Validate the const label values. They can't have a wrong cardinality, so // use in len(labelValues) as expectedNumberOfValues. if err := validateLabelValues(labelValues, len(labelValues)); err != nil { d.err = err return d } // Now add the variable label names, but prefix them with something that // cannot be in a regular label name. That prevents matching the label // dimension with a different mix between preset and variable labels. for _, labelName := range variableLabels { if !checkLabelName(labelName) { d.err = fmt.Errorf("%q is not a valid label name", labelName) return d } labelNames = append(labelNames, "$"+labelName) labelNameSet[labelName] = struct{}{} } if len(labelNames) != len(labelNameSet) { d.err = errors.New("duplicate label names") return d } vh := hashNew() for _, val := range labelValues { vh = hashAdd(vh, val) vh = hashAddByte(vh, separatorByte) } d.id = vh // Sort labelNames so that order doesn't matter for the hash. sort.Strings(labelNames) // Now hash together (in this order) the help string and the sorted // label names. lh := hashNew() lh = hashAdd(lh, help) lh = hashAddByte(lh, separatorByte) for _, labelName := range labelNames { lh = hashAdd(lh, labelName) lh = hashAddByte(lh, separatorByte) } d.dimHash = lh d.constLabelPairs = make([]*dto.LabelPair, 0, len(constLabels)) for n, v := range constLabels { d.constLabelPairs = append(d.constLabelPairs, &dto.LabelPair{ Name: proto.String(n), Value: proto.String(v), }) } sort.Sort(LabelPairSorter(d.constLabelPairs)) return d } // NewInvalidDesc returns an invalid descriptor, i.e. a descriptor with the // provided error set. If a collector returning such a descriptor is registered, // registration will fail with the provided error. NewInvalidDesc can be used by // a Collector to signal inability to describe itself. func NewInvalidDesc(err error) *Desc { return &Desc{ err: err, } } func (d *Desc) String() string { lpStrings := make([]string, 0, len(d.constLabelPairs)) for _, lp := range d.constLabelPairs { lpStrings = append( lpStrings, fmt.Sprintf("%s=%q", lp.GetName(), lp.GetValue()), ) } return fmt.Sprintf( "Desc{fqName: %q, help: %q, constLabels: {%s}, variableLabels: %v}", d.fqName, d.help, strings.Join(lpStrings, ","), d.variableLabels, ) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/desc_test.go ================================================ package prometheus import ( "testing" ) func TestNewDescInvalidLabelValues(t *testing.T) { desc := NewDesc( "sample_label", "sample label", nil, Labels{"a": "\xFF"}, ) if desc.err == nil { t.Errorf("NewDesc: expected error because: %s", desc.err) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/doc.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package prometheus provides metrics primitives to instrument code for // monitoring. It also offers a registry for metrics. Sub-packages allow to // expose the registered metrics via HTTP (package promhttp) or push them to a // Pushgateway (package push). // // All exported functions and methods are safe to be used concurrently unless // specified otherwise. // // A Basic Example // // As a starting point, a very basic usage example: // // package main // // import ( // "log" // "net/http" // // "github.com/prometheus/client_golang/prometheus" // "github.com/prometheus/client_golang/prometheus/promhttp" // ) // // var ( // cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{ // Name: "cpu_temperature_celsius", // Help: "Current temperature of the CPU.", // }) // hdFailures = prometheus.NewCounterVec( // prometheus.CounterOpts{ // Name: "hd_errors_total", // Help: "Number of hard-disk errors.", // }, // []string{"device"}, // ) // ) // // func init() { // // Metrics have to be registered to be exposed: // prometheus.MustRegister(cpuTemp) // prometheus.MustRegister(hdFailures) // } // // func main() { // cpuTemp.Set(65.3) // hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc() // // // The Handler function provides a default handler to expose metrics // // via an HTTP server. "/metrics" is the usual endpoint for that. // http.Handle("/metrics", promhttp.Handler()) // log.Fatal(http.ListenAndServe(":8080", nil)) // } // // // This is a complete program that exports two metrics, a Gauge and a Counter, // the latter with a label attached to turn it into a (one-dimensional) vector. // // Metrics // // The number of exported identifiers in this package might appear a bit // overwhelming. However, in addition to the basic plumbing shown in the example // above, you only need to understand the different metric types and their // vector versions for basic usage. // // Above, you have already touched the Counter and the Gauge. There are two more // advanced metric types: the Summary and Histogram. A more thorough description // of those four metric types can be found in the Prometheus docs: // https://prometheus.io/docs/concepts/metric_types/ // // A fifth "type" of metric is Untyped. It behaves like a Gauge, but signals the // Prometheus server not to assume anything about its type. // // In addition to the fundamental metric types Gauge, Counter, Summary, // Histogram, and Untyped, a very important part of the Prometheus data model is // the partitioning of samples along dimensions called labels, which results in // metric vectors. The fundamental types are GaugeVec, CounterVec, SummaryVec, // HistogramVec, and UntypedVec. // // While only the fundamental metric types implement the Metric interface, both // the metrics and their vector versions implement the Collector interface. A // Collector manages the collection of a number of Metrics, but for convenience, // a Metric can also “collect itself”. Note that Gauge, Counter, Summary, // Histogram, and Untyped are interfaces themselves while GaugeVec, CounterVec, // SummaryVec, HistogramVec, and UntypedVec are not. // // To create instances of Metrics and their vector versions, you need a suitable // …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts, HistogramOpts, or // UntypedOpts. // // Custom Collectors and constant Metrics // // While you could create your own implementations of Metric, most likely you // will only ever implement the Collector interface on your own. At a first // glance, a custom Collector seems handy to bundle Metrics for common // registration (with the prime example of the different metric vectors above, // which bundle all the metrics of the same name but with different labels). // // There is a more involved use case, too: If you already have metrics // available, created outside of the Prometheus context, you don't need the // interface of the various Metric types. You essentially want to mirror the // existing numbers into Prometheus Metrics during collection. An own // implementation of the Collector interface is perfect for that. You can create // Metric instances “on the fly” using NewConstMetric, NewConstHistogram, and // NewConstSummary (and their respective Must… versions). That will happen in // the Collect method. The Describe method has to return separate Desc // instances, representative of the “throw-away” metrics to be created later. // NewDesc comes in handy to create those Desc instances. // // The Collector example illustrates the use case. You can also look at the // source code of the processCollector (mirroring process metrics), the // goCollector (mirroring Go metrics), or the expvarCollector (mirroring expvar // metrics) as examples that are used in this package itself. // // If you just need to call a function to get a single float value to collect as // a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting // shortcuts. // // Advanced Uses of the Registry // // While MustRegister is the by far most common way of registering a Collector, // sometimes you might want to handle the errors the registration might cause. // As suggested by the name, MustRegister panics if an error occurs. With the // Register function, the error is returned and can be handled. // // An error is returned if the registered Collector is incompatible or // inconsistent with already registered metrics. The registry aims for // consistency of the collected metrics according to the Prometheus data model. // Inconsistencies are ideally detected at registration time, not at collect // time. The former will usually be detected at start-up time of a program, // while the latter will only happen at scrape time, possibly not even on the // first scrape if the inconsistency only becomes relevant later. That is the // main reason why a Collector and a Metric have to describe themselves to the // registry. // // So far, everything we did operated on the so-called default registry, as it // can be found in the global DefaultRegistry variable. With NewRegistry, you // can create a custom registry, or you can even implement the Registerer or // Gatherer interfaces yourself. The methods Register and Unregister work in the // same way on a custom registry as the global functions Register and Unregister // on the default registry. // // There are a number of uses for custom registries: You can use registries with // special properties, see NewPedanticRegistry. You can avoid global state, as // it is imposed by the DefaultRegistry. You can use multiple registries at the // same time to expose different metrics in different ways. You can use separate // registries for testing purposes. // // Also note that the DefaultRegistry comes registered with a Collector for Go // runtime metrics (via NewGoCollector) and a Collector for process metrics (via // NewProcessCollector). With a custom registry, you are in control and decide // yourself about the Collectors to register. // // HTTP Exposition // // The Registry implements the Gatherer interface. The caller of the Gather // method can then expose the gathered metrics in some way. Usually, the metrics // are served via HTTP on the /metrics endpoint. That's happening in the example // above. The tools to expose metrics via HTTP are in the promhttp sub-package. // (The top-level functions in the prometheus package are deprecated.) // // Pushing to the Pushgateway // // Function for pushing to the Pushgateway can be found in the push sub-package. // // Graphite Bridge // // Functions and examples to push metrics from a Gatherer to Graphite can be // found in the graphite sub-package. // // Other Means of Exposition // // More ways of exposing metrics can easily be added by following the approaches // of the existing implementations. package prometheus ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/example_clustermanager_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus_test import "github.com/prometheus/client_golang/prometheus" // ClusterManager is an example for a system that might have been built without // Prometheus in mind. It models a central manager of jobs running in a // cluster. To turn it into something that collects Prometheus metrics, we // simply add the two methods required for the Collector interface. // // An additional challenge is that multiple instances of the ClusterManager are // run within the same binary, each in charge of a different zone. We need to // make use of ConstLabels to be able to register each ClusterManager instance // with Prometheus. type ClusterManager struct { Zone string OOMCountDesc *prometheus.Desc RAMUsageDesc *prometheus.Desc // ... many more fields } // ReallyExpensiveAssessmentOfTheSystemState is a mock for the data gathering a // real cluster manager would have to do. Since it may actually be really // expensive, it must only be called once per collection. This implementation, // obviously, only returns some made-up data. func (c *ClusterManager) ReallyExpensiveAssessmentOfTheSystemState() ( oomCountByHost map[string]int, ramUsageByHost map[string]float64, ) { // Just example fake data. oomCountByHost = map[string]int{ "foo.example.org": 42, "bar.example.org": 2001, } ramUsageByHost = map[string]float64{ "foo.example.org": 6.023e23, "bar.example.org": 3.14, } return } // Describe simply sends the two Descs in the struct to the channel. func (c *ClusterManager) Describe(ch chan<- *prometheus.Desc) { ch <- c.OOMCountDesc ch <- c.RAMUsageDesc } // Collect first triggers the ReallyExpensiveAssessmentOfTheSystemState. Then it // creates constant metrics for each host on the fly based on the returned data. // // Note that Collect could be called concurrently, so we depend on // ReallyExpensiveAssessmentOfTheSystemState to be concurrency-safe. func (c *ClusterManager) Collect(ch chan<- prometheus.Metric) { oomCountByHost, ramUsageByHost := c.ReallyExpensiveAssessmentOfTheSystemState() for host, oomCount := range oomCountByHost { ch <- prometheus.MustNewConstMetric( c.OOMCountDesc, prometheus.CounterValue, float64(oomCount), host, ) } for host, ramUsage := range ramUsageByHost { ch <- prometheus.MustNewConstMetric( c.RAMUsageDesc, prometheus.GaugeValue, ramUsage, host, ) } } // NewClusterManager creates the two Descs OOMCountDesc and RAMUsageDesc. Note // that the zone is set as a ConstLabel. (It's different in each instance of the // ClusterManager, but constant over the lifetime of an instance.) Then there is // a variable label "host", since we want to partition the collected metrics by // host. Since all Descs created in this way are consistent across instances, // with a guaranteed distinction by the "zone" label, we can register different // ClusterManager instances with the same registry. func NewClusterManager(zone string) *ClusterManager { return &ClusterManager{ Zone: zone, OOMCountDesc: prometheus.NewDesc( "clustermanager_oom_crashes_total", "Number of OOM crashes.", []string{"host"}, prometheus.Labels{"zone": zone}, ), RAMUsageDesc: prometheus.NewDesc( "clustermanager_ram_usage_bytes", "RAM usage as reported to the cluster manager.", []string{"host"}, prometheus.Labels{"zone": zone}, ), } } func ExampleCollector() { workerDB := NewClusterManager("db") workerCA := NewClusterManager("ca") // Since we are dealing with custom Collector implementations, it might // be a good idea to try it out with a pedantic registry. reg := prometheus.NewPedanticRegistry() reg.MustRegister(workerDB) reg.MustRegister(workerCA) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/example_timer_complex_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus_test import ( "net/http" "github.com/prometheus/client_golang/prometheus" ) var ( // apiRequestDuration tracks the duration separate for each HTTP status // class (1xx, 2xx, ...). This creates a fair amount of time series on // the Prometheus server. Usually, you would track the duration of // serving HTTP request without partitioning by outcome. Do something // like this only if needed. Also note how only status classes are // tracked, not every single status code. The latter would create an // even larger amount of time series. Request counters partitioned by // status code are usually OK as each counter only creates one time // series. Histograms are way more expensive, so partition with care and // only where you really need separate latency tracking. Partitioning by // status class is only an example. In concrete cases, other partitions // might make more sense. apiRequestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "api_request_duration_seconds", Help: "Histogram for the request duration of the public API, partitioned by status class.", Buckets: prometheus.ExponentialBuckets(0.1, 1.5, 5), }, []string{"status_class"}, ) ) func handler(w http.ResponseWriter, r *http.Request) { status := http.StatusOK // The ObserverFunc gets called by the deferred ObserveDuration and // decides which Histogram's Observe method is called. timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) { switch { case status >= 500: // Server error. apiRequestDuration.WithLabelValues("5xx").Observe(v) case status >= 400: // Client error. apiRequestDuration.WithLabelValues("4xx").Observe(v) case status >= 300: // Redirection. apiRequestDuration.WithLabelValues("3xx").Observe(v) case status >= 200: // Success. apiRequestDuration.WithLabelValues("2xx").Observe(v) default: // Informational. apiRequestDuration.WithLabelValues("1xx").Observe(v) } })) defer timer.ObserveDuration() // Handle the request. Set status accordingly. // ... } func ExampleTimer_complex() { http.HandleFunc("/api", handler) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/example_timer_gauge_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus_test import ( "os" "github.com/prometheus/client_golang/prometheus" ) var ( // If a function is called rarely (i.e. not more often than scrapes // happen) or ideally only once (like in a batch job), it can make sense // to use a Gauge for timing the function call. For timing a batch job // and pushing the result to a Pushgateway, see also the comprehensive // example in the push package. funcDuration = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "example_function_duration_seconds", Help: "Duration of the last call of an example function.", }) ) func run() error { // The Set method of the Gauge is used to observe the duration. timer := prometheus.NewTimer(prometheus.ObserverFunc(funcDuration.Set)) defer timer.ObserveDuration() // Do something. Return errors as encountered. The use of 'defer' above // makes sure the function is still timed properly. return nil } func ExampleTimer_gauge() { if err := run(); err != nil { os.Exit(1) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/example_timer_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus_test import ( "math/rand" "time" "github.com/prometheus/client_golang/prometheus" ) var ( requestDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "example_request_duration_seconds", Help: "Histogram for the runtime of a simple example function.", Buckets: prometheus.LinearBuckets(0.01, 0.01, 10), }) ) func ExampleTimer() { // timer times this example function. It uses a Histogram, but a Summary // would also work, as both implement Observer. Check out // https://prometheus.io/docs/practices/histograms/ for differences. timer := prometheus.NewTimer(requestDuration) defer timer.ObserveDuration() // Do something here that takes time. time.Sleep(time.Duration(rand.NormFloat64()*10000+50000) * time.Microsecond) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/examples_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus_test import ( "bytes" "fmt" "math" "net/http" "runtime" "sort" "strings" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/golang/protobuf/proto" "github.com/prometheus/client_golang/prometheus" ) func ExampleGauge() { opsQueued := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "our_company", Subsystem: "blob_storage", Name: "ops_queued", Help: "Number of blob storage operations waiting to be processed.", }) prometheus.MustRegister(opsQueued) // 10 operations queued by the goroutine managing incoming requests. opsQueued.Add(10) // A worker goroutine has picked up a waiting operation. opsQueued.Dec() // And once more... opsQueued.Dec() } func ExampleGaugeVec() { opsQueued := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "our_company", Subsystem: "blob_storage", Name: "ops_queued", Help: "Number of blob storage operations waiting to be processed, partitioned by user and type.", }, []string{ // Which user has requested the operation? "user", // Of what type is the operation? "type", }, ) prometheus.MustRegister(opsQueued) // Increase a value using compact (but order-sensitive!) WithLabelValues(). opsQueued.WithLabelValues("bob", "put").Add(4) // Increase a value with a map using WithLabels. More verbose, but order // doesn't matter anymore. opsQueued.With(prometheus.Labels{"type": "delete", "user": "alice"}).Inc() } func ExampleGaugeFunc() { if err := prometheus.Register(prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Subsystem: "runtime", Name: "goroutines_count", Help: "Number of goroutines that currently exist.", }, func() float64 { return float64(runtime.NumGoroutine()) }, )); err == nil { fmt.Println("GaugeFunc 'goroutines_count' registered.") } // Note that the count of goroutines is a gauge (and not a counter) as // it can go up and down. // Output: // GaugeFunc 'goroutines_count' registered. } func ExampleCounter() { pushCounter := prometheus.NewCounter(prometheus.CounterOpts{ Name: "repository_pushes", // Note: No help string... }) err := prometheus.Register(pushCounter) // ... so this will return an error. if err != nil { fmt.Println("Push counter couldn't be registered, no counting will happen:", err) return } // Try it once more, this time with a help string. pushCounter = prometheus.NewCounter(prometheus.CounterOpts{ Name: "repository_pushes", Help: "Number of pushes to external repository.", }) err = prometheus.Register(pushCounter) if err != nil { fmt.Println("Push counter couldn't be registered AGAIN, no counting will happen:", err) return } pushComplete := make(chan struct{}) // TODO: Start a goroutine that performs repository pushes and reports // each completion via the channel. for range pushComplete { pushCounter.Inc() } // Output: // Push counter couldn't be registered, no counting will happen: descriptor Desc{fqName: "repository_pushes", help: "", constLabels: {}, variableLabels: []} is invalid: empty help string } func ExampleCounterVec() { httpReqs := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "http_requests_total", Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", }, []string{"code", "method"}, ) prometheus.MustRegister(httpReqs) httpReqs.WithLabelValues("404", "POST").Add(42) // If you have to access the same set of labels very frequently, it // might be good to retrieve the metric only once and keep a handle to // it. But beware of deletion of that metric, see below! m := httpReqs.WithLabelValues("200", "GET") for i := 0; i < 1000000; i++ { m.Inc() } // Delete a metric from the vector. If you have previously kept a handle // to that metric (as above), future updates via that handle will go // unseen (even if you re-create a metric with the same label set // later). httpReqs.DeleteLabelValues("200", "GET") // Same thing with the more verbose Labels syntax. httpReqs.Delete(prometheus.Labels{"method": "GET", "code": "200"}) } func ExampleInstrumentHandler() { // Handle the "/doc" endpoint with the standard http.FileServer handler. // By wrapping the handler with InstrumentHandler, request count, // request and response sizes, and request latency are automatically // exported to Prometheus, partitioned by HTTP status code and method // and by the handler name (here "fileserver"). http.Handle("/doc", prometheus.InstrumentHandler( "fileserver", http.FileServer(http.Dir("/usr/share/doc")), )) // The Prometheus handler still has to be registered to handle the // "/metrics" endpoint. The handler returned by prometheus.Handler() is // already instrumented - with "prometheus" as the handler name. In this // example, we want the handler name to be "metrics", so we instrument // the uninstrumented Prometheus handler ourselves. http.Handle("/metrics", prometheus.InstrumentHandler( "metrics", prometheus.UninstrumentedHandler(), )) } func ExampleLabelPairSorter() { labelPairs := []*dto.LabelPair{ {Name: proto.String("status"), Value: proto.String("404")}, {Name: proto.String("method"), Value: proto.String("get")}, } sort.Sort(prometheus.LabelPairSorter(labelPairs)) fmt.Println(labelPairs) // Output: // [name:"method" value:"get" name:"status" value:"404" ] } func ExampleRegister() { // Imagine you have a worker pool and want to count the tasks completed. taskCounter := prometheus.NewCounter(prometheus.CounterOpts{ Subsystem: "worker_pool", Name: "completed_tasks_total", Help: "Total number of tasks completed.", }) // This will register fine. if err := prometheus.Register(taskCounter); err != nil { fmt.Println(err) } else { fmt.Println("taskCounter registered.") } // Don't forget to tell the HTTP server about the Prometheus handler. // (In a real program, you still need to start the HTTP server...) http.Handle("/metrics", prometheus.Handler()) // Now you can start workers and give every one of them a pointer to // taskCounter and let it increment it whenever it completes a task. taskCounter.Inc() // This has to happen somewhere in the worker code. // But wait, you want to see how individual workers perform. So you need // a vector of counters, with one element for each worker. taskCounterVec := prometheus.NewCounterVec( prometheus.CounterOpts{ Subsystem: "worker_pool", Name: "completed_tasks_total", Help: "Total number of tasks completed.", }, []string{"worker_id"}, ) // Registering will fail because we already have a metric of that name. if err := prometheus.Register(taskCounterVec); err != nil { fmt.Println("taskCounterVec not registered:", err) } else { fmt.Println("taskCounterVec registered.") } // To fix, first unregister the old taskCounter. if prometheus.Unregister(taskCounter) { fmt.Println("taskCounter unregistered.") } // Try registering taskCounterVec again. if err := prometheus.Register(taskCounterVec); err != nil { fmt.Println("taskCounterVec not registered:", err) } else { fmt.Println("taskCounterVec registered.") } // Bummer! Still doesn't work. // Prometheus will not allow you to ever export metrics with // inconsistent help strings or label names. After unregistering, the // unregistered metrics will cease to show up in the /metrics HTTP // response, but the registry still remembers that those metrics had // been exported before. For this example, we will now choose a // different name. (In a real program, you would obviously not export // the obsolete metric in the first place.) taskCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Subsystem: "worker_pool", Name: "completed_tasks_by_id", Help: "Total number of tasks completed.", }, []string{"worker_id"}, ) if err := prometheus.Register(taskCounterVec); err != nil { fmt.Println("taskCounterVec not registered:", err) } else { fmt.Println("taskCounterVec registered.") } // Finally it worked! // The workers have to tell taskCounterVec their id to increment the // right element in the metric vector. taskCounterVec.WithLabelValues("42").Inc() // Code from worker 42. // Each worker could also keep a reference to their own counter element // around. Pick the counter at initialization time of the worker. myCounter := taskCounterVec.WithLabelValues("42") // From worker 42 initialization code. myCounter.Inc() // Somewhere in the code of that worker. // Note that something like WithLabelValues("42", "spurious arg") would // panic (because you have provided too many label values). If you want // to get an error instead, use GetMetricWithLabelValues(...) instead. notMyCounter, err := taskCounterVec.GetMetricWithLabelValues("42", "spurious arg") if err != nil { fmt.Println("Worker initialization failed:", err) } if notMyCounter == nil { fmt.Println("notMyCounter is nil.") } // A different (and somewhat tricky) approach is to use // ConstLabels. ConstLabels are pairs of label names and label values // that never change. You might ask what those labels are good for (and // rightfully so - if they never change, they could as well be part of // the metric name). There are essentially two use-cases: The first is // if labels are constant throughout the lifetime of a binary execution, // but they vary over time or between different instances of a running // binary. The second is what we have here: Each worker creates and // registers an own Counter instance where the only difference is in the // value of the ConstLabels. Those Counters can all be registered // because the different ConstLabel values guarantee that each worker // will increment a different Counter metric. counterOpts := prometheus.CounterOpts{ Subsystem: "worker_pool", Name: "completed_tasks", Help: "Total number of tasks completed.", ConstLabels: prometheus.Labels{"worker_id": "42"}, } taskCounterForWorker42 := prometheus.NewCounter(counterOpts) if err := prometheus.Register(taskCounterForWorker42); err != nil { fmt.Println("taskCounterVForWorker42 not registered:", err) } else { fmt.Println("taskCounterForWorker42 registered.") } // Obviously, in real code, taskCounterForWorker42 would be a member // variable of a worker struct, and the "42" would be retrieved with a // GetId() method or something. The Counter would be created and // registered in the initialization code of the worker. // For the creation of the next Counter, we can recycle // counterOpts. Just change the ConstLabels. counterOpts.ConstLabels = prometheus.Labels{"worker_id": "2001"} taskCounterForWorker2001 := prometheus.NewCounter(counterOpts) if err := prometheus.Register(taskCounterForWorker2001); err != nil { fmt.Println("taskCounterVForWorker2001 not registered:", err) } else { fmt.Println("taskCounterForWorker2001 registered.") } taskCounterForWorker2001.Inc() taskCounterForWorker42.Inc() taskCounterForWorker2001.Inc() // Yet another approach would be to turn the workers themselves into // Collectors and register them. See the Collector example for details. // Output: // taskCounter registered. // taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [worker_id]} has different label names or a different help string // taskCounter unregistered. // taskCounterVec not registered: a previously registered descriptor with the same fully-qualified name as Desc{fqName: "worker_pool_completed_tasks_total", help: "Total number of tasks completed.", constLabels: {}, variableLabels: [worker_id]} has different label names or a different help string // taskCounterVec registered. // Worker initialization failed: inconsistent label cardinality // notMyCounter is nil. // taskCounterForWorker42 registered. // taskCounterForWorker2001 registered. } func ExampleSummary() { temps := prometheus.NewSummary(prometheus.SummaryOpts{ Name: "pond_temperature_celsius", Help: "The temperature of the frog pond.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }) // Simulate some observations. for i := 0; i < 1000; i++ { temps.Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10) } // Just for demonstration, let's check the state of the summary by // (ab)using its Write method (which is usually only used by Prometheus // internally). metric := &dto.Metric{} temps.Write(metric) fmt.Println(proto.MarshalTextString(metric)) // Output: // summary: < // sample_count: 1000 // sample_sum: 29969.50000000001 // quantile: < // quantile: 0.5 // value: 31.1 // > // quantile: < // quantile: 0.9 // value: 41.3 // > // quantile: < // quantile: 0.99 // value: 41.9 // > // > } func ExampleSummaryVec() { temps := prometheus.NewSummaryVec( prometheus.SummaryOpts{ Name: "pond_temperature_celsius", Help: "The temperature of the frog pond.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, []string{"species"}, ) // Simulate some observations. for i := 0; i < 1000; i++ { temps.WithLabelValues("litoria-caerulea").Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10) temps.WithLabelValues("lithobates-catesbeianus").Observe(32 + math.Floor(100*math.Cos(float64(i)*0.11))/10) } // Create a Summary without any observations. temps.WithLabelValues("leiopelma-hochstetteri") // Just for demonstration, let's check the state of the summary vector // by registering it with a custom registry and then let it collect the // metrics. reg := prometheus.NewRegistry() reg.MustRegister(temps) metricFamilies, err := reg.Gather() if err != nil || len(metricFamilies) != 1 { panic("unexpected behavior of custom test registry") } fmt.Println(proto.MarshalTextString(metricFamilies[0])) // Output: // name: "pond_temperature_celsius" // help: "The temperature of the frog pond." // type: SUMMARY // metric: < // label: < // name: "species" // value: "leiopelma-hochstetteri" // > // summary: < // sample_count: 0 // sample_sum: 0 // quantile: < // quantile: 0.5 // value: nan // > // quantile: < // quantile: 0.9 // value: nan // > // quantile: < // quantile: 0.99 // value: nan // > // > // > // metric: < // label: < // name: "species" // value: "lithobates-catesbeianus" // > // summary: < // sample_count: 1000 // sample_sum: 31956.100000000017 // quantile: < // quantile: 0.5 // value: 32.4 // > // quantile: < // quantile: 0.9 // value: 41.4 // > // quantile: < // quantile: 0.99 // value: 41.9 // > // > // > // metric: < // label: < // name: "species" // value: "litoria-caerulea" // > // summary: < // sample_count: 1000 // sample_sum: 29969.50000000001 // quantile: < // quantile: 0.5 // value: 31.1 // > // quantile: < // quantile: 0.9 // value: 41.3 // > // quantile: < // quantile: 0.99 // value: 41.9 // > // > // > } func ExampleNewConstSummary() { desc := prometheus.NewDesc( "http_request_duration_seconds", "A summary of the HTTP request durations.", []string{"code", "method"}, prometheus.Labels{"owner": "example"}, ) // Create a constant summary from values we got from a 3rd party telemetry system. s := prometheus.MustNewConstSummary( desc, 4711, 403.34, map[float64]float64{0.5: 42.3, 0.9: 323.3}, "200", "get", ) // Just for demonstration, let's check the state of the summary by // (ab)using its Write method (which is usually only used by Prometheus // internally). metric := &dto.Metric{} s.Write(metric) fmt.Println(proto.MarshalTextString(metric)) // Output: // label: < // name: "code" // value: "200" // > // label: < // name: "method" // value: "get" // > // label: < // name: "owner" // value: "example" // > // summary: < // sample_count: 4711 // sample_sum: 403.34 // quantile: < // quantile: 0.5 // value: 42.3 // > // quantile: < // quantile: 0.9 // value: 323.3 // > // > } func ExampleHistogram() { temps := prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "pond_temperature_celsius", Help: "The temperature of the frog pond.", // Sorry, we can't measure how badly it smells. Buckets: prometheus.LinearBuckets(20, 5, 5), // 5 buckets, each 5 centigrade wide. }) // Simulate some observations. for i := 0; i < 1000; i++ { temps.Observe(30 + math.Floor(120*math.Sin(float64(i)*0.1))/10) } // Just for demonstration, let's check the state of the histogram by // (ab)using its Write method (which is usually only used by Prometheus // internally). metric := &dto.Metric{} temps.Write(metric) fmt.Println(proto.MarshalTextString(metric)) // Output: // histogram: < // sample_count: 1000 // sample_sum: 29969.50000000001 // bucket: < // cumulative_count: 192 // upper_bound: 20 // > // bucket: < // cumulative_count: 366 // upper_bound: 25 // > // bucket: < // cumulative_count: 501 // upper_bound: 30 // > // bucket: < // cumulative_count: 638 // upper_bound: 35 // > // bucket: < // cumulative_count: 816 // upper_bound: 40 // > // > } func ExampleNewConstHistogram() { desc := prometheus.NewDesc( "http_request_duration_seconds", "A histogram of the HTTP request durations.", []string{"code", "method"}, prometheus.Labels{"owner": "example"}, ) // Create a constant histogram from values we got from a 3rd party telemetry system. h := prometheus.MustNewConstHistogram( desc, 4711, 403.34, map[float64]uint64{25: 121, 50: 2403, 100: 3221, 200: 4233}, "200", "get", ) // Just for demonstration, let's check the state of the histogram by // (ab)using its Write method (which is usually only used by Prometheus // internally). metric := &dto.Metric{} h.Write(metric) fmt.Println(proto.MarshalTextString(metric)) // Output: // label: < // name: "code" // value: "200" // > // label: < // name: "method" // value: "get" // > // label: < // name: "owner" // value: "example" // > // histogram: < // sample_count: 4711 // sample_sum: 403.34 // bucket: < // cumulative_count: 121 // upper_bound: 25 // > // bucket: < // cumulative_count: 2403 // upper_bound: 50 // > // bucket: < // cumulative_count: 3221 // upper_bound: 100 // > // bucket: < // cumulative_count: 4233 // upper_bound: 200 // > // > } func ExampleAlreadyRegisteredError() { reqCounter := prometheus.NewCounter(prometheus.CounterOpts{ Name: "requests_total", Help: "The total number of requests served.", }) if err := prometheus.Register(reqCounter); err != nil { if are, ok := err.(prometheus.AlreadyRegisteredError); ok { // A counter for that metric has been registered before. // Use the old counter from now on. reqCounter = are.ExistingCollector.(prometheus.Counter) } else { // Something else went wrong! panic(err) } } reqCounter.Inc() } func ExampleGatherers() { reg := prometheus.NewRegistry() temp := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "temperature_kelvin", Help: "Temperature in Kelvin.", }, []string{"location"}, ) reg.MustRegister(temp) temp.WithLabelValues("outside").Set(273.14) temp.WithLabelValues("inside").Set(298.44) var parser expfmt.TextParser text := ` # TYPE humidity_percent gauge # HELP humidity_percent Humidity in %. humidity_percent{location="outside"} 45.4 humidity_percent{location="inside"} 33.2 # TYPE temperature_kelvin gauge # HELP temperature_kelvin Temperature in Kelvin. temperature_kelvin{location="somewhere else"} 4.5 ` parseText := func() ([]*dto.MetricFamily, error) { parsed, err := parser.TextToMetricFamilies(strings.NewReader(text)) if err != nil { return nil, err } var result []*dto.MetricFamily for _, mf := range parsed { result = append(result, mf) } return result, nil } gatherers := prometheus.Gatherers{ reg, prometheus.GathererFunc(parseText), } gathering, err := gatherers.Gather() if err != nil { fmt.Println(err) } out := &bytes.Buffer{} for _, mf := range gathering { if _, err := expfmt.MetricFamilyToText(out, mf); err != nil { panic(err) } } fmt.Print(out.String()) fmt.Println("----------") // Note how the temperature_kelvin metric family has been merged from // different sources. Now try text = ` # TYPE humidity_percent gauge # HELP humidity_percent Humidity in %. humidity_percent{location="outside"} 45.4 humidity_percent{location="inside"} 33.2 # TYPE temperature_kelvin gauge # HELP temperature_kelvin Temperature in Kelvin. # Duplicate metric: temperature_kelvin{location="outside"} 265.3 # Wrong labels: temperature_kelvin 4.5 ` gathering, err = gatherers.Gather() if err != nil { fmt.Println(err) } // Note that still as many metrics as possible are returned: out.Reset() for _, mf := range gathering { if _, err := expfmt.MetricFamilyToText(out, mf); err != nil { panic(err) } } fmt.Print(out.String()) // Output: // # HELP humidity_percent Humidity in %. // # TYPE humidity_percent gauge // humidity_percent{location="inside"} 33.2 // humidity_percent{location="outside"} 45.4 // # HELP temperature_kelvin Temperature in Kelvin. // # TYPE temperature_kelvin gauge // temperature_kelvin{location="inside"} 298.44 // temperature_kelvin{location="outside"} 273.14 // temperature_kelvin{location="somewhere else"} 4.5 // ---------- // 2 error(s) occurred: // * collected metric temperature_kelvin label: gauge: was collected before with the same name and label values // * collected metric temperature_kelvin gauge: has label dimensions inconsistent with previously collected metrics in the same metric family // # HELP humidity_percent Humidity in %. // # TYPE humidity_percent gauge // humidity_percent{location="inside"} 33.2 // humidity_percent{location="outside"} 45.4 // # HELP temperature_kelvin Temperature in Kelvin. // # TYPE temperature_kelvin gauge // temperature_kelvin{location="inside"} 298.44 // temperature_kelvin{location="outside"} 273.14 } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "encoding/json" "expvar" ) type expvarCollector struct { exports map[string]*Desc } // NewExpvarCollector returns a newly allocated expvar Collector that still has // to be registered with a Prometheus registry. // // An expvar Collector collects metrics from the expvar interface. It provides a // quick way to expose numeric values that are already exported via expvar as // Prometheus metrics. Note that the data models of expvar and Prometheus are // fundamentally different, and that the expvar Collector is inherently slower // than native Prometheus metrics. Thus, the expvar Collector is probably great // for experiments and prototying, but you should seriously consider a more // direct implementation of Prometheus metrics for monitoring production // systems. // // The exports map has the following meaning: // // The keys in the map correspond to expvar keys, i.e. for every expvar key you // want to export as Prometheus metric, you need an entry in the exports // map. The descriptor mapped to each key describes how to export the expvar // value. It defines the name and the help string of the Prometheus metric // proxying the expvar value. The type will always be Untyped. // // For descriptors without variable labels, the expvar value must be a number or // a bool. The number is then directly exported as the Prometheus sample // value. (For a bool, 'false' translates to 0 and 'true' to 1). Expvar values // that are not numbers or bools are silently ignored. // // If the descriptor has one variable label, the expvar value must be an expvar // map. The keys in the expvar map become the various values of the one // Prometheus label. The values in the expvar map must be numbers or bools again // as above. // // For descriptors with more than one variable label, the expvar must be a // nested expvar map, i.e. where the values of the topmost map are maps again // etc. until a depth is reached that corresponds to the number of labels. The // leaves of that structure must be numbers or bools as above to serve as the // sample values. // // Anything that does not fit into the scheme above is silently ignored. func NewExpvarCollector(exports map[string]*Desc) Collector { return &expvarCollector{ exports: exports, } } // Describe implements Collector. func (e *expvarCollector) Describe(ch chan<- *Desc) { for _, desc := range e.exports { ch <- desc } } // Collect implements Collector. func (e *expvarCollector) Collect(ch chan<- Metric) { for name, desc := range e.exports { var m Metric expVar := expvar.Get(name) if expVar == nil { continue } var v interface{} labels := make([]string, len(desc.variableLabels)) if err := json.Unmarshal([]byte(expVar.String()), &v); err != nil { ch <- NewInvalidMetric(desc, err) continue } var processValue func(v interface{}, i int) processValue = func(v interface{}, i int) { if i >= len(labels) { copiedLabels := append(make([]string, 0, len(labels)), labels...) switch v := v.(type) { case float64: m = MustNewConstMetric(desc, UntypedValue, v, copiedLabels...) case bool: if v { m = MustNewConstMetric(desc, UntypedValue, 1, copiedLabels...) } else { m = MustNewConstMetric(desc, UntypedValue, 0, copiedLabels...) } default: return } ch <- m return } vm, ok := v.(map[string]interface{}) if !ok { return } for lv, val := range vm { labels[i] = lv processValue(val, i+1) } } processValue(v, 0) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/expvar_collector_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus_test import ( "expvar" "fmt" "sort" "strings" dto "github.com/prometheus/client_model/go" "github.com/prometheus/client_golang/prometheus" ) func ExampleNewExpvarCollector() { expvarCollector := prometheus.NewExpvarCollector(map[string]*prometheus.Desc{ "memstats": prometheus.NewDesc( "expvar_memstats", "All numeric memstats as one metric family. Not a good role-model, actually... ;-)", []string{"type"}, nil, ), "lone-int": prometheus.NewDesc( "expvar_lone_int", "Just an expvar int as an example.", nil, nil, ), "http-request-map": prometheus.NewDesc( "expvar_http_request_total", "How many http requests processed, partitioned by status code and http method.", []string{"code", "method"}, nil, ), }) prometheus.MustRegister(expvarCollector) // The Prometheus part is done here. But to show that this example is // doing anything, we have to manually export something via expvar. In // real-life use-cases, some library would already have exported via // expvar what we want to re-export as Prometheus metrics. expvar.NewInt("lone-int").Set(42) expvarMap := expvar.NewMap("http-request-map") var ( expvarMap1, expvarMap2 expvar.Map expvarInt11, expvarInt12, expvarInt21, expvarInt22 expvar.Int ) expvarMap1.Init() expvarMap2.Init() expvarInt11.Set(3) expvarInt12.Set(13) expvarInt21.Set(11) expvarInt22.Set(212) expvarMap1.Set("POST", &expvarInt11) expvarMap1.Set("GET", &expvarInt12) expvarMap2.Set("POST", &expvarInt21) expvarMap2.Set("GET", &expvarInt22) expvarMap.Set("404", &expvarMap1) expvarMap.Set("200", &expvarMap2) // Results in the following expvar map: // "http-request-count": {"200": {"POST": 11, "GET": 212}, "404": {"POST": 3, "GET": 13}} // Let's see what the scrape would yield, but exclude the memstats metrics. metricStrings := []string{} metric := dto.Metric{} metricChan := make(chan prometheus.Metric) go func() { expvarCollector.Collect(metricChan) close(metricChan) }() for m := range metricChan { if strings.Index(m.Desc().String(), "expvar_memstats") == -1 { metric.Reset() m.Write(&metric) metricStrings = append(metricStrings, metric.String()) } } sort.Strings(metricStrings) for _, s := range metricStrings { fmt.Println(strings.TrimRight(s, " ")) } // Output: // label: label: untyped: // label: label: untyped: // label: label: untyped: // label: label: untyped: // untyped: } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/fnv.go ================================================ package prometheus // Inline and byte-free variant of hash/fnv's fnv64a. const ( offset64 = 14695981039346656037 prime64 = 1099511628211 ) // hashNew initializies a new fnv64a hash value. func hashNew() uint64 { return offset64 } // hashAdd adds a string to a fnv64a hash value, returning the updated hash. func hashAdd(h uint64, s string) uint64 { for i := 0; i < len(s); i++ { h ^= uint64(s[i]) h *= prime64 } return h } // hashAddByte adds a byte to a fnv64a hash value, returning the updated hash. func hashAddByte(h uint64, b byte) uint64 { h ^= uint64(b) h *= prime64 return h } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/gauge.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus // Gauge is a Metric that represents a single numerical value that can // arbitrarily go up and down. // // A Gauge is typically used for measured values like temperatures or current // memory usage, but also "counts" that can go up and down, like the number of // running goroutines. // // To create Gauge instances, use NewGauge. type Gauge interface { Metric Collector // Set sets the Gauge to an arbitrary value. Set(float64) // Inc increments the Gauge by 1. Use Add to increment it by arbitrary // values. Inc() // Dec decrements the Gauge by 1. Use Sub to decrement it by arbitrary // values. Dec() // Add adds the given value to the Gauge. (The value can be negative, // resulting in a decrease of the Gauge.) Add(float64) // Sub subtracts the given value from the Gauge. (The value can be // negative, resulting in an increase of the Gauge.) Sub(float64) // SetToCurrentTime sets the Gauge to the current Unix time in seconds. SetToCurrentTime() } // GaugeOpts is an alias for Opts. See there for doc comments. type GaugeOpts Opts // NewGauge creates a new Gauge based on the provided GaugeOpts. func NewGauge(opts GaugeOpts) Gauge { return newValue(NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ), GaugeValue, 0) } // GaugeVec is a Collector that bundles a set of Gauges that all share the same // Desc, but have different values for their variable labels. This is used if // you want to count the same thing partitioned by various dimensions // (e.g. number of operations queued, partitioned by user and operation // type). Create instances with NewGaugeVec. type GaugeVec struct { *metricVec } // NewGaugeVec creates a new GaugeVec based on the provided GaugeOpts and // partitioned by the given label names. func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec { desc := NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, labelNames, opts.ConstLabels, ) return &GaugeVec{ metricVec: newMetricVec(desc, func(lvs ...string) Metric { return newValue(desc, GaugeValue, 0, lvs...) }), } } // GetMetricWithLabelValues returns the Gauge for the given slice of label // values (same order as the VariableLabels in Desc). If that combination of // label values is accessed for the first time, a new Gauge is created. // // It is possible to call this method without using the returned Gauge to only // create the new Gauge but leave it at its starting value 0. See also the // SummaryVec example. // // Keeping the Gauge for later use is possible (and should be considered if // performance is critical), but keep in mind that Reset, DeleteLabelValues and // Delete can be used to delete the Gauge from the GaugeVec. In that case, the // Gauge will still exist, but it will not be exported anymore, even if a // Gauge with the same label values is created later. See also the CounterVec // example. // // An error is returned if the number of label values is not the same as the // number of VariableLabels in Desc. // // Note that for more than one label value, this method is prone to mistakes // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // an alternative to avoid that type of mistake. For higher label numbers, the // latter has a much more readable (albeit more verbose) syntax, but it comes // with a performance overhead (for creating and processing the Labels map). func (m *GaugeVec) GetMetricWithLabelValues(lvs ...string) (Gauge, error) { metric, err := m.metricVec.getMetricWithLabelValues(lvs...) if metric != nil { return metric.(Gauge), err } return nil, err } // GetMetricWith returns the Gauge for the given Labels map (the label names // must match those of the VariableLabels in Desc). If that label map is // accessed for the first time, a new Gauge is created. Implications of // creating a Gauge without using it and keeping the Gauge for later use are // the same as for GetMetricWithLabelValues. // // An error is returned if the number and names of the Labels are inconsistent // with those of the VariableLabels in Desc. // // This method is used for the same purpose as // GetMetricWithLabelValues(...string). See there for pros and cons of the two // methods. func (m *GaugeVec) GetMetricWith(labels Labels) (Gauge, error) { metric, err := m.metricVec.getMetricWith(labels) if metric != nil { return metric.(Gauge), err } return nil, err } // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. By not returning an // error, WithLabelValues allows shortcuts like // myVec.WithLabelValues("404", "GET").Add(42) func (m *GaugeVec) WithLabelValues(lvs ...string) Gauge { return m.metricVec.withLabelValues(lvs...).(Gauge) } // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. By not returning an error, With allows shortcuts like // myVec.With(Labels{"code": "404", "method": "GET"}).Add(42) func (m *GaugeVec) With(labels Labels) Gauge { return m.metricVec.with(labels).(Gauge) } // GaugeFunc is a Gauge whose value is determined at collect time by calling a // provided function. // // To create GaugeFunc instances, use NewGaugeFunc. type GaugeFunc interface { Metric Collector } // NewGaugeFunc creates a new GaugeFunc based on the provided GaugeOpts. The // value reported is determined by calling the given function from within the // Write method. Take into account that metric collection may happen // concurrently. If that results in concurrent calls to Write, like in the case // where a GaugeFunc is directly registered with Prometheus, the provided // function must be concurrency-safe. func NewGaugeFunc(opts GaugeOpts, function func() float64) GaugeFunc { return newValueFunc(NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ), GaugeValue, function) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/gauge_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "math" "math/rand" "sync" "testing" "testing/quick" "time" dto "github.com/prometheus/client_model/go" ) func listenGaugeStream(vals, result chan float64, done chan struct{}) { var sum float64 outer: for { select { case <-done: close(vals) for v := range vals { sum += v } break outer case v := <-vals: sum += v } } result <- sum close(result) } func TestGaugeConcurrency(t *testing.T) { it := func(n uint32) bool { mutations := int(n % 10000) concLevel := int(n%15 + 1) var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) sStream := make(chan float64, mutations*concLevel) result := make(chan float64) done := make(chan struct{}) go listenGaugeStream(sStream, result, done) go func() { end.Wait() close(done) }() gge := NewGauge(GaugeOpts{ Name: "test_gauge", Help: "no help can be found here", }) for i := 0; i < concLevel; i++ { vals := make([]float64, mutations) for j := 0; j < mutations; j++ { vals[j] = rand.Float64() - 0.5 } go func(vals []float64) { start.Wait() for _, v := range vals { sStream <- v gge.Add(v) } end.Done() }(vals) } start.Done() if expected, got := <-result, math.Float64frombits(gge.(*value).valBits); math.Abs(expected-got) > 0.000001 { t.Fatalf("expected approx. %f, got %f", expected, got) return false } return true } if err := quick.Check(it, nil); err != nil { t.Fatal(err) } } func TestGaugeVecConcurrency(t *testing.T) { it := func(n uint32) bool { mutations := int(n % 10000) concLevel := int(n%15 + 1) vecLength := int(n%5 + 1) var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) sStreams := make([]chan float64, vecLength) results := make([]chan float64, vecLength) done := make(chan struct{}) for i := 0; i < vecLength; i++ { sStreams[i] = make(chan float64, mutations*concLevel) results[i] = make(chan float64) go listenGaugeStream(sStreams[i], results[i], done) } go func() { end.Wait() close(done) }() gge := NewGaugeVec( GaugeOpts{ Name: "test_gauge", Help: "no help can be found here", }, []string{"label"}, ) for i := 0; i < concLevel; i++ { vals := make([]float64, mutations) pick := make([]int, mutations) for j := 0; j < mutations; j++ { vals[j] = rand.Float64() - 0.5 pick[j] = rand.Intn(vecLength) } go func(vals []float64) { start.Wait() for i, v := range vals { sStreams[pick[i]] <- v gge.WithLabelValues(string('A' + pick[i])).Add(v) } end.Done() }(vals) } start.Done() for i := range sStreams { if expected, got := <-results[i], math.Float64frombits(gge.WithLabelValues(string('A'+i)).(*value).valBits); math.Abs(expected-got) > 0.000001 { t.Fatalf("expected approx. %f, got %f", expected, got) return false } } return true } if err := quick.Check(it, nil); err != nil { t.Fatal(err) } } func TestGaugeFunc(t *testing.T) { gf := NewGaugeFunc( GaugeOpts{ Name: "test_name", Help: "test help", ConstLabels: Labels{"a": "1", "b": "2"}, }, func() float64 { return 3.1415 }, ) if expected, got := `Desc{fqName: "test_name", help: "test help", constLabels: {a="1",b="2"}, variableLabels: []}`, gf.Desc().String(); expected != got { t.Errorf("expected %q, got %q", expected, got) } m := &dto.Metric{} gf.Write(m) if expected, got := `label: label: gauge: `, m.String(); expected != got { t.Errorf("expected %q, got %q", expected, got) } } func TestGaugeSetCurrentTime(t *testing.T) { g := NewGauge(GaugeOpts{ Name: "test_name", Help: "test help", }) g.SetToCurrentTime() unixTime := float64(time.Now().Unix()) m := &dto.Metric{} g.Write(m) delta := unixTime - m.GetGauge().GetValue() // This is just a smoke test to make sure SetToCurrentTime is not // totally off. Tests with current time involved are hard... if math.Abs(delta) > 5 { t.Errorf("Gauge set to current time deviates from current time by more than 5s, delta is %f seconds", delta) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/go_collector.go ================================================ package prometheus import ( "fmt" "runtime" "runtime/debug" "time" ) type goCollector struct { goroutinesDesc *Desc threadsDesc *Desc gcDesc *Desc goInfoDesc *Desc // metrics to describe and collect metrics memStatsMetrics } // NewGoCollector returns a collector which exports metrics about the current // go process. func NewGoCollector() Collector { return &goCollector{ goroutinesDesc: NewDesc( "go_goroutines", "Number of goroutines that currently exist.", nil, nil), threadsDesc: NewDesc( "go_threads", "Number of OS threads created.", nil, nil), gcDesc: NewDesc( "go_gc_duration_seconds", "A summary of the GC invocation durations.", nil, nil), goInfoDesc: NewDesc( "go_info", "Information about the Go environment.", nil, Labels{"version": runtime.Version()}), metrics: memStatsMetrics{ { desc: NewDesc( memstatNamespace("alloc_bytes"), "Number of bytes allocated and still in use.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("alloc_bytes_total"), "Total number of bytes allocated, even if freed.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) }, valType: CounterValue, }, { desc: NewDesc( memstatNamespace("sys_bytes"), "Number of bytes obtained from system.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.Sys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("lookups_total"), "Total number of pointer lookups.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) }, valType: CounterValue, }, { desc: NewDesc( memstatNamespace("mallocs_total"), "Total number of mallocs.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) }, valType: CounterValue, }, { desc: NewDesc( memstatNamespace("frees_total"), "Total number of frees.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.Frees) }, valType: CounterValue, }, { desc: NewDesc( memstatNamespace("heap_alloc_bytes"), "Number of heap bytes allocated and still in use.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("heap_sys_bytes"), "Number of heap bytes obtained from system.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("heap_idle_bytes"), "Number of heap bytes waiting to be used.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("heap_inuse_bytes"), "Number of heap bytes that are in use.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("heap_released_bytes"), "Number of heap bytes released to OS.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("heap_objects"), "Number of allocated objects.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("stack_inuse_bytes"), "Number of bytes in use by the stack allocator.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("stack_sys_bytes"), "Number of bytes obtained from system for stack allocator.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("mspan_inuse_bytes"), "Number of bytes in use by mspan structures.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("mspan_sys_bytes"), "Number of bytes used for mspan structures obtained from system.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("mcache_inuse_bytes"), "Number of bytes in use by mcache structures.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("mcache_sys_bytes"), "Number of bytes used for mcache structures obtained from system.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("buck_hash_sys_bytes"), "Number of bytes used by the profiling bucket hash table.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("gc_sys_bytes"), "Number of bytes used for garbage collection system metadata.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("other_sys_bytes"), "Number of bytes used for other system allocations.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("next_gc_bytes"), "Number of heap bytes when next garbage collection will take place.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("last_gc_time_seconds"), "Number of seconds since 1970 of last garbage collection.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return float64(ms.LastGC) / 1e9 }, valType: GaugeValue, }, { desc: NewDesc( memstatNamespace("gc_cpu_fraction"), "The fraction of this program's available CPU time used by the GC since the program started.", nil, nil, ), eval: func(ms *runtime.MemStats) float64 { return ms.GCCPUFraction }, valType: GaugeValue, }, }, } } func memstatNamespace(s string) string { return fmt.Sprintf("go_memstats_%s", s) } // Describe returns all descriptions of the collector. func (c *goCollector) Describe(ch chan<- *Desc) { ch <- c.goroutinesDesc ch <- c.threadsDesc ch <- c.gcDesc ch <- c.goInfoDesc for _, i := range c.metrics { ch <- i.desc } } // Collect returns the current state of all metrics of the collector. func (c *goCollector) Collect(ch chan<- Metric) { ch <- MustNewConstMetric(c.goroutinesDesc, GaugeValue, float64(runtime.NumGoroutine())) n, _ := runtime.ThreadCreateProfile(nil) ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, float64(n)) var stats debug.GCStats stats.PauseQuantiles = make([]time.Duration, 5) debug.ReadGCStats(&stats) quantiles := make(map[float64]float64) for idx, pq := range stats.PauseQuantiles[1:] { quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds() } quantiles[0.0] = stats.PauseQuantiles[0].Seconds() ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles) ch <- MustNewConstMetric(c.goInfoDesc, GaugeValue, 1) ms := &runtime.MemStats{} runtime.ReadMemStats(ms) for _, i := range c.metrics { ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms)) } } // memStatsMetrics provide description, value, and value type for memstat metrics. type memStatsMetrics []struct { desc *Desc eval func(*runtime.MemStats) float64 valType ValueType } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/go_collector_test.go ================================================ package prometheus import ( "runtime" "testing" "time" dto "github.com/prometheus/client_model/go" ) func TestGoCollector(t *testing.T) { var ( c = NewGoCollector() ch = make(chan Metric) waitc = make(chan struct{}) closec = make(chan struct{}) old = -1 ) defer close(closec) go func() { c.Collect(ch) go func(c <-chan struct{}) { <-c }(closec) <-waitc c.Collect(ch) }() for { select { case m := <-ch: // m can be Gauge or Counter, // currently just test the go_goroutines Gauge // and ignore others. if m.Desc().fqName != "go_goroutines" { continue } pb := &dto.Metric{} m.Write(pb) if pb.GetGauge() == nil { continue } if old == -1 { old = int(pb.GetGauge().GetValue()) close(waitc) continue } if diff := int(pb.GetGauge().GetValue()) - old; diff != 1 { // TODO: This is flaky in highly concurrent situations. t.Errorf("want 1 new goroutine, got %d", diff) } // GoCollector performs three sends per call. // On line 27 we need to receive three more sends // to shut down cleanly. <-ch <-ch <-ch return case <-time.After(1 * time.Second): t.Fatalf("expected collect timed out") } } } func TestGCCollector(t *testing.T) { var ( c = NewGoCollector() ch = make(chan Metric) waitc = make(chan struct{}) closec = make(chan struct{}) oldGC uint64 oldPause float64 ) defer close(closec) go func() { c.Collect(ch) // force GC runtime.GC() <-waitc c.Collect(ch) }() first := true for { select { case metric := <-ch: switch m := metric.(type) { case *constSummary, *value: pb := &dto.Metric{} m.Write(pb) if pb.GetSummary() == nil { continue } if len(pb.GetSummary().Quantile) != 5 { t.Errorf("expected 4 buckets, got %d", len(pb.GetSummary().Quantile)) } for idx, want := range []float64{0.0, 0.25, 0.5, 0.75, 1.0} { if *pb.GetSummary().Quantile[idx].Quantile != want { t.Errorf("bucket #%d is off, got %f, want %f", idx, *pb.GetSummary().Quantile[idx].Quantile, want) } } if first { first = false oldGC = *pb.GetSummary().SampleCount oldPause = *pb.GetSummary().SampleSum close(waitc) continue } if diff := *pb.GetSummary().SampleCount - oldGC; diff != 1 { t.Errorf("want 1 new garbage collection run, got %d", diff) } if diff := *pb.GetSummary().SampleSum - oldPause; diff <= 0 { t.Errorf("want moar pause, got %f", diff) } return } case <-time.After(1 * time.Second): t.Fatalf("expected collect timed out") } } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/histogram.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "fmt" "math" "sort" "sync/atomic" "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) // A Histogram counts individual observations from an event or sample stream in // configurable buckets. Similar to a summary, it also provides a sum of // observations and an observation count. // // On the Prometheus server, quantiles can be calculated from a Histogram using // the histogram_quantile function in the query language. // // Note that Histograms, in contrast to Summaries, can be aggregated with the // Prometheus query language (see the documentation for detailed // procedures). However, Histograms require the user to pre-define suitable // buckets, and they are in general less accurate. The Observe method of a // Histogram has a very low performance overhead in comparison with the Observe // method of a Summary. // // To create Histogram instances, use NewHistogram. type Histogram interface { Metric Collector // Observe adds a single observation to the histogram. Observe(float64) } // bucketLabel is used for the label that defines the upper bound of a // bucket of a histogram ("le" -> "less or equal"). const bucketLabel = "le" // DefBuckets are the default Histogram buckets. The default buckets are // tailored to broadly measure the response time (in seconds) of a network // service. Most likely, however, you will be required to define buckets // customized to your use case. var ( DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} errBucketLabelNotAllowed = fmt.Errorf( "%q is not allowed as label name in histograms", bucketLabel, ) ) // LinearBuckets creates 'count' buckets, each 'width' wide, where the lowest // bucket has an upper bound of 'start'. The final +Inf bucket is not counted // and not included in the returned slice. The returned slice is meant to be // used for the Buckets field of HistogramOpts. // // The function panics if 'count' is zero or negative. func LinearBuckets(start, width float64, count int) []float64 { if count < 1 { panic("LinearBuckets needs a positive count") } buckets := make([]float64, count) for i := range buckets { buckets[i] = start start += width } return buckets } // ExponentialBuckets creates 'count' buckets, where the lowest bucket has an // upper bound of 'start' and each following bucket's upper bound is 'factor' // times the previous bucket's upper bound. The final +Inf bucket is not counted // and not included in the returned slice. The returned slice is meant to be // used for the Buckets field of HistogramOpts. // // The function panics if 'count' is 0 or negative, if 'start' is 0 or negative, // or if 'factor' is less than or equal 1. func ExponentialBuckets(start, factor float64, count int) []float64 { if count < 1 { panic("ExponentialBuckets needs a positive count") } if start <= 0 { panic("ExponentialBuckets needs a positive start value") } if factor <= 1 { panic("ExponentialBuckets needs a factor greater than 1") } buckets := make([]float64, count) for i := range buckets { buckets[i] = start start *= factor } return buckets } // HistogramOpts bundles the options for creating a Histogram metric. It is // mandatory to set Name and Help to a non-empty string. All other fields are // optional and can safely be left at their zero value. type HistogramOpts struct { // Namespace, Subsystem, and Name are components of the fully-qualified // name of the Histogram (created by joining these components with // "_"). Only Name is mandatory, the others merely help structuring the // name. Note that the fully-qualified name of the Histogram must be a // valid Prometheus metric name. Namespace string Subsystem string Name string // Help provides information about this Histogram. Mandatory! // // Metrics with the same fully-qualified name must have the same Help // string. Help string // ConstLabels are used to attach fixed labels to this // Histogram. Histograms with the same fully-qualified name must have the // same label names in their ConstLabels. // // Note that in most cases, labels have a value that varies during the // lifetime of a process. Those labels are usually managed with a // HistogramVec. ConstLabels serve only special purposes. One is for the // special case where the value of a label does not change during the // lifetime of a process, e.g. if the revision of the running binary is // put into a label. Another, more advanced purpose is if more than one // Collector needs to collect Histograms with the same fully-qualified // name. In that case, those Summaries must differ in the values of // their ConstLabels. See the Collector examples. // // If the value of a label never changes (not even between binaries), // that label most likely should not be a label at all (but part of the // metric name). ConstLabels Labels // Buckets defines the buckets into which observations are counted. Each // element in the slice is the upper inclusive bound of a bucket. The // values must be sorted in strictly increasing order. There is no need // to add a highest bucket with +Inf bound, it will be added // implicitly. The default value is DefBuckets. Buckets []float64 } // NewHistogram creates a new Histogram based on the provided HistogramOpts. It // panics if the buckets in HistogramOpts are not in strictly increasing order. func NewHistogram(opts HistogramOpts) Histogram { return newHistogram( NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ), opts, ) } func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogram { if len(desc.variableLabels) != len(labelValues) { panic(errInconsistentCardinality) } for _, n := range desc.variableLabels { if n == bucketLabel { panic(errBucketLabelNotAllowed) } } for _, lp := range desc.constLabelPairs { if lp.GetName() == bucketLabel { panic(errBucketLabelNotAllowed) } } if len(opts.Buckets) == 0 { opts.Buckets = DefBuckets } h := &histogram{ desc: desc, upperBounds: opts.Buckets, labelPairs: makeLabelPairs(desc, labelValues), } for i, upperBound := range h.upperBounds { if i < len(h.upperBounds)-1 { if upperBound >= h.upperBounds[i+1] { panic(fmt.Errorf( "histogram buckets must be in increasing order: %f >= %f", upperBound, h.upperBounds[i+1], )) } } else { if math.IsInf(upperBound, +1) { // The +Inf bucket is implicit. Remove it here. h.upperBounds = h.upperBounds[:i] } } } // Finally we know the final length of h.upperBounds and can make counts. h.counts = make([]uint64, len(h.upperBounds)) h.init(h) // Init self-collection. return h } type histogram struct { // sumBits contains the bits of the float64 representing the sum of all // observations. sumBits and count have to go first in the struct to // guarantee alignment for atomic operations. // http://golang.org/pkg/sync/atomic/#pkg-note-BUG sumBits uint64 count uint64 selfCollector // Note that there is no mutex required. desc *Desc upperBounds []float64 counts []uint64 labelPairs []*dto.LabelPair } func (h *histogram) Desc() *Desc { return h.desc } func (h *histogram) Observe(v float64) { // TODO(beorn7): For small numbers of buckets (<30), a linear search is // slightly faster than the binary search. If we really care, we could // switch from one search strategy to the other depending on the number // of buckets. // // Microbenchmarks (BenchmarkHistogramNoLabels): // 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op // 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op // 300 buckets: 154 ns/op linear - binary 61.6 ns/op i := sort.SearchFloat64s(h.upperBounds, v) if i < len(h.counts) { atomic.AddUint64(&h.counts[i], 1) } atomic.AddUint64(&h.count, 1) for { oldBits := atomic.LoadUint64(&h.sumBits) newBits := math.Float64bits(math.Float64frombits(oldBits) + v) if atomic.CompareAndSwapUint64(&h.sumBits, oldBits, newBits) { break } } } func (h *histogram) Write(out *dto.Metric) error { his := &dto.Histogram{} buckets := make([]*dto.Bucket, len(h.upperBounds)) his.SampleSum = proto.Float64(math.Float64frombits(atomic.LoadUint64(&h.sumBits))) his.SampleCount = proto.Uint64(atomic.LoadUint64(&h.count)) var count uint64 for i, upperBound := range h.upperBounds { count += atomic.LoadUint64(&h.counts[i]) buckets[i] = &dto.Bucket{ CumulativeCount: proto.Uint64(count), UpperBound: proto.Float64(upperBound), } } his.Bucket = buckets out.Histogram = his out.Label = h.labelPairs return nil } // HistogramVec is a Collector that bundles a set of Histograms that all share the // same Desc, but have different values for their variable labels. This is used // if you want to count the same thing partitioned by various dimensions // (e.g. HTTP request latencies, partitioned by status code and method). Create // instances with NewHistogramVec. type HistogramVec struct { *metricVec } // NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and // partitioned by the given label names. func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { desc := NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, labelNames, opts.ConstLabels, ) return &HistogramVec{ metricVec: newMetricVec(desc, func(lvs ...string) Metric { return newHistogram(desc, opts, lvs...) }), } } // GetMetricWithLabelValues returns the Histogram for the given slice of label // values (same order as the VariableLabels in Desc). If that combination of // label values is accessed for the first time, a new Histogram is created. // // It is possible to call this method without using the returned Histogram to only // create the new Histogram but leave it at its starting value, a Histogram without // any observations. // // Keeping the Histogram for later use is possible (and should be considered if // performance is critical), but keep in mind that Reset, DeleteLabelValues and // Delete can be used to delete the Histogram from the HistogramVec. In that case, the // Histogram will still exist, but it will not be exported anymore, even if a // Histogram with the same label values is created later. See also the CounterVec // example. // // An error is returned if the number of label values is not the same as the // number of VariableLabels in Desc. // // Note that for more than one label value, this method is prone to mistakes // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // an alternative to avoid that type of mistake. For higher label numbers, the // latter has a much more readable (albeit more verbose) syntax, but it comes // with a performance overhead (for creating and processing the Labels map). // See also the GaugeVec example. func (m *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) { metric, err := m.metricVec.getMetricWithLabelValues(lvs...) if metric != nil { return metric.(Observer), err } return nil, err } // GetMetricWith returns the Histogram for the given Labels map (the label names // must match those of the VariableLabels in Desc). If that label map is // accessed for the first time, a new Histogram is created. Implications of // creating a Histogram without using it and keeping the Histogram for later use // are the same as for GetMetricWithLabelValues. // // An error is returned if the number and names of the Labels are inconsistent // with those of the VariableLabels in Desc. // // This method is used for the same purpose as // GetMetricWithLabelValues(...string). See there for pros and cons of the two // methods. func (m *HistogramVec) GetMetricWith(labels Labels) (Observer, error) { metric, err := m.metricVec.getMetricWith(labels) if metric != nil { return metric.(Observer), err } return nil, err } // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. By not returning an // error, WithLabelValues allows shortcuts like // myVec.WithLabelValues("404", "GET").Observe(42.21) func (m *HistogramVec) WithLabelValues(lvs ...string) Observer { return m.metricVec.withLabelValues(lvs...).(Observer) } // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. By not returning an error, With allows shortcuts like // myVec.With(Labels{"code": "404", "method": "GET"}).Observe(42.21) func (m *HistogramVec) With(labels Labels) Observer { return m.metricVec.with(labels).(Observer) } type constHistogram struct { desc *Desc count uint64 sum float64 buckets map[float64]uint64 labelPairs []*dto.LabelPair } func (h *constHistogram) Desc() *Desc { return h.desc } func (h *constHistogram) Write(out *dto.Metric) error { his := &dto.Histogram{} buckets := make([]*dto.Bucket, 0, len(h.buckets)) his.SampleCount = proto.Uint64(h.count) his.SampleSum = proto.Float64(h.sum) for upperBound, count := range h.buckets { buckets = append(buckets, &dto.Bucket{ CumulativeCount: proto.Uint64(count), UpperBound: proto.Float64(upperBound), }) } if len(buckets) > 0 { sort.Sort(buckSort(buckets)) } his.Bucket = buckets out.Histogram = his out.Label = h.labelPairs return nil } // NewConstHistogram returns a metric representing a Prometheus histogram with // fixed values for the count, sum, and bucket counts. As those parameters // cannot be changed, the returned value does not implement the Histogram // interface (but only the Metric interface). Users of this package will not // have much use for it in regular operations. However, when implementing custom // Collectors, it is useful as a throw-away metric that is generated on the fly // to send it to Prometheus in the Collect method. // // buckets is a map of upper bounds to cumulative counts, excluding the +Inf // bucket. // // NewConstHistogram returns an error if the length of labelValues is not // consistent with the variable labels in Desc. func NewConstHistogram( desc *Desc, count uint64, sum float64, buckets map[float64]uint64, labelValues ...string, ) (Metric, error) { if err := validateLabelValues(labelValues, len(desc.variableLabels)); err != nil { return nil, err } return &constHistogram{ desc: desc, count: count, sum: sum, buckets: buckets, labelPairs: makeLabelPairs(desc, labelValues), }, nil } // MustNewConstHistogram is a version of NewConstHistogram that panics where // NewConstMetric would have returned an error. func MustNewConstHistogram( desc *Desc, count uint64, sum float64, buckets map[float64]uint64, labelValues ...string, ) Metric { m, err := NewConstHistogram(desc, count, sum, buckets, labelValues...) if err != nil { panic(err) } return m } type buckSort []*dto.Bucket func (s buckSort) Len() int { return len(s) } func (s buckSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s buckSort) Less(i, j int) bool { return s[i].GetUpperBound() < s[j].GetUpperBound() } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/histogram_test.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "math" "math/rand" "reflect" "sort" "sync" "testing" "testing/quick" dto "github.com/prometheus/client_model/go" ) func benchmarkHistogramObserve(w int, b *testing.B) { b.StopTimer() wg := new(sync.WaitGroup) wg.Add(w) g := new(sync.WaitGroup) g.Add(1) s := NewHistogram(HistogramOpts{}) for i := 0; i < w; i++ { go func() { g.Wait() for i := 0; i < b.N; i++ { s.Observe(float64(i)) } wg.Done() }() } b.StartTimer() g.Done() wg.Wait() } func BenchmarkHistogramObserve1(b *testing.B) { benchmarkHistogramObserve(1, b) } func BenchmarkHistogramObserve2(b *testing.B) { benchmarkHistogramObserve(2, b) } func BenchmarkHistogramObserve4(b *testing.B) { benchmarkHistogramObserve(4, b) } func BenchmarkHistogramObserve8(b *testing.B) { benchmarkHistogramObserve(8, b) } func benchmarkHistogramWrite(w int, b *testing.B) { b.StopTimer() wg := new(sync.WaitGroup) wg.Add(w) g := new(sync.WaitGroup) g.Add(1) s := NewHistogram(HistogramOpts{}) for i := 0; i < 1000000; i++ { s.Observe(float64(i)) } for j := 0; j < w; j++ { outs := make([]dto.Metric, b.N) go func(o []dto.Metric) { g.Wait() for i := 0; i < b.N; i++ { s.Write(&o[i]) } wg.Done() }(outs) } b.StartTimer() g.Done() wg.Wait() } func BenchmarkHistogramWrite1(b *testing.B) { benchmarkHistogramWrite(1, b) } func BenchmarkHistogramWrite2(b *testing.B) { benchmarkHistogramWrite(2, b) } func BenchmarkHistogramWrite4(b *testing.B) { benchmarkHistogramWrite(4, b) } func BenchmarkHistogramWrite8(b *testing.B) { benchmarkHistogramWrite(8, b) } func TestHistogramNonMonotonicBuckets(t *testing.T) { testCases := map[string][]float64{ "not strictly monotonic": {1, 2, 2, 3}, "not monotonic at all": {1, 2, 4, 3, 5}, "have +Inf in the middle": {1, 2, math.Inf(+1), 3}, } for name, buckets := range testCases { func() { defer func() { if r := recover(); r == nil { t.Errorf("Buckets %v are %s but NewHistogram did not panic.", buckets, name) } }() _ = NewHistogram(HistogramOpts{ Name: "test_histogram", Help: "helpless", Buckets: buckets, }) }() } } // Intentionally adding +Inf here to test if that case is handled correctly. // Also, getCumulativeCounts depends on it. var testBuckets = []float64{-2, -1, -0.5, 0, 0.5, 1, 2, math.Inf(+1)} func TestHistogramConcurrency(t *testing.T) { if testing.Short() { t.Skip("Skipping test in short mode.") } rand.Seed(42) it := func(n uint32) bool { mutations := int(n%1e4 + 1e4) concLevel := int(n%5 + 1) total := mutations * concLevel var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) sum := NewHistogram(HistogramOpts{ Name: "test_histogram", Help: "helpless", Buckets: testBuckets, }) allVars := make([]float64, total) var sampleSum float64 for i := 0; i < concLevel; i++ { vals := make([]float64, mutations) for j := 0; j < mutations; j++ { v := rand.NormFloat64() vals[j] = v allVars[i*mutations+j] = v sampleSum += v } go func(vals []float64) { start.Wait() for _, v := range vals { sum.Observe(v) } end.Done() }(vals) } sort.Float64s(allVars) start.Done() end.Wait() m := &dto.Metric{} sum.Write(m) if got, want := int(*m.Histogram.SampleCount), total; got != want { t.Errorf("got sample count %d, want %d", got, want) } if got, want := *m.Histogram.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 { t.Errorf("got sample sum %f, want %f", got, want) } wantCounts := getCumulativeCounts(allVars) if got, want := len(m.Histogram.Bucket), len(testBuckets)-1; got != want { t.Errorf("got %d buckets in protobuf, want %d", got, want) } for i, wantBound := range testBuckets { if i == len(testBuckets)-1 { break // No +Inf bucket in protobuf. } if gotBound := *m.Histogram.Bucket[i].UpperBound; gotBound != wantBound { t.Errorf("got bound %f, want %f", gotBound, wantBound) } if gotCount, wantCount := *m.Histogram.Bucket[i].CumulativeCount, wantCounts[i]; gotCount != wantCount { t.Errorf("got count %d, want %d", gotCount, wantCount) } } return true } if err := quick.Check(it, nil); err != nil { t.Error(err) } } func TestHistogramVecConcurrency(t *testing.T) { if testing.Short() { t.Skip("Skipping test in short mode.") } rand.Seed(42) objectives := make([]float64, 0, len(DefObjectives)) for qu := range DefObjectives { objectives = append(objectives, qu) } sort.Float64s(objectives) it := func(n uint32) bool { mutations := int(n%1e4 + 1e4) concLevel := int(n%7 + 1) vecLength := int(n%3 + 1) var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) his := NewHistogramVec( HistogramOpts{ Name: "test_histogram", Help: "helpless", Buckets: []float64{-2, -1, -0.5, 0, 0.5, 1, 2, math.Inf(+1)}, }, []string{"label"}, ) allVars := make([][]float64, vecLength) sampleSums := make([]float64, vecLength) for i := 0; i < concLevel; i++ { vals := make([]float64, mutations) picks := make([]int, mutations) for j := 0; j < mutations; j++ { v := rand.NormFloat64() vals[j] = v pick := rand.Intn(vecLength) picks[j] = pick allVars[pick] = append(allVars[pick], v) sampleSums[pick] += v } go func(vals []float64) { start.Wait() for i, v := range vals { his.WithLabelValues(string('A' + picks[i])).Observe(v) } end.Done() }(vals) } for _, vars := range allVars { sort.Float64s(vars) } start.Done() end.Wait() for i := 0; i < vecLength; i++ { m := &dto.Metric{} s := his.WithLabelValues(string('A' + i)) s.(Histogram).Write(m) if got, want := len(m.Histogram.Bucket), len(testBuckets)-1; got != want { t.Errorf("got %d buckets in protobuf, want %d", got, want) } if got, want := int(*m.Histogram.SampleCount), len(allVars[i]); got != want { t.Errorf("got sample count %d, want %d", got, want) } if got, want := *m.Histogram.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 { t.Errorf("got sample sum %f, want %f", got, want) } wantCounts := getCumulativeCounts(allVars[i]) for j, wantBound := range testBuckets { if j == len(testBuckets)-1 { break // No +Inf bucket in protobuf. } if gotBound := *m.Histogram.Bucket[j].UpperBound; gotBound != wantBound { t.Errorf("got bound %f, want %f", gotBound, wantBound) } if gotCount, wantCount := *m.Histogram.Bucket[j].CumulativeCount, wantCounts[j]; gotCount != wantCount { t.Errorf("got count %d, want %d", gotCount, wantCount) } } } return true } if err := quick.Check(it, nil); err != nil { t.Error(err) } } func getCumulativeCounts(vars []float64) []uint64 { counts := make([]uint64, len(testBuckets)) for _, v := range vars { for i := len(testBuckets) - 1; i >= 0; i-- { if v > testBuckets[i] { break } counts[i]++ } } return counts } func TestBuckets(t *testing.T) { got := LinearBuckets(-15, 5, 6) want := []float64{-15, -10, -5, 0, 5, 10} if !reflect.DeepEqual(got, want) { t.Errorf("linear buckets: got %v, want %v", got, want) } got = ExponentialBuckets(100, 1.2, 3) want = []float64{100, 120, 144} if !reflect.DeepEqual(got, want) { t.Errorf("linear buckets: got %v, want %v", got, want) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/http.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "bufio" "bytes" "compress/gzip" "fmt" "io" "net" "net/http" "strconv" "strings" "sync" "time" "github.com/prometheus/common/expfmt" ) // TODO(beorn7): Remove this whole file. It is a partial mirror of // promhttp/http.go (to avoid circular import chains) where everything HTTP // related should live. The functions here are just for avoiding // breakage. Everything is deprecated. const ( contentTypeHeader = "Content-Type" contentLengthHeader = "Content-Length" contentEncodingHeader = "Content-Encoding" acceptEncodingHeader = "Accept-Encoding" ) var bufPool sync.Pool func getBuf() *bytes.Buffer { buf := bufPool.Get() if buf == nil { return &bytes.Buffer{} } return buf.(*bytes.Buffer) } func giveBuf(buf *bytes.Buffer) { buf.Reset() bufPool.Put(buf) } // Handler returns an HTTP handler for the DefaultGatherer. It is // already instrumented with InstrumentHandler (using "prometheus" as handler // name). // // Deprecated: Please note the issues described in the doc comment of // InstrumentHandler. You might want to consider using promhttp.Handler instead // (which is not instrumented, but can be instrumented with the tooling provided // in package promhttp). func Handler() http.Handler { return InstrumentHandler("prometheus", UninstrumentedHandler()) } // UninstrumentedHandler returns an HTTP handler for the DefaultGatherer. // // Deprecated: Use promhttp.Handler instead. See there for further documentation. func UninstrumentedHandler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { mfs, err := DefaultGatherer.Gather() if err != nil { http.Error(w, "An error has occurred during metrics collection:\n\n"+err.Error(), http.StatusInternalServerError) return } contentType := expfmt.Negotiate(req.Header) buf := getBuf() defer giveBuf(buf) writer, encoding := decorateWriter(req, buf) enc := expfmt.NewEncoder(writer, contentType) var lastErr error for _, mf := range mfs { if err := enc.Encode(mf); err != nil { lastErr = err http.Error(w, "An error has occurred during metrics encoding:\n\n"+err.Error(), http.StatusInternalServerError) return } } if closer, ok := writer.(io.Closer); ok { closer.Close() } if lastErr != nil && buf.Len() == 0 { http.Error(w, "No metrics encoded, last error:\n\n"+lastErr.Error(), http.StatusInternalServerError) return } header := w.Header() header.Set(contentTypeHeader, string(contentType)) header.Set(contentLengthHeader, fmt.Sprint(buf.Len())) if encoding != "" { header.Set(contentEncodingHeader, encoding) } w.Write(buf.Bytes()) }) } // decorateWriter wraps a writer to handle gzip compression if requested. It // returns the decorated writer and the appropriate "Content-Encoding" header // (which is empty if no compression is enabled). func decorateWriter(request *http.Request, writer io.Writer) (io.Writer, string) { header := request.Header.Get(acceptEncodingHeader) parts := strings.Split(header, ",") for _, part := range parts { part := strings.TrimSpace(part) if part == "gzip" || strings.HasPrefix(part, "gzip;") { return gzip.NewWriter(writer), "gzip" } } return writer, "" } var instLabels = []string{"method", "code"} type nower interface { Now() time.Time } type nowFunc func() time.Time func (n nowFunc) Now() time.Time { return n() } var now nower = nowFunc(func() time.Time { return time.Now() }) func nowSeries(t ...time.Time) nower { return nowFunc(func() time.Time { defer func() { t = t[1:] }() return t[0] }) } // InstrumentHandler wraps the given HTTP handler for instrumentation. It // registers four metric collectors (if not already done) and reports HTTP // metrics to the (newly or already) registered collectors: http_requests_total // (CounterVec), http_request_duration_microseconds (Summary), // http_request_size_bytes (Summary), http_response_size_bytes (Summary). Each // has a constant label named "handler" with the provided handlerName as // value. http_requests_total is a metric vector partitioned by HTTP method // (label name "method") and HTTP status code (label name "code"). // // Deprecated: InstrumentHandler has several issues. Use the tooling provided in // package promhttp instead. The issues are the following: // // - It uses Summaries rather than Histograms. Summaries are not useful if // aggregation across multiple instances is required. // // - It uses microseconds as unit, which is deprecated and should be replaced by // seconds. // // - The size of the request is calculated in a separate goroutine. Since this // calculator requires access to the request header, it creates a race with // any writes to the header performed during request handling. // httputil.ReverseProxy is a prominent example for a handler // performing such writes. // // - It has additional issues with HTTP/2, cf. // https://github.com/prometheus/client_golang/issues/272. func InstrumentHandler(handlerName string, handler http.Handler) http.HandlerFunc { return InstrumentHandlerFunc(handlerName, handler.ServeHTTP) } // InstrumentHandlerFunc wraps the given function for instrumentation. It // otherwise works in the same way as InstrumentHandler (and shares the same // issues). // // Deprecated: InstrumentHandlerFunc is deprecated for the same reasons as // InstrumentHandler is. Use the tooling provided in package promhttp instead. func InstrumentHandlerFunc(handlerName string, handlerFunc func(http.ResponseWriter, *http.Request)) http.HandlerFunc { return InstrumentHandlerFuncWithOpts( SummaryOpts{ Subsystem: "http", ConstLabels: Labels{"handler": handlerName}, Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, handlerFunc, ) } // InstrumentHandlerWithOpts works like InstrumentHandler (and shares the same // issues) but provides more flexibility (at the cost of a more complex call // syntax). As InstrumentHandler, this function registers four metric // collectors, but it uses the provided SummaryOpts to create them. However, the // fields "Name" and "Help" in the SummaryOpts are ignored. "Name" is replaced // by "requests_total", "request_duration_microseconds", "request_size_bytes", // and "response_size_bytes", respectively. "Help" is replaced by an appropriate // help string. The names of the variable labels of the http_requests_total // CounterVec are "method" (get, post, etc.), and "code" (HTTP status code). // // If InstrumentHandlerWithOpts is called as follows, it mimics exactly the // behavior of InstrumentHandler: // // prometheus.InstrumentHandlerWithOpts( // prometheus.SummaryOpts{ // Subsystem: "http", // ConstLabels: prometheus.Labels{"handler": handlerName}, // }, // handler, // ) // // Technical detail: "requests_total" is a CounterVec, not a SummaryVec, so it // cannot use SummaryOpts. Instead, a CounterOpts struct is created internally, // and all its fields are set to the equally named fields in the provided // SummaryOpts. // // Deprecated: InstrumentHandlerWithOpts is deprecated for the same reasons as // InstrumentHandler is. Use the tooling provided in package promhttp instead. func InstrumentHandlerWithOpts(opts SummaryOpts, handler http.Handler) http.HandlerFunc { return InstrumentHandlerFuncWithOpts(opts, handler.ServeHTTP) } // InstrumentHandlerFuncWithOpts works like InstrumentHandlerFunc (and shares // the same issues) but provides more flexibility (at the cost of a more complex // call syntax). See InstrumentHandlerWithOpts for details how the provided // SummaryOpts are used. // // Deprecated: InstrumentHandlerFuncWithOpts is deprecated for the same reasons // as InstrumentHandler is. Use the tooling provided in package promhttp instead. func InstrumentHandlerFuncWithOpts(opts SummaryOpts, handlerFunc func(http.ResponseWriter, *http.Request)) http.HandlerFunc { reqCnt := NewCounterVec( CounterOpts{ Namespace: opts.Namespace, Subsystem: opts.Subsystem, Name: "requests_total", Help: "Total number of HTTP requests made.", ConstLabels: opts.ConstLabels, }, instLabels, ) if err := Register(reqCnt); err != nil { if are, ok := err.(AlreadyRegisteredError); ok { reqCnt = are.ExistingCollector.(*CounterVec) } else { panic(err) } } opts.Name = "request_duration_microseconds" opts.Help = "The HTTP request latencies in microseconds." reqDur := NewSummary(opts) if err := Register(reqDur); err != nil { if are, ok := err.(AlreadyRegisteredError); ok { reqDur = are.ExistingCollector.(Summary) } else { panic(err) } } opts.Name = "request_size_bytes" opts.Help = "The HTTP request sizes in bytes." reqSz := NewSummary(opts) if err := Register(reqSz); err != nil { if are, ok := err.(AlreadyRegisteredError); ok { reqSz = are.ExistingCollector.(Summary) } else { panic(err) } } opts.Name = "response_size_bytes" opts.Help = "The HTTP response sizes in bytes." resSz := NewSummary(opts) if err := Register(resSz); err != nil { if are, ok := err.(AlreadyRegisteredError); ok { resSz = are.ExistingCollector.(Summary) } else { panic(err) } } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { now := time.Now() delegate := &responseWriterDelegator{ResponseWriter: w} out := computeApproximateRequestSize(r) _, cn := w.(http.CloseNotifier) _, fl := w.(http.Flusher) _, hj := w.(http.Hijacker) _, rf := w.(io.ReaderFrom) var rw http.ResponseWriter if cn && fl && hj && rf { rw = &fancyResponseWriterDelegator{delegate} } else { rw = delegate } handlerFunc(rw, r) elapsed := float64(time.Since(now)) / float64(time.Microsecond) method := sanitizeMethod(r.Method) code := sanitizeCode(delegate.status) reqCnt.WithLabelValues(method, code).Inc() reqDur.Observe(elapsed) resSz.Observe(float64(delegate.written)) reqSz.Observe(float64(<-out)) }) } func computeApproximateRequestSize(r *http.Request) <-chan int { // Get URL length in current go routine for avoiding a race condition. // HandlerFunc that runs in parallel may modify the URL. s := 0 if r.URL != nil { s += len(r.URL.String()) } out := make(chan int, 1) go func() { s += len(r.Method) s += len(r.Proto) for name, values := range r.Header { s += len(name) for _, value := range values { s += len(value) } } s += len(r.Host) // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. if r.ContentLength != -1 { s += int(r.ContentLength) } out <- s close(out) }() return out } type responseWriterDelegator struct { http.ResponseWriter handler, method string status int written int64 wroteHeader bool } func (r *responseWriterDelegator) WriteHeader(code int) { r.status = code r.wroteHeader = true r.ResponseWriter.WriteHeader(code) } func (r *responseWriterDelegator) Write(b []byte) (int, error) { if !r.wroteHeader { r.WriteHeader(http.StatusOK) } n, err := r.ResponseWriter.Write(b) r.written += int64(n) return n, err } type fancyResponseWriterDelegator struct { *responseWriterDelegator } func (f *fancyResponseWriterDelegator) CloseNotify() <-chan bool { return f.ResponseWriter.(http.CloseNotifier).CloseNotify() } func (f *fancyResponseWriterDelegator) Flush() { f.ResponseWriter.(http.Flusher).Flush() } func (f *fancyResponseWriterDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) { return f.ResponseWriter.(http.Hijacker).Hijack() } func (f *fancyResponseWriterDelegator) ReadFrom(r io.Reader) (int64, error) { if !f.wroteHeader { f.WriteHeader(http.StatusOK) } n, err := f.ResponseWriter.(io.ReaderFrom).ReadFrom(r) f.written += n return n, err } func sanitizeMethod(m string) string { switch m { case "GET", "get": return "get" case "PUT", "put": return "put" case "HEAD", "head": return "head" case "POST", "post": return "post" case "DELETE", "delete": return "delete" case "CONNECT", "connect": return "connect" case "OPTIONS", "options": return "options" case "NOTIFY", "notify": return "notify" default: return strings.ToLower(m) } } func sanitizeCode(s int) string { switch s { case 100: return "100" case 101: return "101" case 200: return "200" case 201: return "201" case 202: return "202" case 203: return "203" case 204: return "204" case 205: return "205" case 206: return "206" case 300: return "300" case 301: return "301" case 302: return "302" case 304: return "304" case 305: return "305" case 307: return "307" case 400: return "400" case 401: return "401" case 402: return "402" case 403: return "403" case 404: return "404" case 405: return "405" case 406: return "406" case 407: return "407" case 408: return "408" case 409: return "409" case 410: return "410" case 411: return "411" case 412: return "412" case 413: return "413" case 414: return "414" case 415: return "415" case 416: return "416" case 417: return "417" case 418: return "418" case 500: return "500" case 501: return "501" case 502: return "502" case 503: return "503" case 504: return "504" case 505: return "505" case 428: return "428" case 429: return "429" case 431: return "431" case 511: return "511" default: return strconv.Itoa(s) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/http_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "net/http" "net/http/httptest" "testing" "time" dto "github.com/prometheus/client_model/go" ) type respBody string func (b respBody) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTeapot) w.Write([]byte(b)) } func TestInstrumentHandler(t *testing.T) { defer func(n nower) { now = n.(nower) }(now) instant := time.Now() end := instant.Add(30 * time.Second) now = nowSeries(instant, end) respBody := respBody("Howdy there!") hndlr := InstrumentHandler("test-handler", respBody) opts := SummaryOpts{ Subsystem: "http", ConstLabels: Labels{"handler": "test-handler"}, Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, } reqCnt := NewCounterVec( CounterOpts{ Namespace: opts.Namespace, Subsystem: opts.Subsystem, Name: "requests_total", Help: "Total number of HTTP requests made.", ConstLabels: opts.ConstLabels, }, instLabels, ) err := Register(reqCnt) if err == nil { t.Fatal("expected reqCnt to be registered already") } if are, ok := err.(AlreadyRegisteredError); ok { reqCnt = are.ExistingCollector.(*CounterVec) } else { t.Fatal("unexpected registration error:", err) } opts.Name = "request_duration_microseconds" opts.Help = "The HTTP request latencies in microseconds." reqDur := NewSummary(opts) err = Register(reqDur) if err == nil { t.Fatal("expected reqDur to be registered already") } if are, ok := err.(AlreadyRegisteredError); ok { reqDur = are.ExistingCollector.(Summary) } else { t.Fatal("unexpected registration error:", err) } opts.Name = "request_size_bytes" opts.Help = "The HTTP request sizes in bytes." reqSz := NewSummary(opts) err = Register(reqSz) if err == nil { t.Fatal("expected reqSz to be registered already") } if _, ok := err.(AlreadyRegisteredError); !ok { t.Fatal("unexpected registration error:", err) } opts.Name = "response_size_bytes" opts.Help = "The HTTP response sizes in bytes." resSz := NewSummary(opts) err = Register(resSz) if err == nil { t.Fatal("expected resSz to be registered already") } if _, ok := err.(AlreadyRegisteredError); !ok { t.Fatal("unexpected registration error:", err) } reqCnt.Reset() resp := httptest.NewRecorder() req := &http.Request{ Method: "GET", } hndlr.ServeHTTP(resp, req) if resp.Code != http.StatusTeapot { t.Fatalf("expected status %d, got %d", http.StatusTeapot, resp.Code) } if string(resp.Body.Bytes()) != "Howdy there!" { t.Fatalf("expected body %s, got %s", "Howdy there!", string(resp.Body.Bytes())) } out := &dto.Metric{} reqDur.Write(out) if want, got := "test-handler", out.Label[0].GetValue(); want != got { t.Errorf("want label value %q in reqDur, got %q", want, got) } if want, got := uint64(1), out.Summary.GetSampleCount(); want != got { t.Errorf("want sample count %d in reqDur, got %d", want, got) } out.Reset() if want, got := 1, len(reqCnt.children); want != got { t.Errorf("want %d children in reqCnt, got %d", want, got) } cnt, err := reqCnt.GetMetricWithLabelValues("get", "418") if err != nil { t.Fatal(err) } cnt.Write(out) if want, got := "418", out.Label[0].GetValue(); want != got { t.Errorf("want label value %q in reqCnt, got %q", want, got) } if want, got := "test-handler", out.Label[1].GetValue(); want != got { t.Errorf("want label value %q in reqCnt, got %q", want, got) } if want, got := "get", out.Label[2].GetValue(); want != got { t.Errorf("want label value %q in reqCnt, got %q", want, got) } if out.Counter == nil { t.Fatal("expected non-nil counter in reqCnt") } if want, got := 1., out.Counter.GetValue(); want != got { t.Errorf("want reqCnt of %f, got %f", want, got) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/labels.go ================================================ package prometheus import ( "errors" "fmt" "strings" "unicode/utf8" "github.com/prometheus/common/model" ) // Labels represents a collection of label name -> value mappings. This type is // commonly used with the With(Labels) and GetMetricWith(Labels) methods of // metric vector Collectors, e.g.: // myVec.With(Labels{"code": "404", "method": "GET"}).Add(42) // // The other use-case is the specification of constant label pairs in Opts or to // create a Desc. type Labels map[string]string // reservedLabelPrefix is a prefix which is not legal in user-supplied // label names. const reservedLabelPrefix = "__" var errInconsistentCardinality = errors.New("inconsistent label cardinality") func validateValuesInLabels(labels Labels, expectedNumberOfValues int) error { if len(labels) != expectedNumberOfValues { return errInconsistentCardinality } for name, val := range labels { if !utf8.ValidString(val) { return fmt.Errorf("label %s: value %q is not valid UTF-8", name, val) } } return nil } func validateLabelValues(vals []string, expectedNumberOfValues int) error { if len(vals) != expectedNumberOfValues { return errInconsistentCardinality } for _, val := range vals { if !utf8.ValidString(val) { return fmt.Errorf("label value %q is not valid UTF-8", val) } } return nil } func checkLabelName(l string) bool { return model.LabelName(l).IsValid() && !strings.HasPrefix(l, reservedLabelPrefix) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/metric.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "strings" dto "github.com/prometheus/client_model/go" ) const separatorByte byte = 255 // A Metric models a single sample value with its meta data being exported to // Prometheus. Implementations of Metric in this package are Gauge, Counter, // Histogram, Summary, and Untyped. type Metric interface { // Desc returns the descriptor for the Metric. This method idempotently // returns the same descriptor throughout the lifetime of the // Metric. The returned descriptor is immutable by contract. A Metric // unable to describe itself must return an invalid descriptor (created // with NewInvalidDesc). Desc() *Desc // Write encodes the Metric into a "Metric" Protocol Buffer data // transmission object. // // Metric implementations must observe concurrency safety as reads of // this metric may occur at any time, and any blocking occurs at the // expense of total performance of rendering all registered // metrics. Ideally, Metric implementations should support concurrent // readers. // // While populating dto.Metric, it is the responsibility of the // implementation to ensure validity of the Metric protobuf (like valid // UTF-8 strings or syntactically valid metric and label names). It is // recommended to sort labels lexicographically. (Implementers may find // LabelPairSorter useful for that.) Callers of Write should still make // sure of sorting if they depend on it. Write(*dto.Metric) error // TODO(beorn7): The original rationale of passing in a pre-allocated // dto.Metric protobuf to save allocations has disappeared. The // signature of this method should be changed to "Write() (*dto.Metric, // error)". } // Opts bundles the options for creating most Metric types. Each metric // implementation XXX has its own XXXOpts type, but in most cases, it is just be // an alias of this type (which might change when the requirement arises.) // // It is mandatory to set Name and Help to a non-empty string. All other fields // are optional and can safely be left at their zero value. type Opts struct { // Namespace, Subsystem, and Name are components of the fully-qualified // name of the Metric (created by joining these components with // "_"). Only Name is mandatory, the others merely help structuring the // name. Note that the fully-qualified name of the metric must be a // valid Prometheus metric name. Namespace string Subsystem string Name string // Help provides information about this metric. Mandatory! // // Metrics with the same fully-qualified name must have the same Help // string. Help string // ConstLabels are used to attach fixed labels to this metric. Metrics // with the same fully-qualified name must have the same label names in // their ConstLabels. // // Note that in most cases, labels have a value that varies during the // lifetime of a process. Those labels are usually managed with a metric // vector collector (like CounterVec, GaugeVec, UntypedVec). ConstLabels // serve only special purposes. One is for the special case where the // value of a label does not change during the lifetime of a process, // e.g. if the revision of the running binary is put into a // label. Another, more advanced purpose is if more than one Collector // needs to collect Metrics with the same fully-qualified name. In that // case, those Metrics must differ in the values of their // ConstLabels. See the Collector examples. // // If the value of a label never changes (not even between binaries), // that label most likely should not be a label at all (but part of the // metric name). ConstLabels Labels } // BuildFQName joins the given three name components by "_". Empty name // components are ignored. If the name parameter itself is empty, an empty // string is returned, no matter what. Metric implementations included in this // library use this function internally to generate the fully-qualified metric // name from the name component in their Opts. Users of the library will only // need this function if they implement their own Metric or instantiate a Desc // (with NewDesc) directly. func BuildFQName(namespace, subsystem, name string) string { if name == "" { return "" } switch { case namespace != "" && subsystem != "": return strings.Join([]string{namespace, subsystem, name}, "_") case namespace != "": return strings.Join([]string{namespace, name}, "_") case subsystem != "": return strings.Join([]string{subsystem, name}, "_") } return name } // LabelPairSorter implements sort.Interface. It is used to sort a slice of // dto.LabelPair pointers. This is useful for implementing the Write method of // custom metrics. type LabelPairSorter []*dto.LabelPair func (s LabelPairSorter) Len() int { return len(s) } func (s LabelPairSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s LabelPairSorter) Less(i, j int) bool { return s[i].GetName() < s[j].GetName() } type hashSorter []uint64 func (s hashSorter) Len() int { return len(s) } func (s hashSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s hashSorter) Less(i, j int) bool { return s[i] < s[j] } type invalidMetric struct { desc *Desc err error } // NewInvalidMetric returns a metric whose Write method always returns the // provided error. It is useful if a Collector finds itself unable to collect // a metric and wishes to report an error to the registry. func NewInvalidMetric(desc *Desc, err error) Metric { return &invalidMetric{desc, err} } func (m *invalidMetric) Desc() *Desc { return m.desc } func (m *invalidMetric) Write(*dto.Metric) error { return m.err } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/metric_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import "testing" func TestBuildFQName(t *testing.T) { scenarios := []struct{ namespace, subsystem, name, result string }{ {"a", "b", "c", "a_b_c"}, {"", "b", "c", "b_c"}, {"a", "", "c", "a_c"}, {"", "", "c", "c"}, {"a", "b", "", ""}, {"a", "", "", ""}, {"", "b", "", ""}, {" ", "", "", ""}, } for i, s := range scenarios { if want, got := s.result, BuildFQName(s.namespace, s.subsystem, s.name); want != got { t.Errorf("%d. want %s, got %s", i, want, got) } } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/observer.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus // Observer is the interface that wraps the Observe method, which is used by // Histogram and Summary to add observations. type Observer interface { Observe(float64) } // The ObserverFunc type is an adapter to allow the use of ordinary // functions as Observers. If f is a function with the appropriate // signature, ObserverFunc(f) is an Observer that calls f. // // This adapter is usually used in connection with the Timer type, and there are // two general use cases: // // The most common one is to use a Gauge as the Observer for a Timer. // See the "Gauge" Timer example. // // The more advanced use case is to create a function that dynamically decides // which Observer to use for observing the duration. See the "Complex" Timer // example. type ObserverFunc func(float64) // Observe calls f(value). It implements Observer. func (f ObserverFunc) Observe(value float64) { f(value) } // ObserverVec is an interface implemented by `HistogramVec` and `SummaryVec`. type ObserverVec interface { GetMetricWith(Labels) (Observer, error) GetMetricWithLabelValues(lvs ...string) (Observer, error) With(Labels) Observer WithLabelValues(...string) Observer Collector } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/process_collector.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import "github.com/prometheus/procfs" type processCollector struct { pid int collectFn func(chan<- Metric) pidFn func() (int, error) cpuTotal *Desc openFDs, maxFDs *Desc vsize, rss *Desc startTime *Desc } // NewProcessCollector returns a collector which exports the current state of // process metrics including cpu, memory and file descriptor usage as well as // the process start time for the given process id under the given namespace. func NewProcessCollector(pid int, namespace string) Collector { return NewProcessCollectorPIDFn( func() (int, error) { return pid, nil }, namespace, ) } // NewProcessCollectorPIDFn returns a collector which exports the current state // of process metrics including cpu, memory and file descriptor usage as well // as the process start time under the given namespace. The given pidFn is // called on each collect and is used to determine the process to export // metrics for. func NewProcessCollectorPIDFn( pidFn func() (int, error), namespace string, ) Collector { ns := "" if len(namespace) > 0 { ns = namespace + "_" } c := processCollector{ pidFn: pidFn, collectFn: func(chan<- Metric) {}, cpuTotal: NewDesc( ns+"process_cpu_seconds_total", "Total user and system CPU time spent in seconds.", nil, nil, ), openFDs: NewDesc( ns+"process_open_fds", "Number of open file descriptors.", nil, nil, ), maxFDs: NewDesc( ns+"process_max_fds", "Maximum number of open file descriptors.", nil, nil, ), vsize: NewDesc( ns+"process_virtual_memory_bytes", "Virtual memory size in bytes.", nil, nil, ), rss: NewDesc( ns+"process_resident_memory_bytes", "Resident memory size in bytes.", nil, nil, ), startTime: NewDesc( ns+"process_start_time_seconds", "Start time of the process since unix epoch in seconds.", nil, nil, ), } // Set up process metric collection if supported by the runtime. if _, err := procfs.NewStat(); err == nil { c.collectFn = c.processCollect } return &c } // Describe returns all descriptions of the collector. func (c *processCollector) Describe(ch chan<- *Desc) { ch <- c.cpuTotal ch <- c.openFDs ch <- c.maxFDs ch <- c.vsize ch <- c.rss ch <- c.startTime } // Collect returns the current state of all metrics of the collector. func (c *processCollector) Collect(ch chan<- Metric) { c.collectFn(ch) } // TODO(ts): Bring back error reporting by reverting 7faf9e7 as soon as the // client allows users to configure the error behavior. func (c *processCollector) processCollect(ch chan<- Metric) { pid, err := c.pidFn() if err != nil { return } p, err := procfs.NewProc(pid) if err != nil { return } if stat, err := p.NewStat(); err == nil { ch <- MustNewConstMetric(c.cpuTotal, CounterValue, stat.CPUTime()) ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(stat.VirtualMemory())) ch <- MustNewConstMetric(c.rss, GaugeValue, float64(stat.ResidentMemory())) if startTime, err := stat.StartTime(); err == nil { ch <- MustNewConstMetric(c.startTime, GaugeValue, startTime) } } if fds, err := p.FileDescriptorsLen(); err == nil { ch <- MustNewConstMetric(c.openFDs, GaugeValue, float64(fds)) } if limits, err := p.NewLimits(); err == nil { ch <- MustNewConstMetric(c.maxFDs, GaugeValue, float64(limits.OpenFiles)) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/process_collector_test.go ================================================ package prometheus import ( "bytes" "os" "regexp" "testing" "github.com/prometheus/common/expfmt" "github.com/prometheus/procfs" ) func TestProcessCollector(t *testing.T) { if _, err := procfs.Self(); err != nil { t.Skipf("skipping TestProcessCollector, procfs not available: %s", err) } registry := NewRegistry() if err := registry.Register(NewProcessCollector(os.Getpid(), "")); err != nil { t.Fatal(err) } if err := registry.Register(NewProcessCollectorPIDFn( func() (int, error) { return os.Getpid(), nil }, "foobar"), ); err != nil { t.Fatal(err) } mfs, err := registry.Gather() if err != nil { t.Fatal(err) } var buf bytes.Buffer for _, mf := range mfs { if _, err := expfmt.MetricFamilyToText(&buf, mf); err != nil { t.Fatal(err) } } for _, re := range []*regexp.Regexp{ regexp.MustCompile("\nprocess_cpu_seconds_total [0-9]"), regexp.MustCompile("\nprocess_max_fds [1-9]"), regexp.MustCompile("\nprocess_open_fds [1-9]"), regexp.MustCompile("\nprocess_virtual_memory_bytes [1-9]"), regexp.MustCompile("\nprocess_resident_memory_bytes [1-9]"), regexp.MustCompile("\nprocess_start_time_seconds [0-9.]{10,}"), regexp.MustCompile("\nfoobar_process_cpu_seconds_total [0-9]"), regexp.MustCompile("\nfoobar_process_max_fds [1-9]"), regexp.MustCompile("\nfoobar_process_open_fds [1-9]"), regexp.MustCompile("\nfoobar_process_virtual_memory_bytes [1-9]"), regexp.MustCompile("\nfoobar_process_resident_memory_bytes [1-9]"), regexp.MustCompile("\nfoobar_process_start_time_seconds [0-9.]{10,}"), } { if !re.Match(buf.Bytes()) { t.Errorf("want body to match %s\n%s", re, buf.String()) } } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package promhttp import ( "bufio" "io" "net" "net/http" ) const ( closeNotifier = 1 << iota flusher hijacker readerFrom pusher ) type delegator interface { http.ResponseWriter Status() int Written() int64 } type responseWriterDelegator struct { http.ResponseWriter handler, method string status int written int64 wroteHeader bool observeWriteHeader func(int) } func (r *responseWriterDelegator) Status() int { return r.status } func (r *responseWriterDelegator) Written() int64 { return r.written } func (r *responseWriterDelegator) WriteHeader(code int) { r.status = code r.wroteHeader = true r.ResponseWriter.WriteHeader(code) if r.observeWriteHeader != nil { r.observeWriteHeader(code) } } func (r *responseWriterDelegator) Write(b []byte) (int, error) { if !r.wroteHeader { r.WriteHeader(http.StatusOK) } n, err := r.ResponseWriter.Write(b) r.written += int64(n) return n, err } type closeNotifierDelegator struct{ *responseWriterDelegator } type flusherDelegator struct{ *responseWriterDelegator } type hijackerDelegator struct{ *responseWriterDelegator } type readerFromDelegator struct{ *responseWriterDelegator } func (d *closeNotifierDelegator) CloseNotify() <-chan bool { return d.ResponseWriter.(http.CloseNotifier).CloseNotify() } func (d *flusherDelegator) Flush() { d.ResponseWriter.(http.Flusher).Flush() } func (d *hijackerDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) { return d.ResponseWriter.(http.Hijacker).Hijack() } func (d *readerFromDelegator) ReadFrom(re io.Reader) (int64, error) { if !d.wroteHeader { d.WriteHeader(http.StatusOK) } n, err := d.ResponseWriter.(io.ReaderFrom).ReadFrom(re) d.written += n return n, err } var pickDelegator = make([]func(*responseWriterDelegator) delegator, 32) func init() { // TODO(beorn7): Code generation would help here. pickDelegator[0] = func(d *responseWriterDelegator) delegator { // 0 return d } pickDelegator[closeNotifier] = func(d *responseWriterDelegator) delegator { // 1 return closeNotifierDelegator{d} } pickDelegator[flusher] = func(d *responseWriterDelegator) delegator { // 2 return flusherDelegator{d} } pickDelegator[flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 3 return struct { *responseWriterDelegator http.Flusher http.CloseNotifier }{d, &flusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[hijacker] = func(d *responseWriterDelegator) delegator { // 4 return hijackerDelegator{d} } pickDelegator[hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 5 return struct { *responseWriterDelegator http.Hijacker http.CloseNotifier }{d, &hijackerDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 6 return struct { *responseWriterDelegator http.Hijacker http.Flusher }{d, &hijackerDelegator{d}, &flusherDelegator{d}} } pickDelegator[hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 7 return struct { *responseWriterDelegator http.Hijacker http.Flusher http.CloseNotifier }{d, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[readerFrom] = func(d *responseWriterDelegator) delegator { // 8 return readerFromDelegator{d} } pickDelegator[readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 9 return struct { *responseWriterDelegator io.ReaderFrom http.CloseNotifier }{d, &readerFromDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 10 return struct { *responseWriterDelegator io.ReaderFrom http.Flusher }{d, &readerFromDelegator{d}, &flusherDelegator{d}} } pickDelegator[readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 11 return struct { *responseWriterDelegator io.ReaderFrom http.Flusher http.CloseNotifier }{d, &readerFromDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 12 return struct { *responseWriterDelegator io.ReaderFrom http.Hijacker }{d, &readerFromDelegator{d}, &hijackerDelegator{d}} } pickDelegator[readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 13 return struct { *responseWriterDelegator io.ReaderFrom http.Hijacker http.CloseNotifier }{d, &readerFromDelegator{d}, &hijackerDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 14 return struct { *responseWriterDelegator io.ReaderFrom http.Hijacker http.Flusher }{d, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}} } pickDelegator[readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 15 return struct { *responseWriterDelegator io.ReaderFrom http.Hijacker http.Flusher http.CloseNotifier }{d, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build go1.8 package promhttp import ( "io" "net/http" ) type pusherDelegator struct{ *responseWriterDelegator } func (d *pusherDelegator) Push(target string, opts *http.PushOptions) error { return d.ResponseWriter.(http.Pusher).Push(target, opts) } func init() { pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16 return pusherDelegator{d} } pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17 return struct { *responseWriterDelegator http.Pusher http.CloseNotifier }{d, &pusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+flusher] = func(d *responseWriterDelegator) delegator { // 18 return struct { *responseWriterDelegator http.Pusher http.Flusher }{d, &pusherDelegator{d}, &flusherDelegator{d}} } pickDelegator[pusher+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 19 return struct { *responseWriterDelegator http.Pusher http.Flusher http.CloseNotifier }{d, &pusherDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+hijacker] = func(d *responseWriterDelegator) delegator { // 20 return struct { *responseWriterDelegator http.Pusher http.Hijacker }{d, &pusherDelegator{d}, &hijackerDelegator{d}} } pickDelegator[pusher+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 21 return struct { *responseWriterDelegator http.Pusher http.Hijacker http.CloseNotifier }{d, &pusherDelegator{d}, &hijackerDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 22 return struct { *responseWriterDelegator http.Pusher http.Hijacker http.Flusher }{d, &pusherDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}} } pickDelegator[pusher+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { //23 return struct { *responseWriterDelegator http.Pusher http.Hijacker http.Flusher http.CloseNotifier }{d, &pusherDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+readerFrom] = func(d *responseWriterDelegator) delegator { // 24 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom }{d, &pusherDelegator{d}, &readerFromDelegator{d}} } pickDelegator[pusher+readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 25 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.CloseNotifier }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 26 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.Flusher }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &flusherDelegator{d}} } pickDelegator[pusher+readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 27 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.Flusher http.CloseNotifier }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 28 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.Hijacker }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}} } pickDelegator[pusher+readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 29 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.Hijacker http.CloseNotifier }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}, &closeNotifierDelegator{d}} } pickDelegator[pusher+readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 30 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.Hijacker http.Flusher }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}} } pickDelegator[pusher+readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 31 return struct { *responseWriterDelegator http.Pusher io.ReaderFrom http.Hijacker http.Flusher http.CloseNotifier }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} } } func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { d := &responseWriterDelegator{ ResponseWriter: w, observeWriteHeader: observeWriteHeaderFunc, } id := 0 if _, ok := w.(http.CloseNotifier); ok { id += closeNotifier } if _, ok := w.(http.Flusher); ok { id += flusher } if _, ok := w.(http.Hijacker); ok { id += hijacker } if _, ok := w.(io.ReaderFrom); ok { id += readerFrom } if _, ok := w.(http.Pusher); ok { id += pusher } return pickDelegator[id](d) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_pre_1_8.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build !go1.8 package promhttp import ( "io" "net/http" ) func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { d := &responseWriterDelegator{ ResponseWriter: w, observeWriteHeader: observeWriteHeaderFunc, } id := 0 if _, ok := w.(http.CloseNotifier); ok { id += closeNotifier } if _, ok := w.(http.Flusher); ok { id += flusher } if _, ok := w.(http.Hijacker); ok { id += hijacker } if _, ok := w.(io.ReaderFrom); ok { id += readerFrom } return pickDelegator[id](d) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go ================================================ // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package promhttp provides tooling around HTTP servers and clients. // // First, the package allows the creation of http.Handler instances to expose // Prometheus metrics via HTTP. promhttp.Handler acts on the // prometheus.DefaultGatherer. With HandlerFor, you can create a handler for a // custom registry or anything that implements the Gatherer interface. It also // allows the creation of handlers that act differently on errors or allow to // log errors. // // Second, the package provides tooling to instrument instances of http.Handler // via middleware. Middleware wrappers follow the naming scheme // InstrumentHandlerX, where X describes the intended use of the middleware. // See each function's doc comment for specific details. // // Finally, the package allows for an http.RoundTripper to be instrumented via // middleware. Middleware wrappers follow the naming scheme // InstrumentRoundTripperX, where X describes the intended use of the // middleware. See each function's doc comment for specific details. package promhttp import ( "bytes" "compress/gzip" "fmt" "io" "net/http" "strings" "sync" "github.com/prometheus/common/expfmt" "github.com/prometheus/client_golang/prometheus" ) const ( contentTypeHeader = "Content-Type" contentLengthHeader = "Content-Length" contentEncodingHeader = "Content-Encoding" acceptEncodingHeader = "Accept-Encoding" ) var bufPool sync.Pool func getBuf() *bytes.Buffer { buf := bufPool.Get() if buf == nil { return &bytes.Buffer{} } return buf.(*bytes.Buffer) } func giveBuf(buf *bytes.Buffer) { buf.Reset() bufPool.Put(buf) } // Handler returns an HTTP handler for the prometheus.DefaultGatherer. The // Handler uses the default HandlerOpts, i.e. report the first error as an HTTP // error, no error logging, and compression if requested by the client. // // If you want to create a Handler for the DefaultGatherer with different // HandlerOpts, create it with HandlerFor with prometheus.DefaultGatherer and // your desired HandlerOpts. func Handler() http.Handler { return HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}) } // HandlerFor returns an http.Handler for the provided Gatherer. The behavior // of the Handler is defined by the provided HandlerOpts. func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { mfs, err := reg.Gather() if err != nil { if opts.ErrorLog != nil { opts.ErrorLog.Println("error gathering metrics:", err) } switch opts.ErrorHandling { case PanicOnError: panic(err) case ContinueOnError: if len(mfs) == 0 { http.Error(w, "No metrics gathered, last error:\n\n"+err.Error(), http.StatusInternalServerError) return } case HTTPErrorOnError: http.Error(w, "An error has occurred during metrics gathering:\n\n"+err.Error(), http.StatusInternalServerError) return } } contentType := expfmt.Negotiate(req.Header) buf := getBuf() defer giveBuf(buf) writer, encoding := decorateWriter(req, buf, opts.DisableCompression) enc := expfmt.NewEncoder(writer, contentType) var lastErr error for _, mf := range mfs { if err := enc.Encode(mf); err != nil { lastErr = err if opts.ErrorLog != nil { opts.ErrorLog.Println("error encoding metric family:", err) } switch opts.ErrorHandling { case PanicOnError: panic(err) case ContinueOnError: // Handled later. case HTTPErrorOnError: http.Error(w, "An error has occurred during metrics encoding:\n\n"+err.Error(), http.StatusInternalServerError) return } } } if closer, ok := writer.(io.Closer); ok { closer.Close() } if lastErr != nil && buf.Len() == 0 { http.Error(w, "No metrics encoded, last error:\n\n"+lastErr.Error(), http.StatusInternalServerError) return } header := w.Header() header.Set(contentTypeHeader, string(contentType)) header.Set(contentLengthHeader, fmt.Sprint(buf.Len())) if encoding != "" { header.Set(contentEncodingHeader, encoding) } w.Write(buf.Bytes()) // TODO(beorn7): Consider streaming serving of metrics. }) } // HandlerErrorHandling defines how a Handler serving metrics will handle // errors. type HandlerErrorHandling int // These constants cause handlers serving metrics to behave as described if // errors are encountered. const ( // Serve an HTTP status code 500 upon the first error // encountered. Report the error message in the body. HTTPErrorOnError HandlerErrorHandling = iota // Ignore errors and try to serve as many metrics as possible. However, // if no metrics can be served, serve an HTTP status code 500 and the // last error message in the body. Only use this in deliberate "best // effort" metrics collection scenarios. It is recommended to at least // log errors (by providing an ErrorLog in HandlerOpts) to not mask // errors completely. ContinueOnError // Panic upon the first error encountered (useful for "crash only" apps). PanicOnError ) // Logger is the minimal interface HandlerOpts needs for logging. Note that // log.Logger from the standard library implements this interface, and it is // easy to implement by custom loggers, if they don't do so already anyway. type Logger interface { Println(v ...interface{}) } // HandlerOpts specifies options how to serve metrics via an http.Handler. The // zero value of HandlerOpts is a reasonable default. type HandlerOpts struct { // ErrorLog specifies an optional logger for errors collecting and // serving metrics. If nil, errors are not logged at all. ErrorLog Logger // ErrorHandling defines how errors are handled. Note that errors are // logged regardless of the configured ErrorHandling provided ErrorLog // is not nil. ErrorHandling HandlerErrorHandling // If DisableCompression is true, the handler will never compress the // response, even if requested by the client. DisableCompression bool } // decorateWriter wraps a writer to handle gzip compression if requested. It // returns the decorated writer and the appropriate "Content-Encoding" header // (which is empty if no compression is enabled). func decorateWriter(request *http.Request, writer io.Writer, compressionDisabled bool) (io.Writer, string) { if compressionDisabled { return writer, "" } header := request.Header.Get(acceptEncodingHeader) parts := strings.Split(header, ",") for _, part := range parts { part := strings.TrimSpace(part) if part == "gzip" || strings.HasPrefix(part, "gzip;") { return gzip.NewWriter(writer), "gzip" } } return writer, "" } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/http_test.go ================================================ // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package promhttp import ( "bytes" "errors" "log" "net/http" "net/http/httptest" "testing" "github.com/prometheus/client_golang/prometheus" ) type errorCollector struct{} func (e errorCollector) Describe(ch chan<- *prometheus.Desc) { ch <- prometheus.NewDesc("invalid_metric", "not helpful", nil, nil) } func (e errorCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.NewInvalidMetric( prometheus.NewDesc("invalid_metric", "not helpful", nil, nil), errors.New("collect error"), ) } func TestHandlerErrorHandling(t *testing.T) { // Create a registry that collects a MetricFamily with two elements, // another with one, and reports an error. reg := prometheus.NewRegistry() cnt := prometheus.NewCounter(prometheus.CounterOpts{ Name: "the_count", Help: "Ah-ah-ah! Thunder and lightning!", }) reg.MustRegister(cnt) cntVec := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "name", Help: "docstring", ConstLabels: prometheus.Labels{"constname": "constvalue"}, }, []string{"labelname"}, ) cntVec.WithLabelValues("val1").Inc() cntVec.WithLabelValues("val2").Inc() reg.MustRegister(cntVec) reg.MustRegister(errorCollector{}) logBuf := &bytes.Buffer{} logger := log.New(logBuf, "", 0) writer := httptest.NewRecorder() request, _ := http.NewRequest("GET", "/", nil) request.Header.Add("Accept", "test/plain") errorHandler := HandlerFor(reg, HandlerOpts{ ErrorLog: logger, ErrorHandling: HTTPErrorOnError, }) continueHandler := HandlerFor(reg, HandlerOpts{ ErrorLog: logger, ErrorHandling: ContinueOnError, }) panicHandler := HandlerFor(reg, HandlerOpts{ ErrorLog: logger, ErrorHandling: PanicOnError, }) wantMsg := `error gathering metrics: error collecting metric Desc{fqName: "invalid_metric", help: "not helpful", constLabels: {}, variableLabels: []}: collect error ` wantErrorBody := `An error has occurred during metrics gathering: error collecting metric Desc{fqName: "invalid_metric", help: "not helpful", constLabels: {}, variableLabels: []}: collect error ` wantOKBody := `# HELP name docstring # TYPE name counter name{constname="constvalue",labelname="val1"} 1 name{constname="constvalue",labelname="val2"} 1 # HELP the_count Ah-ah-ah! Thunder and lightning! # TYPE the_count counter the_count 0 ` errorHandler.ServeHTTP(writer, request) if got, want := writer.Code, http.StatusInternalServerError; got != want { t.Errorf("got HTTP status code %d, want %d", got, want) } if got := logBuf.String(); got != wantMsg { t.Errorf("got log message:\n%s\nwant log mesage:\n%s\n", got, wantMsg) } if got := writer.Body.String(); got != wantErrorBody { t.Errorf("got body:\n%s\nwant body:\n%s\n", got, wantErrorBody) } logBuf.Reset() writer.Body.Reset() writer.Code = http.StatusOK continueHandler.ServeHTTP(writer, request) if got, want := writer.Code, http.StatusOK; got != want { t.Errorf("got HTTP status code %d, want %d", got, want) } if got := logBuf.String(); got != wantMsg { t.Errorf("got log message %q, want %q", got, wantMsg) } if got := writer.Body.String(); got != wantOKBody { t.Errorf("got body %q, want %q", got, wantOKBody) } defer func() { if err := recover(); err == nil { t.Error("expected panic from panicHandler") } }() panicHandler.ServeHTTP(writer, request) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package promhttp import ( "net/http" "time" "github.com/prometheus/client_golang/prometheus" ) // The RoundTripperFunc type is an adapter to allow the use of ordinary // functions as RoundTrippers. If f is a function with the appropriate // signature, RountTripperFunc(f) is a RoundTripper that calls f. type RoundTripperFunc func(req *http.Request) (*http.Response, error) // RoundTrip implements the RoundTripper interface. func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return rt(r) } // InstrumentRoundTripperInFlight is a middleware that wraps the provided // http.RoundTripper. It sets the provided prometheus.Gauge to the number of // requests currently handled by the wrapped http.RoundTripper. // // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripper) RoundTripperFunc { return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { gauge.Inc() defer gauge.Dec() return next.RoundTrip(r) }) } // InstrumentRoundTripperCounter is a middleware that wraps the provided // http.RoundTripper to observe the request result with the provided CounterVec. // The CounterVec must have zero, one, or two labels. The only allowed label // names are "code" and "method". The function panics if any other instance // labels are provided. Partitioning of the CounterVec happens by HTTP status // code and/or HTTP method if the respective instance label names are present // in the CounterVec. For unpartitioned counting, use a CounterVec with // zero labels. // // If the wrapped RoundTripper panics or returns a non-nil error, the Counter // is not incremented. // // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper) RoundTripperFunc { code, method := checkLabels(counter) return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { resp, err := next.RoundTrip(r) if err == nil { counter.With(labels(code, method, r.Method, resp.StatusCode)).Inc() } return resp, err }) } // InstrumentRoundTripperDuration is a middleware that wraps the provided // http.RoundTripper to observe the request duration with the provided ObserverVec. // The ObserverVec must have zero, one, or two labels. The only allowed label // names are "code" and "method". The function panics if any other instance // labels are provided. The Observe method of the Observer in the ObserverVec // is called with the request duration in seconds. Partitioning happens by HTTP // status code and/or HTTP method if the respective instance label names are // present in the ObserverVec. For unpartitioned observations, use an // ObserverVec with zero labels. Note that partitioning of Histograms is // expensive and should be used judiciously. // // If the wrapped RoundTripper panics or returns a non-nil error, no values are // reported. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper) RoundTripperFunc { code, method := checkLabels(obs) return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { start := time.Now() resp, err := next.RoundTrip(r) if err == nil { obs.With(labels(code, method, r.Method, resp.StatusCode)).Observe(time.Since(start).Seconds()) } return resp, err }) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build go1.8 package promhttp import ( "context" "crypto/tls" "net/http" "net/http/httptrace" "time" ) // InstrumentTrace is used to offer flexibility in instrumenting the available // httptrace.ClientTrace hook functions. Each function is passed a float64 // representing the time in seconds since the start of the http request. A user // may choose to use separately buckets Histograms, or implement custom // instance labels on a per function basis. type InstrumentTrace struct { GotConn func(float64) PutIdleConn func(float64) GotFirstResponseByte func(float64) Got100Continue func(float64) DNSStart func(float64) DNSDone func(float64) ConnectStart func(float64) ConnectDone func(float64) TLSHandshakeStart func(float64) TLSHandshakeDone func(float64) WroteHeaders func(float64) Wait100Continue func(float64) WroteRequest func(float64) } // InstrumentRoundTripperTrace is a middleware that wraps the provided // RoundTripper and reports times to hook functions provided in the // InstrumentTrace struct. Hook functions that are not present in the provided // InstrumentTrace struct are ignored. Times reported to the hook functions are // time since the start of the request. Only with Go1.9+, those times are // guaranteed to never be negative. (Earlier Go versions are not using a // monotonic clock.) Note that partitioning of Histograms is expensive and // should be used judiciously. // // For hook functions that receive an error as an argument, no observations are // made in the event of a non-nil error value. // // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc { return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { start := time.Now() trace := &httptrace.ClientTrace{ GotConn: func(_ httptrace.GotConnInfo) { if it.GotConn != nil { it.GotConn(time.Since(start).Seconds()) } }, PutIdleConn: func(err error) { if err != nil { return } if it.PutIdleConn != nil { it.PutIdleConn(time.Since(start).Seconds()) } }, DNSStart: func(_ httptrace.DNSStartInfo) { if it.DNSStart != nil { it.DNSStart(time.Since(start).Seconds()) } }, DNSDone: func(_ httptrace.DNSDoneInfo) { if it.DNSStart != nil { it.DNSStart(time.Since(start).Seconds()) } }, ConnectStart: func(_, _ string) { if it.ConnectStart != nil { it.ConnectStart(time.Since(start).Seconds()) } }, ConnectDone: func(_, _ string, err error) { if err != nil { return } if it.ConnectDone != nil { it.ConnectDone(time.Since(start).Seconds()) } }, GotFirstResponseByte: func() { if it.GotFirstResponseByte != nil { it.GotFirstResponseByte(time.Since(start).Seconds()) } }, Got100Continue: func() { if it.Got100Continue != nil { it.Got100Continue(time.Since(start).Seconds()) } }, TLSHandshakeStart: func() { if it.TLSHandshakeStart != nil { it.TLSHandshakeStart(time.Since(start).Seconds()) } }, TLSHandshakeDone: func(_ tls.ConnectionState, err error) { if err != nil { return } if it.TLSHandshakeDone != nil { it.TLSHandshakeDone(time.Since(start).Seconds()) } }, WroteHeaders: func() { if it.WroteHeaders != nil { it.WroteHeaders(time.Since(start).Seconds()) } }, Wait100Continue: func() { if it.Wait100Continue != nil { it.Wait100Continue(time.Since(start).Seconds()) } }, WroteRequest: func(_ httptrace.WroteRequestInfo) { if it.WroteRequest != nil { it.WroteRequest(time.Since(start).Seconds()) } }, } r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace)) return next.RoundTrip(r) }) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8_test.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build go1.8 package promhttp import ( "log" "net/http" "testing" "time" "github.com/prometheus/client_golang/prometheus" ) func TestClientMiddlewareAPI(t *testing.T) { client := http.DefaultClient client.Timeout = 1 * time.Second reg := prometheus.NewRegistry() inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "client_in_flight_requests", Help: "A gauge of in-flight requests for the wrapped client.", }) counter := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "client_api_requests_total", Help: "A counter for requests from the wrapped client.", }, []string{"code", "method"}, ) dnsLatencyVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "dns_duration_seconds", Help: "Trace dns latency histogram.", Buckets: []float64{.005, .01, .025, .05}, }, []string{"event"}, ) tlsLatencyVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "tls_duration_seconds", Help: "Trace tls latency histogram.", Buckets: []float64{.05, .1, .25, .5}, }, []string{"event"}, ) histVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "request_duration_seconds", Help: "A histogram of request latencies.", Buckets: prometheus.DefBuckets, }, []string{"method"}, ) reg.MustRegister(counter, tlsLatencyVec, dnsLatencyVec, histVec, inFlightGauge) trace := &InstrumentTrace{ DNSStart: func(t float64) { dnsLatencyVec.WithLabelValues("dns_start") }, DNSDone: func(t float64) { dnsLatencyVec.WithLabelValues("dns_done") }, TLSHandshakeStart: func(t float64) { tlsLatencyVec.WithLabelValues("tls_handshake_start") }, TLSHandshakeDone: func(t float64) { tlsLatencyVec.WithLabelValues("tls_handshake_done") }, } client.Transport = InstrumentRoundTripperInFlight(inFlightGauge, InstrumentRoundTripperCounter(counter, InstrumentRoundTripperTrace(trace, InstrumentRoundTripperDuration(histVec, http.DefaultTransport), ), ), ) resp, err := client.Get("http://google.com") if err != nil { t.Fatalf("%v", err) } defer resp.Body.Close() } func ExampleInstrumentRoundTripperDuration() { client := http.DefaultClient client.Timeout = 1 * time.Second inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "client_in_flight_requests", Help: "A gauge of in-flight requests for the wrapped client.", }) counter := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "client_api_requests_total", Help: "A counter for requests from the wrapped client.", }, []string{"code", "method"}, ) // dnsLatencyVec uses custom buckets based on expected dns durations. // It has an instance label "event", which is set in the // DNSStart and DNSDonehook functions defined in the // InstrumentTrace struct below. dnsLatencyVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "dns_duration_seconds", Help: "Trace dns latency histogram.", Buckets: []float64{.005, .01, .025, .05}, }, []string{"event"}, ) // tlsLatencyVec uses custom buckets based on expected tls durations. // It has an instance label "event", which is set in the // TLSHandshakeStart and TLSHandshakeDone hook functions defined in the // InstrumentTrace struct below. tlsLatencyVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "tls_duration_seconds", Help: "Trace tls latency histogram.", Buckets: []float64{.05, .1, .25, .5}, }, []string{"event"}, ) // histVec has no labels, making it a zero-dimensional ObserverVec. histVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "request_duration_seconds", Help: "A histogram of request latencies.", Buckets: prometheus.DefBuckets, }, []string{}, ) // Register all of the metrics in the standard registry. prometheus.MustRegister(counter, tlsLatencyVec, dnsLatencyVec, histVec, inFlightGauge) // Define functions for the available httptrace.ClientTrace hook // functions that we want to instrument. trace := &InstrumentTrace{ DNSStart: func(t float64) { dnsLatencyVec.WithLabelValues("dns_start") }, DNSDone: func(t float64) { dnsLatencyVec.WithLabelValues("dns_done") }, TLSHandshakeStart: func(t float64) { tlsLatencyVec.WithLabelValues("tls_handshake_start") }, TLSHandshakeDone: func(t float64) { tlsLatencyVec.WithLabelValues("tls_handshake_done") }, } // Wrap the default RoundTripper with middleware. roundTripper := InstrumentRoundTripperInFlight(inFlightGauge, InstrumentRoundTripperCounter(counter, InstrumentRoundTripperTrace(trace, InstrumentRoundTripperDuration(histVec, http.DefaultTransport), ), ), ) // Set the RoundTripper on our client. client.Transport = roundTripper resp, err := client.Get("http://google.com") if err != nil { log.Printf("error: %v", err) } defer resp.Body.Close() } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package promhttp import ( "net/http" "strconv" "strings" "time" dto "github.com/prometheus/client_model/go" "github.com/prometheus/client_golang/prometheus" ) // magicString is used for the hacky label test in checkLabels. Remove once fixed. const magicString = "zZgWfBxLqvG8kc8IMv3POi2Bb0tZI3vAnBx+gBaFi9FyPzB/CzKUer1yufDa" // InstrumentHandlerInFlight is a middleware that wraps the provided // http.Handler. It sets the provided prometheus.Gauge to the number of // requests currently handled by the wrapped http.Handler. // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { g.Inc() defer g.Dec() next.ServeHTTP(w, r) }) } // InstrumentHandlerDuration is a middleware that wraps the provided // http.Handler to observe the request duration with the provided ObserverVec. // The ObserverVec must have zero, one, or two labels. The only allowed label // names are "code" and "method". The function panics if any other instance // labels are provided. The Observe method of the Observer in the ObserverVec // is called with the request duration in seconds. Partitioning happens by HTTP // status code and/or HTTP method if the respective instance label names are // present in the ObserverVec. For unpartitioned observations, use an // ObserverVec with zero labels. Note that partitioning of Histograms is // expensive and should be used judiciously. // // If the wrapped Handler does not set a status code, a status code of 200 is assumed. // // If the wrapped Handler panics, no values are reported. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { code, method := checkLabels(obs) if code { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { now := time.Now() d := newDelegator(w, nil) next.ServeHTTP(d, r) obs.With(labels(code, method, r.Method, d.Status())).Observe(time.Since(now).Seconds()) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { now := time.Now() next.ServeHTTP(w, r) obs.With(labels(code, method, r.Method, 0)).Observe(time.Since(now).Seconds()) }) } // InstrumentHandlerCounter is a middleware that wraps the provided // http.Handler to observe the request result with the provided CounterVec. // The CounterVec must have zero, one, or two labels. The only allowed label // names are "code" and "method". The function panics if any other instance // labels are provided. Partitioning of the CounterVec happens by HTTP status // code and/or HTTP method if the respective instance label names are present // in the CounterVec. For unpartitioned counting, use a CounterVec with // zero labels. // // If the wrapped Handler does not set a status code, a status code of 200 is assumed. // // If the wrapped Handler panics, the Counter is not incremented. // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) http.HandlerFunc { code, method := checkLabels(counter) if code { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) counter.With(labels(code, method, r.Method, d.Status())).Inc() }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) counter.With(labels(code, method, r.Method, 0)).Inc() }) } // InstrumentHandlerTimeToWriteHeader is a middleware that wraps the provided // http.Handler to observe with the provided ObserverVec the request duration // until the response headers are written. The ObserverVec must have zero, one, // or two labels. The only allowed label names are "code" and "method". The // function panics if any other instance labels are provided. The Observe // method of the Observer in the ObserverVec is called with the request // duration in seconds. Partitioning happens by HTTP status code and/or HTTP // method if the respective instance label names are present in the // ObserverVec. For unpartitioned observations, use an ObserverVec with zero // labels. Note that partitioning of Histograms is expensive and should be used // judiciously. // // If the wrapped Handler panics before calling WriteHeader, no value is // reported. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { code, method := checkLabels(obs) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { now := time.Now() d := newDelegator(w, func(status int) { obs.With(labels(code, method, r.Method, status)).Observe(time.Since(now).Seconds()) }) next.ServeHTTP(d, r) }) } // InstrumentHandlerRequestSize is a middleware that wraps the provided // http.Handler to observe the request size with the provided ObserverVec. // The ObserverVec must have zero, one, or two labels. The only allowed label // names are "code" and "method". The function panics if any other instance // labels are provided. The Observe method of the Observer in the ObserverVec // is called with the request size in bytes. Partitioning happens by HTTP // status code and/or HTTP method if the respective instance label names are // present in the ObserverVec. For unpartitioned observations, use an // ObserverVec with zero labels. Note that partitioning of Histograms is // expensive and should be used judiciously. // // If the wrapped Handler does not set a status code, a status code of 200 is assumed. // // If the wrapped Handler panics, no values are reported. // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { code, method := checkLabels(obs) if code { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) size := computeApproximateRequestSize(r) obs.With(labels(code, method, r.Method, d.Status())).Observe(float64(size)) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) size := computeApproximateRequestSize(r) obs.With(labels(code, method, r.Method, 0)).Observe(float64(size)) }) } // InstrumentHandlerResponseSize is a middleware that wraps the provided // http.Handler to observe the response size with the provided ObserverVec. // The ObserverVec must have zero, one, or two labels. The only allowed label // names are "code" and "method". The function panics if any other instance // labels are provided. The Observe method of the Observer in the ObserverVec // is called with the response size in bytes. Partitioning happens by HTTP // status code and/or HTTP method if the respective instance label names are // present in the ObserverVec. For unpartitioned observations, use an // ObserverVec with zero labels. Note that partitioning of Histograms is // expensive and should be used judiciously. // // If the wrapped Handler does not set a status code, a status code of 200 is assumed. // // If the wrapped Handler panics, no values are reported. // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler) http.Handler { code, method := checkLabels(obs) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) obs.With(labels(code, method, r.Method, d.Status())).Observe(float64(d.Written())) }) } func checkLabels(c prometheus.Collector) (code bool, method bool) { // TODO(beorn7): Remove this hacky way to check for instance labels // once Descriptors can have their dimensionality queried. var ( desc *prometheus.Desc pm dto.Metric ) descc := make(chan *prometheus.Desc, 1) c.Describe(descc) select { case desc = <-descc: default: panic("no description provided by collector") } select { case <-descc: panic("more than one description provided by collector") default: } close(descc) if _, err := prometheus.NewConstMetric(desc, prometheus.UntypedValue, 0); err == nil { return } if m, err := prometheus.NewConstMetric(desc, prometheus.UntypedValue, 0, magicString); err == nil { if err := m.Write(&pm); err != nil { panic("error checking metric for labels") } for _, label := range pm.Label { name, value := label.GetName(), label.GetValue() if value != magicString { continue } switch name { case "code": code = true case "method": method = true default: panic("metric partitioned with non-supported labels") } return } panic("previously set label not found – this must never happen") } if m, err := prometheus.NewConstMetric(desc, prometheus.UntypedValue, 0, magicString, magicString); err == nil { if err := m.Write(&pm); err != nil { panic("error checking metric for labels") } for _, label := range pm.Label { name, value := label.GetName(), label.GetValue() if value != magicString { continue } if name == "code" || name == "method" { continue } panic("metric partitioned with non-supported labels") } code = true method = true return } panic("metric partitioned with non-supported labels") } // emptyLabels is a one-time allocation for non-partitioned metrics to avoid // unnecessary allocations on each request. var emptyLabels = prometheus.Labels{} func labels(code, method bool, reqMethod string, status int) prometheus.Labels { if !(code || method) { return emptyLabels } labels := prometheus.Labels{} if code { labels["code"] = sanitizeCode(status) } if method { labels["method"] = sanitizeMethod(reqMethod) } return labels } func computeApproximateRequestSize(r *http.Request) int { s := 0 if r.URL != nil { s += len(r.URL.String()) } s += len(r.Method) s += len(r.Proto) for name, values := range r.Header { s += len(name) for _, value := range values { s += len(value) } } s += len(r.Host) // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. if r.ContentLength != -1 { s += int(r.ContentLength) } return s } func sanitizeMethod(m string) string { switch m { case "GET", "get": return "get" case "PUT", "put": return "put" case "HEAD", "head": return "head" case "POST", "post": return "post" case "DELETE", "delete": return "delete" case "CONNECT", "connect": return "connect" case "OPTIONS", "options": return "options" case "NOTIFY", "notify": return "notify" default: return strings.ToLower(m) } } // If the wrapped http.Handler has not set a status code, i.e. the value is // currently 0, santizeCode will return 200, for consistency with behavior in // the stdlib. func sanitizeCode(s int) string { switch s { case 100: return "100" case 101: return "101" case 200, 0: return "200" case 201: return "201" case 202: return "202" case 203: return "203" case 204: return "204" case 205: return "205" case 206: return "206" case 300: return "300" case 301: return "301" case 302: return "302" case 304: return "304" case 305: return "305" case 307: return "307" case 400: return "400" case 401: return "401" case 402: return "402" case 403: return "403" case 404: return "404" case 405: return "405" case 406: return "406" case 407: return "407" case 408: return "408" case 409: return "409" case 410: return "410" case 411: return "411" case 412: return "412" case 413: return "413" case 414: return "414" case 415: return "415" case 416: return "416" case 417: return "417" case 418: return "418" case 500: return "500" case 501: return "501" case 502: return "502" case 503: return "503" case 504: return "504" case 505: return "505" case 428: return "428" case 429: return "429" case 431: return "431" case 511: return "511" default: return strconv.Itoa(s) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server_test.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package promhttp import ( "io" "log" "net/http" "net/http/httptest" "testing" "github.com/prometheus/client_golang/prometheus" ) func TestMiddlewareAPI(t *testing.T) { reg := prometheus.NewRegistry() inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "in_flight_requests", Help: "A gauge of requests currently being served by the wrapped handler.", }) counter := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "api_requests_total", Help: "A counter for requests to the wrapped handler.", }, []string{"code", "method"}, ) histVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "response_duration_seconds", Help: "A histogram of request latencies.", Buckets: prometheus.DefBuckets, ConstLabels: prometheus.Labels{"handler": "api"}, }, []string{"method"}, ) writeHeaderVec := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "write_header_duration_seconds", Help: "A histogram of time to first write latencies.", Buckets: prometheus.DefBuckets, ConstLabels: prometheus.Labels{"handler": "api"}, }, []string{}, ) responseSize := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "push_request_size_bytes", Help: "A histogram of request sizes for requests.", Buckets: []float64{200, 500, 900, 1500}, }, []string{}, ) handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("OK")) }) reg.MustRegister(inFlightGauge, counter, histVec, responseSize, writeHeaderVec) chain := InstrumentHandlerInFlight(inFlightGauge, InstrumentHandlerCounter(counter, InstrumentHandlerDuration(histVec, InstrumentHandlerTimeToWriteHeader(writeHeaderVec, InstrumentHandlerResponseSize(responseSize, handler), ), ), ), ) r, _ := http.NewRequest("GET", "www.example.com", nil) w := httptest.NewRecorder() chain.ServeHTTP(w, r) } func TestInstrumentTimeToFirstWrite(t *testing.T) { var i int dobs := &responseWriterDelegator{ ResponseWriter: httptest.NewRecorder(), observeWriteHeader: func(status int) { i = status }, } d := newDelegator(dobs, nil) d.WriteHeader(http.StatusOK) if i != http.StatusOK { t.Fatalf("failed to execute observeWriteHeader") } } // testResponseWriter is an http.ResponseWriter that also implements // http.CloseNotifier, http.Flusher, and io.ReaderFrom. type testResponseWriter struct { closeNotifyCalled, flushCalled, readFromCalled bool } func (t *testResponseWriter) Header() http.Header { return nil } func (t *testResponseWriter) Write([]byte) (int, error) { return 0, nil } func (t *testResponseWriter) WriteHeader(int) {} func (t *testResponseWriter) CloseNotify() <-chan bool { t.closeNotifyCalled = true return nil } func (t *testResponseWriter) Flush() { t.flushCalled = true } func (t *testResponseWriter) ReadFrom(io.Reader) (int64, error) { t.readFromCalled = true return 0, nil } func TestInterfaceUpgrade(t *testing.T) { w := &testResponseWriter{} d := newDelegator(w, nil) d.(http.CloseNotifier).CloseNotify() if !w.closeNotifyCalled { t.Error("CloseNotify not called") } d.(http.Flusher).Flush() if !w.flushCalled { t.Error("Flush not called") } d.(io.ReaderFrom).ReadFrom(nil) if !w.readFromCalled { t.Error("ReadFrom not called") } if _, ok := d.(http.Hijacker); ok { t.Error("delegator unexpectedly implements http.Hijacker") } } func ExampleInstrumentHandlerDuration() { inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "in_flight_requests", Help: "A gauge of requests currently being served by the wrapped handler.", }) counter := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "api_requests_total", Help: "A counter for requests to the wrapped handler.", }, []string{"code", "method"}, ) // pushVec and pullVec are partitioned by the HTTP method and use custom // buckets based on the expected request duration. ConstLabels are used // to set a handler label to mark pushVec as tracking the durations for // pushes and pullVec as tracking the durations for pulls. Note that // Name, Help, and Buckets need to be the same for consistency, so we // use the same HistogramOpts after just modifying the ConstLabels. histogramOpts := prometheus.HistogramOpts{ Name: "request_duration_seconds", Help: "A histogram of latencies for requests.", Buckets: []float64{.25, .5, 1, 2.5, 5, 10}, ConstLabels: prometheus.Labels{"handler": "push"}, } pushVec := prometheus.NewHistogramVec( histogramOpts, []string{"method"}, ) histogramOpts.ConstLabels = prometheus.Labels{"handler": "pull"} pullVec := prometheus.NewHistogramVec( histogramOpts, []string{"method"}, ) // responseSize has no labels, making it a zero-dimensional // ObserverVec. responseSize := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "response_size_bytes", Help: "A histogram of response sizes for requests.", Buckets: []float64{200, 500, 900, 1500}, }, []string{}, ) // Create the handlers that will be wrapped by the middleware. pushHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Push")) }) pullHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Pull")) }) // Register all of the metrics in the standard registry. prometheus.MustRegister(inFlightGauge, counter, pullVec, pushVec, responseSize) // Wrap the pushHandler with our shared middleware, but use the // endpoint-specific pushVec with InstrumentHandlerDuration. pushChain := InstrumentHandlerInFlight(inFlightGauge, InstrumentHandlerCounter(counter, InstrumentHandlerDuration(pushVec, InstrumentHandlerResponseSize(responseSize, pushHandler), ), ), ) // Wrap the pushHandler with the shared middleware, but use the // endpoint-specific pullVec with InstrumentHandlerDuration. pullChain := InstrumentHandlerInFlight(inFlightGauge, InstrumentHandlerCounter(counter, InstrumentHandlerDuration(pullVec, InstrumentHandlerResponseSize(responseSize, pullHandler), ), ), ) http.Handle("/metrics", Handler()) http.Handle("/push", pushChain) http.Handle("/pull", pullChain) if err := http.ListenAndServe(":3000", nil); err != nil { log.Fatal(err) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/registry.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "bytes" "errors" "fmt" "os" "sort" "sync" "unicode/utf8" "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) const ( // Capacity for the channel to collect metrics and descriptors. capMetricChan = 1000 capDescChan = 10 ) // DefaultRegisterer and DefaultGatherer are the implementations of the // Registerer and Gatherer interface a number of convenience functions in this // package act on. Initially, both variables point to the same Registry, which // has a process collector (see NewProcessCollector) and a Go collector (see // NewGoCollector) already registered. This approach to keep default instances // as global state mirrors the approach of other packages in the Go standard // library. Note that there are caveats. Change the variables with caution and // only if you understand the consequences. Users who want to avoid global state // altogether should not use the convenience function and act on custom // instances instead. var ( defaultRegistry = NewRegistry() DefaultRegisterer Registerer = defaultRegistry DefaultGatherer Gatherer = defaultRegistry ) func init() { MustRegister(NewProcessCollector(os.Getpid(), "")) MustRegister(NewGoCollector()) } // NewRegistry creates a new vanilla Registry without any Collectors // pre-registered. func NewRegistry() *Registry { return &Registry{ collectorsByID: map[uint64]Collector{}, descIDs: map[uint64]struct{}{}, dimHashesByName: map[string]uint64{}, } } // NewPedanticRegistry returns a registry that checks during collection if each // collected Metric is consistent with its reported Desc, and if the Desc has // actually been registered with the registry. // // Usually, a Registry will be happy as long as the union of all collected // Metrics is consistent and valid even if some metrics are not consistent with // their own Desc or a Desc provided by their registered Collector. Well-behaved // Collectors and Metrics will only provide consistent Descs. This Registry is // useful to test the implementation of Collectors and Metrics. func NewPedanticRegistry() *Registry { r := NewRegistry() r.pedanticChecksEnabled = true return r } // Registerer is the interface for the part of a registry in charge of // registering and unregistering. Users of custom registries should use // Registerer as type for registration purposes (rather than the Registry type // directly). In that way, they are free to use custom Registerer implementation // (e.g. for testing purposes). type Registerer interface { // Register registers a new Collector to be included in metrics // collection. It returns an error if the descriptors provided by the // Collector are invalid or if they — in combination with descriptors of // already registered Collectors — do not fulfill the consistency and // uniqueness criteria described in the documentation of metric.Desc. // // If the provided Collector is equal to a Collector already registered // (which includes the case of re-registering the same Collector), the // returned error is an instance of AlreadyRegisteredError, which // contains the previously registered Collector. // // It is in general not safe to register the same Collector multiple // times concurrently. Register(Collector) error // MustRegister works like Register but registers any number of // Collectors and panics upon the first registration that causes an // error. MustRegister(...Collector) // Unregister unregisters the Collector that equals the Collector passed // in as an argument. (Two Collectors are considered equal if their // Describe method yields the same set of descriptors.) The function // returns whether a Collector was unregistered. // // Note that even after unregistering, it will not be possible to // register a new Collector that is inconsistent with the unregistered // Collector, e.g. a Collector collecting metrics with the same name but // a different help string. The rationale here is that the same registry // instance must only collect consistent metrics throughout its // lifetime. Unregister(Collector) bool } // Gatherer is the interface for the part of a registry in charge of gathering // the collected metrics into a number of MetricFamilies. The Gatherer interface // comes with the same general implication as described for the Registerer // interface. type Gatherer interface { // Gather calls the Collect method of the registered Collectors and then // gathers the collected metrics into a lexicographically sorted slice // of MetricFamily protobufs. Even if an error occurs, Gather attempts // to gather as many metrics as possible. Hence, if a non-nil error is // returned, the returned MetricFamily slice could be nil (in case of a // fatal error that prevented any meaningful metric collection) or // contain a number of MetricFamily protobufs, some of which might be // incomplete, and some might be missing altogether. The returned error // (which might be a MultiError) explains the details. In scenarios // where complete collection is critical, the returned MetricFamily // protobufs should be disregarded if the returned error is non-nil. Gather() ([]*dto.MetricFamily, error) } // Register registers the provided Collector with the DefaultRegisterer. // // Register is a shortcut for DefaultRegisterer.Register(c). See there for more // details. func Register(c Collector) error { return DefaultRegisterer.Register(c) } // MustRegister registers the provided Collectors with the DefaultRegisterer and // panics if any error occurs. // // MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See // there for more details. func MustRegister(cs ...Collector) { DefaultRegisterer.MustRegister(cs...) } // Unregister removes the registration of the provided Collector from the // DefaultRegisterer. // // Unregister is a shortcut for DefaultRegisterer.Unregister(c). See there for // more details. func Unregister(c Collector) bool { return DefaultRegisterer.Unregister(c) } // GathererFunc turns a function into a Gatherer. type GathererFunc func() ([]*dto.MetricFamily, error) // Gather implements Gatherer. func (gf GathererFunc) Gather() ([]*dto.MetricFamily, error) { return gf() } // AlreadyRegisteredError is returned by the Register method if the Collector to // be registered has already been registered before, or a different Collector // that collects the same metrics has been registered before. Registration fails // in that case, but you can detect from the kind of error what has // happened. The error contains fields for the existing Collector and the // (rejected) new Collector that equals the existing one. This can be used to // find out if an equal Collector has been registered before and switch over to // using the old one, as demonstrated in the example. type AlreadyRegisteredError struct { ExistingCollector, NewCollector Collector } func (err AlreadyRegisteredError) Error() string { return "duplicate metrics collector registration attempted" } // MultiError is a slice of errors implementing the error interface. It is used // by a Gatherer to report multiple errors during MetricFamily gathering. type MultiError []error func (errs MultiError) Error() string { if len(errs) == 0 { return "" } buf := &bytes.Buffer{} fmt.Fprintf(buf, "%d error(s) occurred:", len(errs)) for _, err := range errs { fmt.Fprintf(buf, "\n* %s", err) } return buf.String() } // MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only // contained error as error if len(errs is 1). In all other cases, it returns // the MultiError directly. This is helpful for returning a MultiError in a way // that only uses the MultiError if needed. func (errs MultiError) MaybeUnwrap() error { switch len(errs) { case 0: return nil case 1: return errs[0] default: return errs } } // Registry registers Prometheus collectors, collects their metrics, and gathers // them into MetricFamilies for exposition. It implements both Registerer and // Gatherer. The zero value is not usable. Create instances with NewRegistry or // NewPedanticRegistry. type Registry struct { mtx sync.RWMutex collectorsByID map[uint64]Collector // ID is a hash of the descIDs. descIDs map[uint64]struct{} dimHashesByName map[string]uint64 pedanticChecksEnabled bool } // Register implements Registerer. func (r *Registry) Register(c Collector) error { var ( descChan = make(chan *Desc, capDescChan) newDescIDs = map[uint64]struct{}{} newDimHashesByName = map[string]uint64{} collectorID uint64 // Just a sum of all desc IDs. duplicateDescErr error ) go func() { c.Describe(descChan) close(descChan) }() r.mtx.Lock() defer r.mtx.Unlock() // Conduct various tests... for desc := range descChan { // Is the descriptor valid at all? if desc.err != nil { return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err) } // Is the descID unique? // (In other words: Is the fqName + constLabel combination unique?) if _, exists := r.descIDs[desc.id]; exists { duplicateDescErr = fmt.Errorf("descriptor %s already exists with the same fully-qualified name and const label values", desc) } // If it is not a duplicate desc in this collector, add it to // the collectorID. (We allow duplicate descs within the same // collector, but their existence must be a no-op.) if _, exists := newDescIDs[desc.id]; !exists { newDescIDs[desc.id] = struct{}{} collectorID += desc.id } // Are all the label names and the help string consistent with // previous descriptors of the same name? // First check existing descriptors... if dimHash, exists := r.dimHashesByName[desc.fqName]; exists { if dimHash != desc.dimHash { return fmt.Errorf("a previously registered descriptor with the same fully-qualified name as %s has different label names or a different help string", desc) } } else { // ...then check the new descriptors already seen. if dimHash, exists := newDimHashesByName[desc.fqName]; exists { if dimHash != desc.dimHash { return fmt.Errorf("descriptors reported by collector have inconsistent label names or help strings for the same fully-qualified name, offender is %s", desc) } } else { newDimHashesByName[desc.fqName] = desc.dimHash } } } // Did anything happen at all? if len(newDescIDs) == 0 { return errors.New("collector has no descriptors") } if existing, exists := r.collectorsByID[collectorID]; exists { return AlreadyRegisteredError{ ExistingCollector: existing, NewCollector: c, } } // If the collectorID is new, but at least one of the descs existed // before, we are in trouble. if duplicateDescErr != nil { return duplicateDescErr } // Only after all tests have passed, actually register. r.collectorsByID[collectorID] = c for hash := range newDescIDs { r.descIDs[hash] = struct{}{} } for name, dimHash := range newDimHashesByName { r.dimHashesByName[name] = dimHash } return nil } // Unregister implements Registerer. func (r *Registry) Unregister(c Collector) bool { var ( descChan = make(chan *Desc, capDescChan) descIDs = map[uint64]struct{}{} collectorID uint64 // Just a sum of the desc IDs. ) go func() { c.Describe(descChan) close(descChan) }() for desc := range descChan { if _, exists := descIDs[desc.id]; !exists { collectorID += desc.id descIDs[desc.id] = struct{}{} } } r.mtx.RLock() if _, exists := r.collectorsByID[collectorID]; !exists { r.mtx.RUnlock() return false } r.mtx.RUnlock() r.mtx.Lock() defer r.mtx.Unlock() delete(r.collectorsByID, collectorID) for id := range descIDs { delete(r.descIDs, id) } // dimHashesByName is left untouched as those must be consistent // throughout the lifetime of a program. return true } // MustRegister implements Registerer. func (r *Registry) MustRegister(cs ...Collector) { for _, c := range cs { if err := r.Register(c); err != nil { panic(err) } } } // Gather implements Gatherer. func (r *Registry) Gather() ([]*dto.MetricFamily, error) { var ( metricChan = make(chan Metric, capMetricChan) metricHashes = map[uint64]struct{}{} dimHashes = map[string]uint64{} wg sync.WaitGroup errs MultiError // The collected errors to return in the end. registeredDescIDs map[uint64]struct{} // Only used for pedantic checks ) r.mtx.RLock() metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName)) // Scatter. // (Collectors could be complex and slow, so we call them all at once.) wg.Add(len(r.collectorsByID)) go func() { wg.Wait() close(metricChan) }() for _, collector := range r.collectorsByID { go func(collector Collector) { defer wg.Done() collector.Collect(metricChan) }(collector) } // In case pedantic checks are enabled, we have to copy the map before // giving up the RLock. if r.pedanticChecksEnabled { registeredDescIDs = make(map[uint64]struct{}, len(r.descIDs)) for id := range r.descIDs { registeredDescIDs[id] = struct{}{} } } r.mtx.RUnlock() // Drain metricChan in case of premature return. defer func() { for range metricChan { } }() // Gather. for metric := range metricChan { // This could be done concurrently, too, but it required locking // of metricFamiliesByName (and of metricHashes if checks are // enabled). Most likely not worth it. desc := metric.Desc() dtoMetric := &dto.Metric{} if err := metric.Write(dtoMetric); err != nil { errs = append(errs, fmt.Errorf( "error collecting metric %v: %s", desc, err, )) continue } metricFamily, ok := metricFamiliesByName[desc.fqName] if ok { if metricFamily.GetHelp() != desc.help { errs = append(errs, fmt.Errorf( "collected metric %s %s has help %q but should have %q", desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(), )) continue } // TODO(beorn7): Simplify switch once Desc has type. switch metricFamily.GetType() { case dto.MetricType_COUNTER: if dtoMetric.Counter == nil { errs = append(errs, fmt.Errorf( "collected metric %s %s should be a Counter", desc.fqName, dtoMetric, )) continue } case dto.MetricType_GAUGE: if dtoMetric.Gauge == nil { errs = append(errs, fmt.Errorf( "collected metric %s %s should be a Gauge", desc.fqName, dtoMetric, )) continue } case dto.MetricType_SUMMARY: if dtoMetric.Summary == nil { errs = append(errs, fmt.Errorf( "collected metric %s %s should be a Summary", desc.fqName, dtoMetric, )) continue } case dto.MetricType_UNTYPED: if dtoMetric.Untyped == nil { errs = append(errs, fmt.Errorf( "collected metric %s %s should be Untyped", desc.fqName, dtoMetric, )) continue } case dto.MetricType_HISTOGRAM: if dtoMetric.Histogram == nil { errs = append(errs, fmt.Errorf( "collected metric %s %s should be a Histogram", desc.fqName, dtoMetric, )) continue } default: panic("encountered MetricFamily with invalid type") } } else { metricFamily = &dto.MetricFamily{} metricFamily.Name = proto.String(desc.fqName) metricFamily.Help = proto.String(desc.help) // TODO(beorn7): Simplify switch once Desc has type. switch { case dtoMetric.Gauge != nil: metricFamily.Type = dto.MetricType_GAUGE.Enum() case dtoMetric.Counter != nil: metricFamily.Type = dto.MetricType_COUNTER.Enum() case dtoMetric.Summary != nil: metricFamily.Type = dto.MetricType_SUMMARY.Enum() case dtoMetric.Untyped != nil: metricFamily.Type = dto.MetricType_UNTYPED.Enum() case dtoMetric.Histogram != nil: metricFamily.Type = dto.MetricType_HISTOGRAM.Enum() default: errs = append(errs, fmt.Errorf( "empty metric collected: %s", dtoMetric, )) continue } metricFamiliesByName[desc.fqName] = metricFamily } if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes, dimHashes); err != nil { errs = append(errs, err) continue } if r.pedanticChecksEnabled { // Is the desc registered at all? if _, exist := registeredDescIDs[desc.id]; !exist { errs = append(errs, fmt.Errorf( "collected metric %s %s with unregistered descriptor %s", metricFamily.GetName(), dtoMetric, desc, )) continue } if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil { errs = append(errs, err) continue } } metricFamily.Metric = append(metricFamily.Metric, dtoMetric) } return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() } // Gatherers is a slice of Gatherer instances that implements the Gatherer // interface itself. Its Gather method calls Gather on all Gatherers in the // slice in order and returns the merged results. Errors returned from the // Gather calles are all returned in a flattened MultiError. Duplicate and // inconsistent Metrics are skipped (first occurrence in slice order wins) and // reported in the returned error. // // Gatherers can be used to merge the Gather results from multiple // Registries. It also provides a way to directly inject existing MetricFamily // protobufs into the gathering by creating a custom Gatherer with a Gather // method that simply returns the existing MetricFamily protobufs. Note that no // registration is involved (in contrast to Collector registration), so // obviously registration-time checks cannot happen. Any inconsistencies between // the gathered MetricFamilies are reported as errors by the Gather method, and // inconsistent Metrics are dropped. Invalid parts of the MetricFamilies // (e.g. syntactically invalid metric or label names) will go undetected. type Gatherers []Gatherer // Gather implements Gatherer. func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) { var ( metricFamiliesByName = map[string]*dto.MetricFamily{} metricHashes = map[uint64]struct{}{} dimHashes = map[string]uint64{} errs MultiError // The collected errors to return in the end. ) for i, g := range gs { mfs, err := g.Gather() if err != nil { if multiErr, ok := err.(MultiError); ok { for _, err := range multiErr { errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) } } else { errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) } } for _, mf := range mfs { existingMF, exists := metricFamiliesByName[mf.GetName()] if exists { if existingMF.GetHelp() != mf.GetHelp() { errs = append(errs, fmt.Errorf( "gathered metric family %s has help %q but should have %q", mf.GetName(), mf.GetHelp(), existingMF.GetHelp(), )) continue } if existingMF.GetType() != mf.GetType() { errs = append(errs, fmt.Errorf( "gathered metric family %s has type %s but should have %s", mf.GetName(), mf.GetType(), existingMF.GetType(), )) continue } } else { existingMF = &dto.MetricFamily{} existingMF.Name = mf.Name existingMF.Help = mf.Help existingMF.Type = mf.Type metricFamiliesByName[mf.GetName()] = existingMF } for _, m := range mf.Metric { if err := checkMetricConsistency(existingMF, m, metricHashes, dimHashes); err != nil { errs = append(errs, err) continue } existingMF.Metric = append(existingMF.Metric, m) } } } return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() } // metricSorter is a sortable slice of *dto.Metric. type metricSorter []*dto.Metric func (s metricSorter) Len() int { return len(s) } func (s metricSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s metricSorter) Less(i, j int) bool { if len(s[i].Label) != len(s[j].Label) { // This should not happen. The metrics are // inconsistent. However, we have to deal with the fact, as // people might use custom collectors or metric family injection // to create inconsistent metrics. So let's simply compare the // number of labels in this case. That will still yield // reproducible sorting. return len(s[i].Label) < len(s[j].Label) } for n, lp := range s[i].Label { vi := lp.GetValue() vj := s[j].Label[n].GetValue() if vi != vj { return vi < vj } } // We should never arrive here. Multiple metrics with the same // label set in the same scrape will lead to undefined ingestion // behavior. However, as above, we have to provide stable sorting // here, even for inconsistent metrics. So sort equal metrics // by their timestamp, with missing timestamps (implying "now") // coming last. if s[i].TimestampMs == nil { return false } if s[j].TimestampMs == nil { return true } return s[i].GetTimestampMs() < s[j].GetTimestampMs() } // normalizeMetricFamilies returns a MetricFamily slice with empty // MetricFamilies pruned and the remaining MetricFamilies sorted by name within // the slice, with the contained Metrics sorted within each MetricFamily. func normalizeMetricFamilies(metricFamiliesByName map[string]*dto.MetricFamily) []*dto.MetricFamily { for _, mf := range metricFamiliesByName { sort.Sort(metricSorter(mf.Metric)) } names := make([]string, 0, len(metricFamiliesByName)) for name, mf := range metricFamiliesByName { if len(mf.Metric) > 0 { names = append(names, name) } } sort.Strings(names) result := make([]*dto.MetricFamily, 0, len(names)) for _, name := range names { result = append(result, metricFamiliesByName[name]) } return result } // checkMetricConsistency checks if the provided Metric is consistent with the // provided MetricFamily. It also hashed the Metric labels and the MetricFamily // name. If the resulting hash is alread in the provided metricHashes, an error // is returned. If not, it is added to metricHashes. The provided dimHashes maps // MetricFamily names to their dimHash (hashed sorted label names). If dimHashes // doesn't yet contain a hash for the provided MetricFamily, it is // added. Otherwise, an error is returned if the existing dimHashes in not equal // the calculated dimHash. func checkMetricConsistency( metricFamily *dto.MetricFamily, dtoMetric *dto.Metric, metricHashes map[uint64]struct{}, dimHashes map[string]uint64, ) error { // Type consistency with metric family. if metricFamily.GetType() == dto.MetricType_GAUGE && dtoMetric.Gauge == nil || metricFamily.GetType() == dto.MetricType_COUNTER && dtoMetric.Counter == nil || metricFamily.GetType() == dto.MetricType_SUMMARY && dtoMetric.Summary == nil || metricFamily.GetType() == dto.MetricType_HISTOGRAM && dtoMetric.Histogram == nil || metricFamily.GetType() == dto.MetricType_UNTYPED && dtoMetric.Untyped == nil { return fmt.Errorf( "collected metric %s %s is not a %s", metricFamily.GetName(), dtoMetric, metricFamily.GetType(), ) } for _, labelPair := range dtoMetric.GetLabel() { if !utf8.ValidString(*labelPair.Value) { return fmt.Errorf("collected metric's label %s is not utf8: %#v", *labelPair.Name, *labelPair.Value) } } // Is the metric unique (i.e. no other metric with the same name and the same label values)? h := hashNew() h = hashAdd(h, metricFamily.GetName()) h = hashAddByte(h, separatorByte) dh := hashNew() // Make sure label pairs are sorted. We depend on it for the consistency // check. sort.Sort(LabelPairSorter(dtoMetric.Label)) for _, lp := range dtoMetric.Label { h = hashAdd(h, lp.GetValue()) h = hashAddByte(h, separatorByte) dh = hashAdd(dh, lp.GetName()) dh = hashAddByte(dh, separatorByte) } if _, exists := metricHashes[h]; exists { return fmt.Errorf( "collected metric %s %s was collected before with the same name and label values", metricFamily.GetName(), dtoMetric, ) } if dimHash, ok := dimHashes[metricFamily.GetName()]; ok { if dimHash != dh { return fmt.Errorf( "collected metric %s %s has label dimensions inconsistent with previously collected metrics in the same metric family", metricFamily.GetName(), dtoMetric, ) } } else { dimHashes[metricFamily.GetName()] = dh } metricHashes[h] = struct{}{} return nil } func checkDescConsistency( metricFamily *dto.MetricFamily, dtoMetric *dto.Metric, desc *Desc, ) error { // Desc help consistency with metric family help. if metricFamily.GetHelp() != desc.help { return fmt.Errorf( "collected metric %s %s has help %q but should have %q", metricFamily.GetName(), dtoMetric, metricFamily.GetHelp(), desc.help, ) } // Is the desc consistent with the content of the metric? lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label)) lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...) for _, l := range desc.variableLabels { lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ Name: proto.String(l), }) } if len(lpsFromDesc) != len(dtoMetric.Label) { return fmt.Errorf( "labels in collected metric %s %s are inconsistent with descriptor %s", metricFamily.GetName(), dtoMetric, desc, ) } sort.Sort(LabelPairSorter(lpsFromDesc)) for i, lpFromDesc := range lpsFromDesc { lpFromMetric := dtoMetric.Label[i] if lpFromDesc.GetName() != lpFromMetric.GetName() || lpFromDesc.Value != nil && lpFromDesc.GetValue() != lpFromMetric.GetValue() { return fmt.Errorf( "labels in collected metric %s %s are inconsistent with descriptor %s", metricFamily.GetName(), dtoMetric, desc, ) } } return nil } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/registry_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Copyright (c) 2013, The Prometheus Authors // All rights reserved. // // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. package prometheus_test import ( "bytes" "net/http" "net/http/httptest" "testing" dto "github.com/prometheus/client_model/go" "github.com/golang/protobuf/proto" "github.com/prometheus/common/expfmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) func testHandler(t testing.TB) { metricVec := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "name", Help: "docstring", ConstLabels: prometheus.Labels{"constname": "constvalue"}, }, []string{"labelname"}, ) metricVec.WithLabelValues("val1").Inc() metricVec.WithLabelValues("val2").Inc() externalMetricFamily := &dto.MetricFamily{ Name: proto.String("externalname"), Help: proto.String("externaldocstring"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ { Label: []*dto.LabelPair{ { Name: proto.String("externalconstname"), Value: proto.String("externalconstvalue"), }, { Name: proto.String("externallabelname"), Value: proto.String("externalval1"), }, }, Counter: &dto.Counter{ Value: proto.Float64(1), }, }, }, } externalBuf := &bytes.Buffer{} enc := expfmt.NewEncoder(externalBuf, expfmt.FmtProtoDelim) if err := enc.Encode(externalMetricFamily); err != nil { t.Fatal(err) } externalMetricFamilyAsBytes := externalBuf.Bytes() externalMetricFamilyAsText := []byte(`# HELP externalname externaldocstring # TYPE externalname counter externalname{externalconstname="externalconstvalue",externallabelname="externalval1"} 1 `) externalMetricFamilyAsProtoText := []byte(`name: "externalname" help: "externaldocstring" type: COUNTER metric: < label: < name: "externalconstname" value: "externalconstvalue" > label: < name: "externallabelname" value: "externalval1" > counter: < value: 1 > > `) externalMetricFamilyAsProtoCompactText := []byte(`name:"externalname" help:"externaldocstring" type:COUNTER metric: label: counter: > `) expectedMetricFamily := &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("docstring"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ { Label: []*dto.LabelPair{ { Name: proto.String("constname"), Value: proto.String("constvalue"), }, { Name: proto.String("labelname"), Value: proto.String("val1"), }, }, Counter: &dto.Counter{ Value: proto.Float64(1), }, }, { Label: []*dto.LabelPair{ { Name: proto.String("constname"), Value: proto.String("constvalue"), }, { Name: proto.String("labelname"), Value: proto.String("val2"), }, }, Counter: &dto.Counter{ Value: proto.Float64(1), }, }, }, } buf := &bytes.Buffer{} enc = expfmt.NewEncoder(buf, expfmt.FmtProtoDelim) if err := enc.Encode(expectedMetricFamily); err != nil { t.Fatal(err) } expectedMetricFamilyAsBytes := buf.Bytes() expectedMetricFamilyAsText := []byte(`# HELP name docstring # TYPE name counter name{constname="constvalue",labelname="val1"} 1 name{constname="constvalue",labelname="val2"} 1 `) expectedMetricFamilyAsProtoText := []byte(`name: "name" help: "docstring" type: COUNTER metric: < label: < name: "constname" value: "constvalue" > label: < name: "labelname" value: "val1" > counter: < value: 1 > > metric: < label: < name: "constname" value: "constvalue" > label: < name: "labelname" value: "val2" > counter: < value: 1 > > `) expectedMetricFamilyAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric: label: counter: > metric: label: counter: > `) externalMetricFamilyWithSameName := &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("docstring"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ { Label: []*dto.LabelPair{ { Name: proto.String("constname"), Value: proto.String("constvalue"), }, { Name: proto.String("labelname"), Value: proto.String("different_val"), }, }, Counter: &dto.Counter{ Value: proto.Float64(42), }, }, }, } expectedMetricFamilyMergedWithExternalAsProtoCompactText := []byte(`name:"name" help:"docstring" type:COUNTER metric: label: counter: > metric: label: counter: > metric: label: counter: > `) externalMetricFamilyWithInvalidLabelValue := &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("docstring"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ { Label: []*dto.LabelPair{ { Name: proto.String("constname"), Value: proto.String("\xFF"), }, { Name: proto.String("labelname"), Value: proto.String("different_val"), }, }, Counter: &dto.Counter{ Value: proto.Float64(42), }, }, }, } expectedMetricFamilyInvalidLabelValueAsText := []byte(`An error has occurred during metrics gathering: collected metric's label constname is not utf8: "\xff" `) type output struct { headers map[string]string body []byte } var scenarios = []struct { headers map[string]string out output collector prometheus.Collector externalMF []*dto.MetricFamily }{ { // 0 headers: map[string]string{ "Accept": "foo/bar;q=0.2, dings/bums;q=0.8", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: []byte{}, }, }, { // 1 headers: map[string]string{ "Accept": "foo/bar;q=0.2, application/quark;q=0.8", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: []byte{}, }, }, { // 2 headers: map[string]string{ "Accept": "foo/bar;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.8", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: []byte{}, }, }, { // 3 headers: map[string]string{ "Accept": "text/plain;q=0.2, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.8", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`, }, body: []byte{}, }, }, { // 4 headers: map[string]string{ "Accept": "application/json", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: expectedMetricFamilyAsText, }, collector: metricVec, }, { // 5 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`, }, body: expectedMetricFamilyAsBytes, }, collector: metricVec, }, { // 6 headers: map[string]string{ "Accept": "application/json", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: externalMetricFamilyAsText, }, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 7 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`, }, body: externalMetricFamilyAsBytes, }, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 8 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`, }, body: bytes.Join( [][]byte{ externalMetricFamilyAsBytes, expectedMetricFamilyAsBytes, }, []byte{}, ), }, collector: metricVec, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 9 headers: map[string]string{ "Accept": "text/plain", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: []byte{}, }, }, { // 10 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: expectedMetricFamilyAsText, }, collector: metricVec, }, { // 11 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=bla;q=0.2, text/plain;q=0.5;version=0.0.4", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; version=0.0.4`, }, body: bytes.Join( [][]byte{ externalMetricFamilyAsText, expectedMetricFamilyAsText, }, []byte{}, ), }, collector: metricVec, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 12 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.2, text/plain;q=0.5;version=0.0.2", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited`, }, body: bytes.Join( [][]byte{ externalMetricFamilyAsBytes, expectedMetricFamilyAsBytes, }, []byte{}, ), }, collector: metricVec, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 13 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=text;q=0.5, application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=text`, }, body: bytes.Join( [][]byte{ externalMetricFamilyAsProtoText, expectedMetricFamilyAsProtoText, }, []byte{}, ), }, collector: metricVec, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 14 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=compact-text`, }, body: bytes.Join( [][]byte{ externalMetricFamilyAsProtoCompactText, expectedMetricFamilyAsProtoCompactText, }, []byte{}, ), }, collector: metricVec, externalMF: []*dto.MetricFamily{externalMetricFamily}, }, { // 15 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text", }, out: output{ headers: map[string]string{ "Content-Type": `application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=compact-text`, }, body: bytes.Join( [][]byte{ externalMetricFamilyAsProtoCompactText, expectedMetricFamilyMergedWithExternalAsProtoCompactText, }, []byte{}, ), }, collector: metricVec, externalMF: []*dto.MetricFamily{ externalMetricFamily, externalMetricFamilyWithSameName, }, }, { // 16 headers: map[string]string{ "Accept": "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text", }, out: output{ headers: map[string]string{ "Content-Type": `text/plain; charset=utf-8`, }, body: expectedMetricFamilyInvalidLabelValueAsText, }, collector: metricVec, externalMF: []*dto.MetricFamily{ externalMetricFamily, externalMetricFamilyWithInvalidLabelValue, }, }, } for i, scenario := range scenarios { registry := prometheus.NewPedanticRegistry() gatherer := prometheus.Gatherer(registry) if scenario.externalMF != nil { gatherer = prometheus.Gatherers{ registry, prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return scenario.externalMF, nil }), } } if scenario.collector != nil { registry.Register(scenario.collector) } writer := httptest.NewRecorder() handler := prometheus.InstrumentHandler("prometheus", promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{})) request, _ := http.NewRequest("GET", "/", nil) for key, value := range scenario.headers { request.Header.Add(key, value) } handler(writer, request) for key, value := range scenario.out.headers { if writer.HeaderMap.Get(key) != value { t.Errorf( "%d. expected %q for header %q, got %q", i, value, key, writer.Header().Get(key), ) } } if !bytes.Equal(scenario.out.body, writer.Body.Bytes()) { t.Errorf( "%d. expected body:\n%s\ngot body:\n%s\n", i, scenario.out.body, writer.Body.Bytes(), ) } } } func TestHandler(t *testing.T) { testHandler(t) } func BenchmarkHandler(b *testing.B) { for i := 0; i < b.N; i++ { testHandler(b) } } func TestRegisterWithOrGet(t *testing.T) { // Replace the default registerer just to be sure. This is bad, but this // whole test will go away once RegisterOrGet is removed. oldRegisterer := prometheus.DefaultRegisterer defer func() { prometheus.DefaultRegisterer = oldRegisterer }() prometheus.DefaultRegisterer = prometheus.NewRegistry() original := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "test", Help: "help", }, []string{"foo", "bar"}, ) equalButNotSame := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "test", Help: "help", }, []string{"foo", "bar"}, ) var err error if err = prometheus.Register(original); err != nil { t.Fatal(err) } if err = prometheus.Register(equalButNotSame); err == nil { t.Fatal("expected error when registringe equal collector") } if are, ok := err.(prometheus.AlreadyRegisteredError); ok { if are.ExistingCollector != original { t.Error("expected original collector but got something else") } if are.ExistingCollector == equalButNotSame { t.Error("expected original callector but got new one") } } else { t.Error("unexpected error:", err) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/summary.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "fmt" "math" "sort" "sync" "time" "github.com/beorn7/perks/quantile" "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) // quantileLabel is used for the label that defines the quantile in a // summary. const quantileLabel = "quantile" // A Summary captures individual observations from an event or sample stream and // summarizes them in a manner similar to traditional summary statistics: 1. sum // of observations, 2. observation count, 3. rank estimations. // // A typical use-case is the observation of request latencies. By default, a // Summary provides the median, the 90th and the 99th percentile of the latency // as rank estimations. // // Note that the rank estimations cannot be aggregated in a meaningful way with // the Prometheus query language (i.e. you cannot average or add them). If you // need aggregatable quantiles (e.g. you want the 99th percentile latency of all // queries served across all instances of a service), consider the Histogram // metric type. See the Prometheus documentation for more details. // // To create Summary instances, use NewSummary. type Summary interface { Metric Collector // Observe adds a single observation to the summary. Observe(float64) } // DefObjectives are the default Summary quantile values. // // Deprecated: DefObjectives will not be used as the default objectives in // v0.10 of the library. The default Summary will have no quantiles then. var ( DefObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} errQuantileLabelNotAllowed = fmt.Errorf( "%q is not allowed as label name in summaries", quantileLabel, ) ) // Default values for SummaryOpts. const ( // DefMaxAge is the default duration for which observations stay // relevant. DefMaxAge time.Duration = 10 * time.Minute // DefAgeBuckets is the default number of buckets used to calculate the // age of observations. DefAgeBuckets = 5 // DefBufCap is the standard buffer size for collecting Summary observations. DefBufCap = 500 ) // SummaryOpts bundles the options for creating a Summary metric. It is // mandatory to set Name and Help to a non-empty string. All other fields are // optional and can safely be left at their zero value. type SummaryOpts struct { // Namespace, Subsystem, and Name are components of the fully-qualified // name of the Summary (created by joining these components with // "_"). Only Name is mandatory, the others merely help structuring the // name. Note that the fully-qualified name of the Summary must be a // valid Prometheus metric name. Namespace string Subsystem string Name string // Help provides information about this Summary. Mandatory! // // Metrics with the same fully-qualified name must have the same Help // string. Help string // ConstLabels are used to attach fixed labels to this // Summary. Summaries with the same fully-qualified name must have the // same label names in their ConstLabels. // // Note that in most cases, labels have a value that varies during the // lifetime of a process. Those labels are usually managed with a // SummaryVec. ConstLabels serve only special purposes. One is for the // special case where the value of a label does not change during the // lifetime of a process, e.g. if the revision of the running binary is // put into a label. Another, more advanced purpose is if more than one // Collector needs to collect Summaries with the same fully-qualified // name. In that case, those Summaries must differ in the values of // their ConstLabels. See the Collector examples. // // If the value of a label never changes (not even between binaries), // that label most likely should not be a label at all (but part of the // metric name). ConstLabels Labels // Objectives defines the quantile rank estimates with their respective // absolute error. If Objectives[q] = e, then the value reported for q // will be the φ-quantile value for some φ between q-e and q+e. The // default value is DefObjectives. It is used if Objectives is left at // its zero value (i.e. nil). To create a Summary without Objectives, // set it to an empty map (i.e. map[float64]float64{}). // // Deprecated: Note that the current value of DefObjectives is // deprecated. It will be replaced by an empty map in v0.10 of the // library. Please explicitly set Objectives to the desired value. Objectives map[float64]float64 // MaxAge defines the duration for which an observation stays relevant // for the summary. Must be positive. The default value is DefMaxAge. MaxAge time.Duration // AgeBuckets is the number of buckets used to exclude observations that // are older than MaxAge from the summary. A higher number has a // resource penalty, so only increase it if the higher resolution is // really required. For very high observation rates, you might want to // reduce the number of age buckets. With only one age bucket, you will // effectively see a complete reset of the summary each time MaxAge has // passed. The default value is DefAgeBuckets. AgeBuckets uint32 // BufCap defines the default sample stream buffer size. The default // value of DefBufCap should suffice for most uses. If there is a need // to increase the value, a multiple of 500 is recommended (because that // is the internal buffer size of the underlying package // "github.com/bmizerany/perks/quantile"). BufCap uint32 } // Great fuck-up with the sliding-window decay algorithm... The Merge method of // perk/quantile is actually not working as advertised - and it might be // unfixable, as the underlying algorithm is apparently not capable of merging // summaries in the first place. To avoid using Merge, we are currently adding // observations to _each_ age bucket, i.e. the effort to add a sample is // essentially multiplied by the number of age buckets. When rotating age // buckets, we empty the previous head stream. On scrape time, we simply take // the quantiles from the head stream (no merging required). Result: More effort // on observation time, less effort on scrape time, which is exactly the // opposite of what we try to accomplish, but at least the results are correct. // // The quite elegant previous contraption to merge the age buckets efficiently // on scrape time (see code up commit 6b9530d72ea715f0ba612c0120e6e09fbf1d49d0) // can't be used anymore. // NewSummary creates a new Summary based on the provided SummaryOpts. func NewSummary(opts SummaryOpts) Summary { return newSummary( NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ), opts, ) } func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary { if len(desc.variableLabels) != len(labelValues) { panic(errInconsistentCardinality) } for _, n := range desc.variableLabels { if n == quantileLabel { panic(errQuantileLabelNotAllowed) } } for _, lp := range desc.constLabelPairs { if lp.GetName() == quantileLabel { panic(errQuantileLabelNotAllowed) } } if opts.Objectives == nil { opts.Objectives = DefObjectives } if opts.MaxAge < 0 { panic(fmt.Errorf("illegal max age MaxAge=%v", opts.MaxAge)) } if opts.MaxAge == 0 { opts.MaxAge = DefMaxAge } if opts.AgeBuckets == 0 { opts.AgeBuckets = DefAgeBuckets } if opts.BufCap == 0 { opts.BufCap = DefBufCap } s := &summary{ desc: desc, objectives: opts.Objectives, sortedObjectives: make([]float64, 0, len(opts.Objectives)), labelPairs: makeLabelPairs(desc, labelValues), hotBuf: make([]float64, 0, opts.BufCap), coldBuf: make([]float64, 0, opts.BufCap), streamDuration: opts.MaxAge / time.Duration(opts.AgeBuckets), } s.headStreamExpTime = time.Now().Add(s.streamDuration) s.hotBufExpTime = s.headStreamExpTime for i := uint32(0); i < opts.AgeBuckets; i++ { s.streams = append(s.streams, s.newStream()) } s.headStream = s.streams[0] for qu := range s.objectives { s.sortedObjectives = append(s.sortedObjectives, qu) } sort.Float64s(s.sortedObjectives) s.init(s) // Init self-collection. return s } type summary struct { selfCollector bufMtx sync.Mutex // Protects hotBuf and hotBufExpTime. mtx sync.Mutex // Protects every other moving part. // Lock bufMtx before mtx if both are needed. desc *Desc objectives map[float64]float64 sortedObjectives []float64 labelPairs []*dto.LabelPair sum float64 cnt uint64 hotBuf, coldBuf []float64 streams []*quantile.Stream streamDuration time.Duration headStream *quantile.Stream headStreamIdx int headStreamExpTime, hotBufExpTime time.Time } func (s *summary) Desc() *Desc { return s.desc } func (s *summary) Observe(v float64) { s.bufMtx.Lock() defer s.bufMtx.Unlock() now := time.Now() if now.After(s.hotBufExpTime) { s.asyncFlush(now) } s.hotBuf = append(s.hotBuf, v) if len(s.hotBuf) == cap(s.hotBuf) { s.asyncFlush(now) } } func (s *summary) Write(out *dto.Metric) error { sum := &dto.Summary{} qs := make([]*dto.Quantile, 0, len(s.objectives)) s.bufMtx.Lock() s.mtx.Lock() // Swap bufs even if hotBuf is empty to set new hotBufExpTime. s.swapBufs(time.Now()) s.bufMtx.Unlock() s.flushColdBuf() sum.SampleCount = proto.Uint64(s.cnt) sum.SampleSum = proto.Float64(s.sum) for _, rank := range s.sortedObjectives { var q float64 if s.headStream.Count() == 0 { q = math.NaN() } else { q = s.headStream.Query(rank) } qs = append(qs, &dto.Quantile{ Quantile: proto.Float64(rank), Value: proto.Float64(q), }) } s.mtx.Unlock() if len(qs) > 0 { sort.Sort(quantSort(qs)) } sum.Quantile = qs out.Summary = sum out.Label = s.labelPairs return nil } func (s *summary) newStream() *quantile.Stream { return quantile.NewTargeted(s.objectives) } // asyncFlush needs bufMtx locked. func (s *summary) asyncFlush(now time.Time) { s.mtx.Lock() s.swapBufs(now) // Unblock the original goroutine that was responsible for the mutation // that triggered the compaction. But hold onto the global non-buffer // state mutex until the operation finishes. go func() { s.flushColdBuf() s.mtx.Unlock() }() } // rotateStreams needs mtx AND bufMtx locked. func (s *summary) maybeRotateStreams() { for !s.hotBufExpTime.Equal(s.headStreamExpTime) { s.headStream.Reset() s.headStreamIdx++ if s.headStreamIdx >= len(s.streams) { s.headStreamIdx = 0 } s.headStream = s.streams[s.headStreamIdx] s.headStreamExpTime = s.headStreamExpTime.Add(s.streamDuration) } } // flushColdBuf needs mtx locked. func (s *summary) flushColdBuf() { for _, v := range s.coldBuf { for _, stream := range s.streams { stream.Insert(v) } s.cnt++ s.sum += v } s.coldBuf = s.coldBuf[0:0] s.maybeRotateStreams() } // swapBufs needs mtx AND bufMtx locked, coldBuf must be empty. func (s *summary) swapBufs(now time.Time) { if len(s.coldBuf) != 0 { panic("coldBuf is not empty") } s.hotBuf, s.coldBuf = s.coldBuf, s.hotBuf // hotBuf is now empty and gets new expiration set. for now.After(s.hotBufExpTime) { s.hotBufExpTime = s.hotBufExpTime.Add(s.streamDuration) } } type quantSort []*dto.Quantile func (s quantSort) Len() int { return len(s) } func (s quantSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s quantSort) Less(i, j int) bool { return s[i].GetQuantile() < s[j].GetQuantile() } // SummaryVec is a Collector that bundles a set of Summaries that all share the // same Desc, but have different values for their variable labels. This is used // if you want to count the same thing partitioned by various dimensions // (e.g. HTTP request latencies, partitioned by status code and method). Create // instances with NewSummaryVec. type SummaryVec struct { *metricVec } // NewSummaryVec creates a new SummaryVec based on the provided SummaryOpts and // partitioned by the given label names. func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec { desc := NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, labelNames, opts.ConstLabels, ) return &SummaryVec{ metricVec: newMetricVec(desc, func(lvs ...string) Metric { return newSummary(desc, opts, lvs...) }), } } // GetMetricWithLabelValues returns the Summary for the given slice of label // values (same order as the VariableLabels in Desc). If that combination of // label values is accessed for the first time, a new Summary is created. // // It is possible to call this method without using the returned Summary to only // create the new Summary but leave it at its starting value, a Summary without // any observations. // // Keeping the Summary for later use is possible (and should be considered if // performance is critical), but keep in mind that Reset, DeleteLabelValues and // Delete can be used to delete the Summary from the SummaryVec. In that case, the // Summary will still exist, but it will not be exported anymore, even if a // Summary with the same label values is created later. See also the CounterVec // example. // // An error is returned if the number of label values is not the same as the // number of VariableLabels in Desc. // // Note that for more than one label value, this method is prone to mistakes // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // an alternative to avoid that type of mistake. For higher label numbers, the // latter has a much more readable (albeit more verbose) syntax, but it comes // with a performance overhead (for creating and processing the Labels map). // See also the GaugeVec example. func (m *SummaryVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) { metric, err := m.metricVec.getMetricWithLabelValues(lvs...) if metric != nil { return metric.(Observer), err } return nil, err } // GetMetricWith returns the Summary for the given Labels map (the label names // must match those of the VariableLabels in Desc). If that label map is // accessed for the first time, a new Summary is created. Implications of // creating a Summary without using it and keeping the Summary for later use are // the same as for GetMetricWithLabelValues. // // An error is returned if the number and names of the Labels are inconsistent // with those of the VariableLabels in Desc. // // This method is used for the same purpose as // GetMetricWithLabelValues(...string). See there for pros and cons of the two // methods. func (m *SummaryVec) GetMetricWith(labels Labels) (Observer, error) { metric, err := m.metricVec.getMetricWith(labels) if metric != nil { return metric.(Observer), err } return nil, err } // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. By not returning an // error, WithLabelValues allows shortcuts like // myVec.WithLabelValues("404", "GET").Observe(42.21) func (m *SummaryVec) WithLabelValues(lvs ...string) Observer { return m.metricVec.withLabelValues(lvs...).(Observer) } // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. By not returning an error, With allows shortcuts like // myVec.With(Labels{"code": "404", "method": "GET"}).Observe(42.21) func (m *SummaryVec) With(labels Labels) Observer { return m.metricVec.with(labels).(Observer) } type constSummary struct { desc *Desc count uint64 sum float64 quantiles map[float64]float64 labelPairs []*dto.LabelPair } func (s *constSummary) Desc() *Desc { return s.desc } func (s *constSummary) Write(out *dto.Metric) error { sum := &dto.Summary{} qs := make([]*dto.Quantile, 0, len(s.quantiles)) sum.SampleCount = proto.Uint64(s.count) sum.SampleSum = proto.Float64(s.sum) for rank, q := range s.quantiles { qs = append(qs, &dto.Quantile{ Quantile: proto.Float64(rank), Value: proto.Float64(q), }) } if len(qs) > 0 { sort.Sort(quantSort(qs)) } sum.Quantile = qs out.Summary = sum out.Label = s.labelPairs return nil } // NewConstSummary returns a metric representing a Prometheus summary with fixed // values for the count, sum, and quantiles. As those parameters cannot be // changed, the returned value does not implement the Summary interface (but // only the Metric interface). Users of this package will not have much use for // it in regular operations. However, when implementing custom Collectors, it is // useful as a throw-away metric that is generated on the fly to send it to // Prometheus in the Collect method. // // quantiles maps ranks to quantile values. For example, a median latency of // 0.23s and a 99th percentile latency of 0.56s would be expressed as: // map[float64]float64{0.5: 0.23, 0.99: 0.56} // // NewConstSummary returns an error if the length of labelValues is not // consistent with the variable labels in Desc. func NewConstSummary( desc *Desc, count uint64, sum float64, quantiles map[float64]float64, labelValues ...string, ) (Metric, error) { if err := validateLabelValues(labelValues, len(desc.variableLabels)); err != nil { return nil, err } return &constSummary{ desc: desc, count: count, sum: sum, quantiles: quantiles, labelPairs: makeLabelPairs(desc, labelValues), }, nil } // MustNewConstSummary is a version of NewConstSummary that panics where // NewConstMetric would have returned an error. func MustNewConstSummary( desc *Desc, count uint64, sum float64, quantiles map[float64]float64, labelValues ...string, ) Metric { m, err := NewConstSummary(desc, count, sum, quantiles, labelValues...) if err != nil { panic(err) } return m } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/summary_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "math" "math/rand" "sort" "sync" "testing" "testing/quick" "time" dto "github.com/prometheus/client_model/go" ) func TestSummaryWithDefaultObjectives(t *testing.T) { reg := NewRegistry() summaryWithDefaultObjectives := NewSummary(SummaryOpts{ Name: "default_objectives", Help: "Test help.", }) if err := reg.Register(summaryWithDefaultObjectives); err != nil { t.Error(err) } m := &dto.Metric{} if err := summaryWithDefaultObjectives.Write(m); err != nil { t.Error(err) } if len(m.GetSummary().Quantile) != len(DefObjectives) { t.Error("expected default objectives in summary") } } func TestSummaryWithoutObjectives(t *testing.T) { reg := NewRegistry() summaryWithEmptyObjectives := NewSummary(SummaryOpts{ Name: "empty_objectives", Help: "Test help.", Objectives: map[float64]float64{}, }) if err := reg.Register(summaryWithEmptyObjectives); err != nil { t.Error(err) } m := &dto.Metric{} if err := summaryWithEmptyObjectives.Write(m); err != nil { t.Error(err) } if len(m.GetSummary().Quantile) != 0 { t.Error("expected no objectives in summary") } } func benchmarkSummaryObserve(w int, b *testing.B) { b.StopTimer() wg := new(sync.WaitGroup) wg.Add(w) g := new(sync.WaitGroup) g.Add(1) s := NewSummary(SummaryOpts{}) for i := 0; i < w; i++ { go func() { g.Wait() for i := 0; i < b.N; i++ { s.Observe(float64(i)) } wg.Done() }() } b.StartTimer() g.Done() wg.Wait() } func BenchmarkSummaryObserve1(b *testing.B) { benchmarkSummaryObserve(1, b) } func BenchmarkSummaryObserve2(b *testing.B) { benchmarkSummaryObserve(2, b) } func BenchmarkSummaryObserve4(b *testing.B) { benchmarkSummaryObserve(4, b) } func BenchmarkSummaryObserve8(b *testing.B) { benchmarkSummaryObserve(8, b) } func benchmarkSummaryWrite(w int, b *testing.B) { b.StopTimer() wg := new(sync.WaitGroup) wg.Add(w) g := new(sync.WaitGroup) g.Add(1) s := NewSummary(SummaryOpts{}) for i := 0; i < 1000000; i++ { s.Observe(float64(i)) } for j := 0; j < w; j++ { outs := make([]dto.Metric, b.N) go func(o []dto.Metric) { g.Wait() for i := 0; i < b.N; i++ { s.Write(&o[i]) } wg.Done() }(outs) } b.StartTimer() g.Done() wg.Wait() } func BenchmarkSummaryWrite1(b *testing.B) { benchmarkSummaryWrite(1, b) } func BenchmarkSummaryWrite2(b *testing.B) { benchmarkSummaryWrite(2, b) } func BenchmarkSummaryWrite4(b *testing.B) { benchmarkSummaryWrite(4, b) } func BenchmarkSummaryWrite8(b *testing.B) { benchmarkSummaryWrite(8, b) } func TestSummaryConcurrency(t *testing.T) { if testing.Short() { t.Skip("Skipping test in short mode.") } rand.Seed(42) it := func(n uint32) bool { mutations := int(n%1e4 + 1e4) concLevel := int(n%5 + 1) total := mutations * concLevel var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) sum := NewSummary(SummaryOpts{ Name: "test_summary", Help: "helpless", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }) allVars := make([]float64, total) var sampleSum float64 for i := 0; i < concLevel; i++ { vals := make([]float64, mutations) for j := 0; j < mutations; j++ { v := rand.NormFloat64() vals[j] = v allVars[i*mutations+j] = v sampleSum += v } go func(vals []float64) { start.Wait() for _, v := range vals { sum.Observe(v) } end.Done() }(vals) } sort.Float64s(allVars) start.Done() end.Wait() m := &dto.Metric{} sum.Write(m) if got, want := int(*m.Summary.SampleCount), total; got != want { t.Errorf("got sample count %d, want %d", got, want) } if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 { t.Errorf("got sample sum %f, want %f", got, want) } objectives := make([]float64, 0, len(DefObjectives)) for qu := range DefObjectives { objectives = append(objectives, qu) } sort.Float64s(objectives) for i, wantQ := range objectives { ε := DefObjectives[wantQ] gotQ := *m.Summary.Quantile[i].Quantile gotV := *m.Summary.Quantile[i].Value min, max := getBounds(allVars, wantQ, ε) if gotQ != wantQ { t.Errorf("got quantile %f, want %f", gotQ, wantQ) } if gotV < min || gotV > max { t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max) } } return true } if err := quick.Check(it, nil); err != nil { t.Error(err) } } func TestSummaryVecConcurrency(t *testing.T) { if testing.Short() { t.Skip("Skipping test in short mode.") } rand.Seed(42) objectives := make([]float64, 0, len(DefObjectives)) for qu := range DefObjectives { objectives = append(objectives, qu) } sort.Float64s(objectives) it := func(n uint32) bool { mutations := int(n%1e4 + 1e4) concLevel := int(n%7 + 1) vecLength := int(n%3 + 1) var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) sum := NewSummaryVec( SummaryOpts{ Name: "test_summary", Help: "helpless", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, []string{"label"}, ) allVars := make([][]float64, vecLength) sampleSums := make([]float64, vecLength) for i := 0; i < concLevel; i++ { vals := make([]float64, mutations) picks := make([]int, mutations) for j := 0; j < mutations; j++ { v := rand.NormFloat64() vals[j] = v pick := rand.Intn(vecLength) picks[j] = pick allVars[pick] = append(allVars[pick], v) sampleSums[pick] += v } go func(vals []float64) { start.Wait() for i, v := range vals { sum.WithLabelValues(string('A' + picks[i])).Observe(v) } end.Done() }(vals) } for _, vars := range allVars { sort.Float64s(vars) } start.Done() end.Wait() for i := 0; i < vecLength; i++ { m := &dto.Metric{} s := sum.WithLabelValues(string('A' + i)) s.(Summary).Write(m) if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want { t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want) } if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 { t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want) } for j, wantQ := range objectives { ε := DefObjectives[wantQ] gotQ := *m.Summary.Quantile[j].Quantile gotV := *m.Summary.Quantile[j].Value min, max := getBounds(allVars[i], wantQ, ε) if gotQ != wantQ { t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ) } if gotV < min || gotV > max { t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max) } } } return true } if err := quick.Check(it, nil); err != nil { t.Error(err) } } func TestSummaryDecay(t *testing.T) { if testing.Short() { t.Skip("Skipping test in short mode.") // More because it depends on timing than because it is particularly long... } sum := NewSummary(SummaryOpts{ Name: "test_summary", Help: "helpless", MaxAge: 100 * time.Millisecond, Objectives: map[float64]float64{0.1: 0.001}, AgeBuckets: 10, }) m := &dto.Metric{} i := 0 tick := time.NewTicker(time.Millisecond) for range tick.C { i++ sum.Observe(float64(i)) if i%10 == 0 { sum.Write(m) if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 { t.Errorf("%d. got %f, want %f", i, got, want) } m.Reset() } if i >= 1000 { break } } tick.Stop() // Wait for MaxAge without observations and make sure quantiles are NaN. time.Sleep(100 * time.Millisecond) sum.Write(m) if got := *m.Summary.Quantile[0].Value; !math.IsNaN(got) { t.Errorf("got %f, want NaN after expiration", got) } } func getBounds(vars []float64, q, ε float64) (min, max float64) { // TODO(beorn7): This currently tolerates an error of up to 2*ε. The // error must be at most ε, but for some reason, it's sometimes slightly // higher. That's a bug. n := float64(len(vars)) lower := int((q - 2*ε) * n) upper := int(math.Ceil((q + 2*ε) * n)) min = vars[0] if lower > 1 { min = vars[lower-1] } max = vars[len(vars)-1] if upper < len(vars) { max = vars[upper-1] } return } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/timer.go ================================================ // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import "time" // Timer is a helper type to time functions. Use NewTimer to create new // instances. type Timer struct { begin time.Time observer Observer } // NewTimer creates a new Timer. The provided Observer is used to observe a // duration in seconds. Timer is usually used to time a function call in the // following way: // func TimeMe() { // timer := NewTimer(myHistogram) // defer timer.ObserveDuration() // // Do actual work. // } func NewTimer(o Observer) *Timer { return &Timer{ begin: time.Now(), observer: o, } } // ObserveDuration records the duration passed since the Timer was created with // NewTimer. It calls the Observe method of the Observer provided during // construction with the duration in seconds as an argument. ObserveDuration is // usually called with a defer statement. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. func (t *Timer) ObserveDuration() { if t.observer != nil { t.observer.Observe(time.Since(t.begin).Seconds()) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/timer_test.go ================================================ // Copyright 2016 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "testing" dto "github.com/prometheus/client_model/go" ) func TestTimerObserve(t *testing.T) { var ( his = NewHistogram(HistogramOpts{Name: "test_histogram"}) sum = NewSummary(SummaryOpts{Name: "test_summary"}) gauge = NewGauge(GaugeOpts{Name: "test_gauge"}) ) func() { hisTimer := NewTimer(his) sumTimer := NewTimer(sum) gaugeTimer := NewTimer(ObserverFunc(gauge.Set)) defer hisTimer.ObserveDuration() defer sumTimer.ObserveDuration() defer gaugeTimer.ObserveDuration() }() m := &dto.Metric{} his.Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for histogram, got %d", want, got) } m.Reset() sum.Write(m) if want, got := uint64(1), m.GetSummary().GetSampleCount(); want != got { t.Errorf("want %d observations for summary, got %d", want, got) } m.Reset() gauge.Write(m) if got := m.GetGauge().GetValue(); got <= 0 { t.Errorf("want value > 0 for gauge, got %f", got) } } func TestTimerEmpty(t *testing.T) { emptyTimer := NewTimer(nil) emptyTimer.ObserveDuration() // Do nothing, just demonstrate it works without panic. } func TestTimerConditionalTiming(t *testing.T) { var ( his = NewHistogram(HistogramOpts{ Name: "test_histogram", }) timeMe = true m = &dto.Metric{} ) timedFunc := func() { timer := NewTimer(ObserverFunc(func(v float64) { if timeMe { his.Observe(v) } })) defer timer.ObserveDuration() } timedFunc() // This will time. his.Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for histogram, got %d", want, got) } timeMe = false timedFunc() // This will not time again. m.Reset() his.Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for histogram, got %d", want, got) } } func TestTimerByOutcome(t *testing.T) { var ( his = NewHistogramVec( HistogramOpts{Name: "test_histogram"}, []string{"outcome"}, ) outcome = "foo" m = &dto.Metric{} ) timedFunc := func() { timer := NewTimer(ObserverFunc(func(v float64) { his.WithLabelValues(outcome).Observe(v) })) defer timer.ObserveDuration() if outcome == "foo" { outcome = "bar" return } outcome = "foo" } timedFunc() his.WithLabelValues("foo").(Histogram).Write(m) if want, got := uint64(0), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for 'foo' histogram, got %d", want, got) } m.Reset() his.WithLabelValues("bar").(Histogram).Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for 'bar' histogram, got %d", want, got) } timedFunc() m.Reset() his.WithLabelValues("foo").(Histogram).Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for 'foo' histogram, got %d", want, got) } m.Reset() his.WithLabelValues("bar").(Histogram).Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for 'bar' histogram, got %d", want, got) } timedFunc() m.Reset() his.WithLabelValues("foo").(Histogram).Write(m) if want, got := uint64(1), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for 'foo' histogram, got %d", want, got) } m.Reset() his.WithLabelValues("bar").(Histogram).Write(m) if want, got := uint64(2), m.GetHistogram().GetSampleCount(); want != got { t.Errorf("want %d observations for 'bar' histogram, got %d", want, got) } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/untyped.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus // UntypedOpts is an alias for Opts. See there for doc comments. type UntypedOpts Opts // UntypedFunc works like GaugeFunc but the collected metric is of type // "Untyped". UntypedFunc is useful to mirror an external metric of unknown // type. // // To create UntypedFunc instances, use NewUntypedFunc. type UntypedFunc interface { Metric Collector } // NewUntypedFunc creates a new UntypedFunc based on the provided // UntypedOpts. The value reported is determined by calling the given function // from within the Write method. Take into account that metric collection may // happen concurrently. If that results in concurrent calls to Write, like in // the case where an UntypedFunc is directly registered with Prometheus, the // provided function must be concurrency-safe. func NewUntypedFunc(opts UntypedOpts, function func() float64) UntypedFunc { return newValueFunc(NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, nil, opts.ConstLabels, ), UntypedValue, function) } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/value.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "fmt" "math" "sort" "sync/atomic" "time" dto "github.com/prometheus/client_model/go" "github.com/golang/protobuf/proto" ) // ValueType is an enumeration of metric types that represent a simple value. type ValueType int // Possible values for the ValueType enum. const ( _ ValueType = iota CounterValue GaugeValue UntypedValue ) // value is a generic metric for simple values. It implements Metric, Collector, // Counter, Gauge, and Untyped. Its effective type is determined by // ValueType. This is a low-level building block used by the library to back the // implementations of Counter, Gauge, and Untyped. type value struct { // valBits contains the bits of the represented float64 value. It has // to go first in the struct to guarantee alignment for atomic // operations. http://golang.org/pkg/sync/atomic/#pkg-note-BUG valBits uint64 selfCollector desc *Desc valType ValueType labelPairs []*dto.LabelPair } // newValue returns a newly allocated value with the given Desc, ValueType, // sample value and label values. It panics if the number of label // values is different from the number of variable labels in Desc. func newValue(desc *Desc, valueType ValueType, val float64, labelValues ...string) *value { if len(labelValues) != len(desc.variableLabels) { panic(errInconsistentCardinality) } result := &value{ desc: desc, valType: valueType, valBits: math.Float64bits(val), labelPairs: makeLabelPairs(desc, labelValues), } result.init(result) return result } func (v *value) Desc() *Desc { return v.desc } func (v *value) Set(val float64) { atomic.StoreUint64(&v.valBits, math.Float64bits(val)) } func (v *value) SetToCurrentTime() { v.Set(float64(time.Now().UnixNano()) / 1e9) } func (v *value) Inc() { v.Add(1) } func (v *value) Dec() { v.Add(-1) } func (v *value) Add(val float64) { for { oldBits := atomic.LoadUint64(&v.valBits) newBits := math.Float64bits(math.Float64frombits(oldBits) + val) if atomic.CompareAndSwapUint64(&v.valBits, oldBits, newBits) { return } } } func (v *value) Sub(val float64) { v.Add(val * -1) } func (v *value) Write(out *dto.Metric) error { val := math.Float64frombits(atomic.LoadUint64(&v.valBits)) return populateMetric(v.valType, val, v.labelPairs, out) } // valueFunc is a generic metric for simple values retrieved on collect time // from a function. It implements Metric and Collector. Its effective type is // determined by ValueType. This is a low-level building block used by the // library to back the implementations of CounterFunc, GaugeFunc, and // UntypedFunc. type valueFunc struct { selfCollector desc *Desc valType ValueType function func() float64 labelPairs []*dto.LabelPair } // newValueFunc returns a newly allocated valueFunc with the given Desc and // ValueType. The value reported is determined by calling the given function // from within the Write method. Take into account that metric collection may // happen concurrently. If that results in concurrent calls to Write, like in // the case where a valueFunc is directly registered with Prometheus, the // provided function must be concurrency-safe. func newValueFunc(desc *Desc, valueType ValueType, function func() float64) *valueFunc { result := &valueFunc{ desc: desc, valType: valueType, function: function, labelPairs: makeLabelPairs(desc, nil), } result.init(result) return result } func (v *valueFunc) Desc() *Desc { return v.desc } func (v *valueFunc) Write(out *dto.Metric) error { return populateMetric(v.valType, v.function(), v.labelPairs, out) } // NewConstMetric returns a metric with one fixed value that cannot be // changed. Users of this package will not have much use for it in regular // operations. However, when implementing custom Collectors, it is useful as a // throw-away metric that is generated on the fly to send it to Prometheus in // the Collect method. NewConstMetric returns an error if the length of // labelValues is not consistent with the variable labels in Desc. func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) (Metric, error) { if err := validateLabelValues(labelValues, len(desc.variableLabels)); err != nil { return nil, err } return &constMetric{ desc: desc, valType: valueType, val: value, labelPairs: makeLabelPairs(desc, labelValues), }, nil } // MustNewConstMetric is a version of NewConstMetric that panics where // NewConstMetric would have returned an error. func MustNewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) Metric { m, err := NewConstMetric(desc, valueType, value, labelValues...) if err != nil { panic(err) } return m } type constMetric struct { desc *Desc valType ValueType val float64 labelPairs []*dto.LabelPair } func (m *constMetric) Desc() *Desc { return m.desc } func (m *constMetric) Write(out *dto.Metric) error { return populateMetric(m.valType, m.val, m.labelPairs, out) } func populateMetric( t ValueType, v float64, labelPairs []*dto.LabelPair, m *dto.Metric, ) error { m.Label = labelPairs switch t { case CounterValue: m.Counter = &dto.Counter{Value: proto.Float64(v)} case GaugeValue: m.Gauge = &dto.Gauge{Value: proto.Float64(v)} case UntypedValue: m.Untyped = &dto.Untyped{Value: proto.Float64(v)} default: return fmt.Errorf("encountered unknown type %v", t) } return nil } func makeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair { totalLen := len(desc.variableLabels) + len(desc.constLabelPairs) if totalLen == 0 { // Super fast path. return nil } if len(desc.variableLabels) == 0 { // Moderately fast path. return desc.constLabelPairs } labelPairs := make([]*dto.LabelPair, 0, totalLen) for i, n := range desc.variableLabels { labelPairs = append(labelPairs, &dto.LabelPair{ Name: proto.String(n), Value: proto.String(labelValues[i]), }) } for _, lp := range desc.constLabelPairs { labelPairs = append(labelPairs, lp) } sort.Sort(LabelPairSorter(labelPairs)) return labelPairs } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/value_test.go ================================================ package prometheus import ( "fmt" "testing" ) func TestNewConstMetricInvalidLabelValues(t *testing.T) { testCases := []struct { desc string labels Labels }{ { desc: "non utf8 label value", labels: Labels{"a": "\xFF"}, }, { desc: "not enough label values", labels: Labels{}, }, { desc: "too many label values", labels: Labels{"a": "1", "b": "2"}, }, } for _, test := range testCases { metricDesc := NewDesc( "sample_value", "sample value", []string{"a"}, Labels{}, ) expectPanic(t, func() { MustNewConstMetric(metricDesc, CounterValue, 0.3, "\xFF") }, fmt.Sprintf("WithLabelValues: expected panic because: %s", test.desc)) if _, err := NewConstMetric(metricDesc, CounterValue, 0.3, "\xFF"); err == nil { t.Errorf("NewConstMetric: expected error because: %s", test.desc) } } } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/vec.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "fmt" "sync" "github.com/prometheus/common/model" ) // metricVec is a Collector to bundle metrics of the same name that differ in // their label values. metricVec is not used directly (and therefore // unexported). It is used as a building block for implementations of vectors of // a given metric type, like GaugeVec, CounterVec, SummaryVec, HistogramVec, and // UntypedVec. type metricVec struct { mtx sync.RWMutex // Protects the children. children map[uint64][]metricWithLabelValues desc *Desc newMetric func(labelValues ...string) Metric hashAdd func(h uint64, s string) uint64 // replace hash function for testing collision handling hashAddByte func(h uint64, b byte) uint64 } // newMetricVec returns an initialized metricVec. func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *metricVec { return &metricVec{ children: map[uint64][]metricWithLabelValues{}, desc: desc, newMetric: newMetric, hashAdd: hashAdd, hashAddByte: hashAddByte, } } // metricWithLabelValues provides the metric and its label values for // disambiguation on hash collision. type metricWithLabelValues struct { values []string metric Metric } // Describe implements Collector. The length of the returned slice // is always one. func (m *metricVec) Describe(ch chan<- *Desc) { ch <- m.desc } // Collect implements Collector. func (m *metricVec) Collect(ch chan<- Metric) { m.mtx.RLock() defer m.mtx.RUnlock() for _, metrics := range m.children { for _, metric := range metrics { ch <- metric.metric } } } func (m *metricVec) getMetricWithLabelValues(lvs ...string) (Metric, error) { h, err := m.hashLabelValues(lvs) if err != nil { return nil, err } return m.getOrCreateMetricWithLabelValues(h, lvs), nil } func (m *metricVec) getMetricWith(labels Labels) (Metric, error) { h, err := m.hashLabels(labels) if err != nil { return nil, err } return m.getOrCreateMetricWithLabels(h, labels), nil } func (m *metricVec) withLabelValues(lvs ...string) Metric { metric, err := m.getMetricWithLabelValues(lvs...) if err != nil { panic(err) } return metric } func (m *metricVec) with(labels Labels) Metric { metric, err := m.getMetricWith(labels) if err != nil { panic(err) } return metric } // DeleteLabelValues removes the metric where the variable labels are the same // as those passed in as labels (same order as the VariableLabels in Desc). It // returns true if a metric was deleted. // // It is not an error if the number of label values is not the same as the // number of VariableLabels in Desc. However, such inconsistent label count can // never match an actual metric, so the method will always return false in that // case. // // Note that for more than one label value, this method is prone to mistakes // caused by an incorrect order of arguments. Consider Delete(Labels) as an // alternative to avoid that type of mistake. For higher label numbers, the // latter has a much more readable (albeit more verbose) syntax, but it comes // with a performance overhead (for creating and processing the Labels map). // See also the CounterVec example. func (m *metricVec) DeleteLabelValues(lvs ...string) bool { m.mtx.Lock() defer m.mtx.Unlock() h, err := m.hashLabelValues(lvs) if err != nil { return false } return m.deleteByHashWithLabelValues(h, lvs) } // Delete deletes the metric where the variable labels are the same as those // passed in as labels. It returns true if a metric was deleted. // // It is not an error if the number and names of the Labels are inconsistent // with those of the VariableLabels in Desc. However, such inconsistent Labels // can never match an actual metric, so the method will always return false in // that case. // // This method is used for the same purpose as DeleteLabelValues(...string). See // there for pros and cons of the two methods. func (m *metricVec) Delete(labels Labels) bool { m.mtx.Lock() defer m.mtx.Unlock() h, err := m.hashLabels(labels) if err != nil { return false } return m.deleteByHashWithLabels(h, labels) } // deleteByHashWithLabelValues removes the metric from the hash bucket h. If // there are multiple matches in the bucket, use lvs to select a metric and // remove only that metric. func (m *metricVec) deleteByHashWithLabelValues(h uint64, lvs []string) bool { metrics, ok := m.children[h] if !ok { return false } i := m.findMetricWithLabelValues(metrics, lvs) if i >= len(metrics) { return false } if len(metrics) > 1 { m.children[h] = append(metrics[:i], metrics[i+1:]...) } else { delete(m.children, h) } return true } // deleteByHashWithLabels removes the metric from the hash bucket h. If there // are multiple matches in the bucket, use lvs to select a metric and remove // only that metric. func (m *metricVec) deleteByHashWithLabels(h uint64, labels Labels) bool { metrics, ok := m.children[h] if !ok { return false } i := m.findMetricWithLabels(metrics, labels) if i >= len(metrics) { return false } if len(metrics) > 1 { m.children[h] = append(metrics[:i], metrics[i+1:]...) } else { delete(m.children, h) } return true } // Reset deletes all metrics in this vector. func (m *metricVec) Reset() { m.mtx.Lock() defer m.mtx.Unlock() for h := range m.children { delete(m.children, h) } } func (m *metricVec) hashLabelValues(vals []string) (uint64, error) { if err := validateLabelValues(vals, len(m.desc.variableLabels)); err != nil { return 0, err } h := hashNew() for _, val := range vals { h = m.hashAdd(h, val) h = m.hashAddByte(h, model.SeparatorByte) } return h, nil } func (m *metricVec) hashLabels(labels Labels) (uint64, error) { if err := validateValuesInLabels(labels, len(m.desc.variableLabels)); err != nil { return 0, err } h := hashNew() for _, label := range m.desc.variableLabels { val, ok := labels[label] if !ok { return 0, fmt.Errorf("label name %q missing in label map", label) } h = m.hashAdd(h, val) h = m.hashAddByte(h, model.SeparatorByte) } return h, nil } // getOrCreateMetricWithLabelValues retrieves the metric by hash and label value // or creates it and returns the new one. // // This function holds the mutex. func (m *metricVec) getOrCreateMetricWithLabelValues(hash uint64, lvs []string) Metric { m.mtx.RLock() metric, ok := m.getMetricWithHashAndLabelValues(hash, lvs) m.mtx.RUnlock() if ok { return metric } m.mtx.Lock() defer m.mtx.Unlock() metric, ok = m.getMetricWithHashAndLabelValues(hash, lvs) if !ok { // Copy to avoid allocation in case wo don't go down this code path. copiedLVs := make([]string, len(lvs)) copy(copiedLVs, lvs) metric = m.newMetric(copiedLVs...) m.children[hash] = append(m.children[hash], metricWithLabelValues{values: copiedLVs, metric: metric}) } return metric } // getOrCreateMetricWithLabelValues retrieves the metric by hash and label value // or creates it and returns the new one. // // This function holds the mutex. func (m *metricVec) getOrCreateMetricWithLabels(hash uint64, labels Labels) Metric { m.mtx.RLock() metric, ok := m.getMetricWithHashAndLabels(hash, labels) m.mtx.RUnlock() if ok { return metric } m.mtx.Lock() defer m.mtx.Unlock() metric, ok = m.getMetricWithHashAndLabels(hash, labels) if !ok { lvs := m.extractLabelValues(labels) metric = m.newMetric(lvs...) m.children[hash] = append(m.children[hash], metricWithLabelValues{values: lvs, metric: metric}) } return metric } // getMetricWithHashAndLabelValues gets a metric while handling possible // collisions in the hash space. Must be called while holding the read mutex. func (m *metricVec) getMetricWithHashAndLabelValues(h uint64, lvs []string) (Metric, bool) { metrics, ok := m.children[h] if ok { if i := m.findMetricWithLabelValues(metrics, lvs); i < len(metrics) { return metrics[i].metric, true } } return nil, false } // getMetricWithHashAndLabels gets a metric while handling possible collisions in // the hash space. Must be called while holding read mutex. func (m *metricVec) getMetricWithHashAndLabels(h uint64, labels Labels) (Metric, bool) { metrics, ok := m.children[h] if ok { if i := m.findMetricWithLabels(metrics, labels); i < len(metrics) { return metrics[i].metric, true } } return nil, false } // findMetricWithLabelValues returns the index of the matching metric or // len(metrics) if not found. func (m *metricVec) findMetricWithLabelValues(metrics []metricWithLabelValues, lvs []string) int { for i, metric := range metrics { if m.matchLabelValues(metric.values, lvs) { return i } } return len(metrics) } // findMetricWithLabels returns the index of the matching metric or len(metrics) // if not found. func (m *metricVec) findMetricWithLabels(metrics []metricWithLabelValues, labels Labels) int { for i, metric := range metrics { if m.matchLabels(metric.values, labels) { return i } } return len(metrics) } func (m *metricVec) matchLabelValues(values []string, lvs []string) bool { if len(values) != len(lvs) { return false } for i, v := range values { if v != lvs[i] { return false } } return true } func (m *metricVec) matchLabels(values []string, labels Labels) bool { if len(labels) != len(values) { return false } for i, k := range m.desc.variableLabels { if values[i] != labels[k] { return false } } return true } func (m *metricVec) extractLabelValues(labels Labels) []string { labelValues := make([]string, len(labels)) for i, k := range m.desc.variableLabels { labelValues[i] = labels[k] } return labelValues } ================================================ FILE: vendor/github.com/prometheus/client_golang/prometheus/vec_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package prometheus import ( "fmt" "testing" dto "github.com/prometheus/client_model/go" ) func TestDelete(t *testing.T) { vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, []string{"l1", "l2"}, ) testDelete(t, vec) } func TestDeleteWithCollisions(t *testing.T) { vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, []string{"l1", "l2"}, ) vec.hashAdd = func(h uint64, s string) uint64 { return 1 } vec.hashAddByte = func(h uint64, b byte) uint64 { return 1 } testDelete(t, vec) } func testDelete(t *testing.T, vec *GaugeVec) { if got, want := vec.Delete(Labels{"l1": "v1", "l2": "v2"}), false; got != want { t.Errorf("got %v, want %v", got, want) } vec.With(Labels{"l1": "v1", "l2": "v2"}).(Gauge).Set(42) if got, want := vec.Delete(Labels{"l1": "v1", "l2": "v2"}), true; got != want { t.Errorf("got %v, want %v", got, want) } if got, want := vec.Delete(Labels{"l1": "v1", "l2": "v2"}), false; got != want { t.Errorf("got %v, want %v", got, want) } vec.With(Labels{"l1": "v1", "l2": "v2"}).(Gauge).Set(42) if got, want := vec.Delete(Labels{"l2": "v2", "l1": "v1"}), true; got != want { t.Errorf("got %v, want %v", got, want) } if got, want := vec.Delete(Labels{"l2": "v2", "l1": "v1"}), false; got != want { t.Errorf("got %v, want %v", got, want) } vec.With(Labels{"l1": "v1", "l2": "v2"}).(Gauge).Set(42) if got, want := vec.Delete(Labels{"l2": "v1", "l1": "v2"}), false; got != want { t.Errorf("got %v, want %v", got, want) } if got, want := vec.Delete(Labels{"l1": "v1"}), false; got != want { t.Errorf("got %v, want %v", got, want) } } func TestDeleteLabelValues(t *testing.T) { vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, []string{"l1", "l2"}, ) testDeleteLabelValues(t, vec) } func TestDeleteLabelValuesWithCollisions(t *testing.T) { vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, []string{"l1", "l2"}, ) vec.hashAdd = func(h uint64, s string) uint64 { return 1 } vec.hashAddByte = func(h uint64, b byte) uint64 { return 1 } testDeleteLabelValues(t, vec) } func testDeleteLabelValues(t *testing.T, vec *GaugeVec) { if got, want := vec.DeleteLabelValues("v1", "v2"), false; got != want { t.Errorf("got %v, want %v", got, want) } vec.With(Labels{"l1": "v1", "l2": "v2"}).(Gauge).Set(42) vec.With(Labels{"l1": "v1", "l2": "v3"}).(Gauge).Set(42) // Add junk data for collision. if got, want := vec.DeleteLabelValues("v1", "v2"), true; got != want { t.Errorf("got %v, want %v", got, want) } if got, want := vec.DeleteLabelValues("v1", "v2"), false; got != want { t.Errorf("got %v, want %v", got, want) } if got, want := vec.DeleteLabelValues("v1", "v3"), true; got != want { t.Errorf("got %v, want %v", got, want) } vec.With(Labels{"l1": "v1", "l2": "v2"}).(Gauge).Set(42) // Delete out of order. if got, want := vec.DeleteLabelValues("v2", "v1"), false; got != want { t.Errorf("got %v, want %v", got, want) } if got, want := vec.DeleteLabelValues("v1"), false; got != want { t.Errorf("got %v, want %v", got, want) } } func TestMetricVec(t *testing.T) { vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, []string{"l1", "l2"}, ) testMetricVec(t, vec) } func TestMetricVecWithCollisions(t *testing.T) { vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, []string{"l1", "l2"}, ) vec.hashAdd = func(h uint64, s string) uint64 { return 1 } vec.hashAddByte = func(h uint64, b byte) uint64 { return 1 } testMetricVec(t, vec) } func testMetricVec(t *testing.T, vec *GaugeVec) { vec.Reset() // Actually test Reset now! var pair [2]string // Keep track of metrics. expected := map[[2]string]int{} for i := 0; i < 1000; i++ { pair[0], pair[1] = fmt.Sprint(i%4), fmt.Sprint(i%5) // Varying combinations multiples. expected[pair]++ vec.WithLabelValues(pair[0], pair[1]).Inc() expected[[2]string{"v1", "v2"}]++ vec.WithLabelValues("v1", "v2").(Gauge).Inc() } var total int for _, metrics := range vec.children { for _, metric := range metrics { total++ copy(pair[:], metric.values) var metricOut dto.Metric if err := metric.metric.Write(&metricOut); err != nil { t.Fatal(err) } actual := *metricOut.Gauge.Value var actualPair [2]string for i, label := range metricOut.Label { actualPair[i] = *label.Value } // Test output pair against metric.values to ensure we've selected // the right one. We check this to ensure the below check means // anything at all. if actualPair != pair { t.Fatalf("unexpected pair association in metric map: %v != %v", actualPair, pair) } if actual != float64(expected[pair]) { t.Fatalf("incorrect counter value for %v: %v != %v", pair, actual, expected[pair]) } } } if total != len(expected) { t.Fatalf("unexpected number of metrics: %v != %v", total, len(expected)) } vec.Reset() if len(vec.children) > 0 { t.Fatalf("reset failed") } } func TestCounterVecEndToEndWithCollision(t *testing.T) { vec := NewCounterVec( CounterOpts{ Name: "test", Help: "helpless", }, []string{"labelname"}, ) vec.WithLabelValues("77kepQFQ8Kl").Inc() vec.WithLabelValues("!0IC=VloaY").Add(2) m := &dto.Metric{} if err := vec.WithLabelValues("77kepQFQ8Kl").Write(m); err != nil { t.Fatal(err) } if got, want := m.GetLabel()[0].GetValue(), "77kepQFQ8Kl"; got != want { t.Errorf("got label value %q, want %q", got, want) } if got, want := m.GetCounter().GetValue(), 1.; got != want { t.Errorf("got value %f, want %f", got, want) } m.Reset() if err := vec.WithLabelValues("!0IC=VloaY").Write(m); err != nil { t.Fatal(err) } if got, want := m.GetLabel()[0].GetValue(), "!0IC=VloaY"; got != want { t.Errorf("got label value %q, want %q", got, want) } if got, want := m.GetCounter().GetValue(), 2.; got != want { t.Errorf("got value %f, want %f", got, want) } } func BenchmarkMetricVecWithLabelValuesBasic(b *testing.B) { benchmarkMetricVecWithLabelValues(b, map[string][]string{ "l1": {"onevalue"}, "l2": {"twovalue"}, }) } func BenchmarkMetricVecWithLabelValues2Keys10ValueCardinality(b *testing.B) { benchmarkMetricVecWithLabelValuesCardinality(b, 2, 10) } func BenchmarkMetricVecWithLabelValues4Keys10ValueCardinality(b *testing.B) { benchmarkMetricVecWithLabelValuesCardinality(b, 4, 10) } func BenchmarkMetricVecWithLabelValues2Keys100ValueCardinality(b *testing.B) { benchmarkMetricVecWithLabelValuesCardinality(b, 2, 100) } func BenchmarkMetricVecWithLabelValues10Keys100ValueCardinality(b *testing.B) { benchmarkMetricVecWithLabelValuesCardinality(b, 10, 100) } func BenchmarkMetricVecWithLabelValues10Keys1000ValueCardinality(b *testing.B) { benchmarkMetricVecWithLabelValuesCardinality(b, 10, 1000) } func benchmarkMetricVecWithLabelValuesCardinality(b *testing.B, nkeys, nvalues int) { labels := map[string][]string{} for i := 0; i < nkeys; i++ { var ( k = fmt.Sprintf("key-%v", i) vs = make([]string, 0, nvalues) ) for j := 0; j < nvalues; j++ { vs = append(vs, fmt.Sprintf("value-%v", j)) } labels[k] = vs } benchmarkMetricVecWithLabelValues(b, labels) } func benchmarkMetricVecWithLabelValues(b *testing.B, labels map[string][]string) { var keys []string for k := range labels { // Map order dependent, who cares though. keys = append(keys, k) } values := make([]string, len(labels)) // Value cache for permutations. vec := NewGaugeVec( GaugeOpts{ Name: "test", Help: "helpless", }, keys, ) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { // Varies input across provide map entries based on key size. for j, k := range keys { candidates := labels[k] values[j] = candidates[i%len(candidates)] } vec.WithLabelValues(values...) } } ================================================ FILE: vendor/github.com/prometheus/client_model/.gitignore ================================================ target/ ================================================ FILE: vendor/github.com/prometheus/client_model/CONTRIBUTING.md ================================================ # Contributing Prometheus uses GitHub to manage reviews of pull requests. * If you have a trivial fix or improvement, go ahead and create a pull request, addressing (with `@...`) the maintainer of this repository (see [MAINTAINERS.md](MAINTAINERS.md)) in the description of the pull request. * If you plan to do something more involved, first discuss your ideas on our [mailing list](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). This will avoid unnecessary work and surely give you and us a good deal of inspiration. * Relevant coding style guidelines are the [Go Code Review Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) and the _Formatting and style_ section of Peter Bourgon's [Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/#formatting-and-style). ================================================ FILE: vendor/github.com/prometheus/client_model/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: vendor/github.com/prometheus/client_model/MAINTAINERS.md ================================================ * Björn Rabenstein ================================================ FILE: vendor/github.com/prometheus/client_model/Makefile ================================================ # Copyright 2013 Prometheus Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. KEY_ID ?= _DEFINE_ME_ all: cpp go java python ruby SUFFIXES: cpp: cpp/metrics.pb.cc cpp/metrics.pb.h cpp/metrics.pb.cc: metrics.proto protoc $< --cpp_out=cpp/ cpp/metrics.pb.h: metrics.proto protoc $< --cpp_out=cpp/ go: go/metrics.pb.go go/metrics.pb.go: metrics.proto protoc $< --go_out=go/ java: src/main/java/io/prometheus/client/Metrics.java pom.xml mvn clean compile package src/main/java/io/prometheus/client/Metrics.java: metrics.proto protoc $< --java_out=src/main/java python: python/prometheus/client/model/metrics_pb2.py python/prometheus/client/model/metrics_pb2.py: metrics.proto protoc $< --python_out=python/prometheus/client/model ruby: $(MAKE) -C ruby build clean: -rm -rf cpp/* -rm -rf go/* -rm -rf java/* -rm -rf python/* -$(MAKE) -C ruby clean -mvn clean maven-deploy-snapshot: java mvn clean deploy -Dgpg.keyname=$(KEY_ID) -DperformRelease=true maven-deploy-release: java mvn clean release:clean release:prepare release:perform -Dgpg.keyname=$(KEY_ID) -DperformRelease=true .PHONY: all clean cpp go java maven-deploy-snapshot maven-deploy-release python ruby ================================================ FILE: vendor/github.com/prometheus/client_model/NOTICE ================================================ Data model artifacts for Prometheus. Copyright 2012-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). ================================================ FILE: vendor/github.com/prometheus/client_model/README.md ================================================ # Background Under most circumstances, manually downloading this repository should never be required. # Prerequisites # Base * [Google Protocol Buffers](https://developers.google.com/protocol-buffers) ## Java * [Apache Maven](http://maven.apache.org) * [Prometheus Maven Repository](https://github.com/prometheus/io.prometheus-maven-repository) checked out into ../io.prometheus-maven-repository ## Go * [Go](http://golang.org) * [goprotobuf](https://code.google.com/p/goprotobuf) ## Ruby * [Ruby](https://www.ruby-lang.org) * [bundler](https://rubygems.org/gems/bundler) # Building $ make # Getting Started * The Go source code is periodically indexed: [Go Protocol Buffer Model](http://godoc.org/github.com/prometheus/client_model/go). * All of the core developers are accessible via the [Prometheus Developers Mailinglist](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). ================================================ FILE: vendor/github.com/prometheus/client_model/go/metrics.pb.go ================================================ // Code generated by protoc-gen-go. // source: metrics.proto // DO NOT EDIT! /* Package io_prometheus_client is a generated protocol buffer package. It is generated from these files: metrics.proto It has these top-level messages: LabelPair Gauge Counter Quantile Summary Untyped Histogram Bucket Metric MetricFamily */ package io_prometheus_client import proto "github.com/golang/protobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = math.Inf type MetricType int32 const ( MetricType_COUNTER MetricType = 0 MetricType_GAUGE MetricType = 1 MetricType_SUMMARY MetricType = 2 MetricType_UNTYPED MetricType = 3 MetricType_HISTOGRAM MetricType = 4 ) var MetricType_name = map[int32]string{ 0: "COUNTER", 1: "GAUGE", 2: "SUMMARY", 3: "UNTYPED", 4: "HISTOGRAM", } var MetricType_value = map[string]int32{ "COUNTER": 0, "GAUGE": 1, "SUMMARY": 2, "UNTYPED": 3, "HISTOGRAM": 4, } func (x MetricType) Enum() *MetricType { p := new(MetricType) *p = x return p } func (x MetricType) String() string { return proto.EnumName(MetricType_name, int32(x)) } func (x *MetricType) UnmarshalJSON(data []byte) error { value, err := proto.UnmarshalJSONEnum(MetricType_value, data, "MetricType") if err != nil { return err } *x = MetricType(value) return nil } type LabelPair struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` Value *string `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *LabelPair) Reset() { *m = LabelPair{} } func (m *LabelPair) String() string { return proto.CompactTextString(m) } func (*LabelPair) ProtoMessage() {} func (m *LabelPair) GetName() string { if m != nil && m.Name != nil { return *m.Name } return "" } func (m *LabelPair) GetValue() string { if m != nil && m.Value != nil { return *m.Value } return "" } type Gauge struct { Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Gauge) Reset() { *m = Gauge{} } func (m *Gauge) String() string { return proto.CompactTextString(m) } func (*Gauge) ProtoMessage() {} func (m *Gauge) GetValue() float64 { if m != nil && m.Value != nil { return *m.Value } return 0 } type Counter struct { Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Counter) Reset() { *m = Counter{} } func (m *Counter) String() string { return proto.CompactTextString(m) } func (*Counter) ProtoMessage() {} func (m *Counter) GetValue() float64 { if m != nil && m.Value != nil { return *m.Value } return 0 } type Quantile struct { Quantile *float64 `protobuf:"fixed64,1,opt,name=quantile" json:"quantile,omitempty"` Value *float64 `protobuf:"fixed64,2,opt,name=value" json:"value,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Quantile) Reset() { *m = Quantile{} } func (m *Quantile) String() string { return proto.CompactTextString(m) } func (*Quantile) ProtoMessage() {} func (m *Quantile) GetQuantile() float64 { if m != nil && m.Quantile != nil { return *m.Quantile } return 0 } func (m *Quantile) GetValue() float64 { if m != nil && m.Value != nil { return *m.Value } return 0 } type Summary struct { SampleCount *uint64 `protobuf:"varint,1,opt,name=sample_count" json:"sample_count,omitempty"` SampleSum *float64 `protobuf:"fixed64,2,opt,name=sample_sum" json:"sample_sum,omitempty"` Quantile []*Quantile `protobuf:"bytes,3,rep,name=quantile" json:"quantile,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Summary) Reset() { *m = Summary{} } func (m *Summary) String() string { return proto.CompactTextString(m) } func (*Summary) ProtoMessage() {} func (m *Summary) GetSampleCount() uint64 { if m != nil && m.SampleCount != nil { return *m.SampleCount } return 0 } func (m *Summary) GetSampleSum() float64 { if m != nil && m.SampleSum != nil { return *m.SampleSum } return 0 } func (m *Summary) GetQuantile() []*Quantile { if m != nil { return m.Quantile } return nil } type Untyped struct { Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Untyped) Reset() { *m = Untyped{} } func (m *Untyped) String() string { return proto.CompactTextString(m) } func (*Untyped) ProtoMessage() {} func (m *Untyped) GetValue() float64 { if m != nil && m.Value != nil { return *m.Value } return 0 } type Histogram struct { SampleCount *uint64 `protobuf:"varint,1,opt,name=sample_count" json:"sample_count,omitempty"` SampleSum *float64 `protobuf:"fixed64,2,opt,name=sample_sum" json:"sample_sum,omitempty"` Bucket []*Bucket `protobuf:"bytes,3,rep,name=bucket" json:"bucket,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Histogram) Reset() { *m = Histogram{} } func (m *Histogram) String() string { return proto.CompactTextString(m) } func (*Histogram) ProtoMessage() {} func (m *Histogram) GetSampleCount() uint64 { if m != nil && m.SampleCount != nil { return *m.SampleCount } return 0 } func (m *Histogram) GetSampleSum() float64 { if m != nil && m.SampleSum != nil { return *m.SampleSum } return 0 } func (m *Histogram) GetBucket() []*Bucket { if m != nil { return m.Bucket } return nil } type Bucket struct { CumulativeCount *uint64 `protobuf:"varint,1,opt,name=cumulative_count" json:"cumulative_count,omitempty"` UpperBound *float64 `protobuf:"fixed64,2,opt,name=upper_bound" json:"upper_bound,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Bucket) Reset() { *m = Bucket{} } func (m *Bucket) String() string { return proto.CompactTextString(m) } func (*Bucket) ProtoMessage() {} func (m *Bucket) GetCumulativeCount() uint64 { if m != nil && m.CumulativeCount != nil { return *m.CumulativeCount } return 0 } func (m *Bucket) GetUpperBound() float64 { if m != nil && m.UpperBound != nil { return *m.UpperBound } return 0 } type Metric struct { Label []*LabelPair `protobuf:"bytes,1,rep,name=label" json:"label,omitempty"` Gauge *Gauge `protobuf:"bytes,2,opt,name=gauge" json:"gauge,omitempty"` Counter *Counter `protobuf:"bytes,3,opt,name=counter" json:"counter,omitempty"` Summary *Summary `protobuf:"bytes,4,opt,name=summary" json:"summary,omitempty"` Untyped *Untyped `protobuf:"bytes,5,opt,name=untyped" json:"untyped,omitempty"` Histogram *Histogram `protobuf:"bytes,7,opt,name=histogram" json:"histogram,omitempty"` TimestampMs *int64 `protobuf:"varint,6,opt,name=timestamp_ms" json:"timestamp_ms,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *Metric) Reset() { *m = Metric{} } func (m *Metric) String() string { return proto.CompactTextString(m) } func (*Metric) ProtoMessage() {} func (m *Metric) GetLabel() []*LabelPair { if m != nil { return m.Label } return nil } func (m *Metric) GetGauge() *Gauge { if m != nil { return m.Gauge } return nil } func (m *Metric) GetCounter() *Counter { if m != nil { return m.Counter } return nil } func (m *Metric) GetSummary() *Summary { if m != nil { return m.Summary } return nil } func (m *Metric) GetUntyped() *Untyped { if m != nil { return m.Untyped } return nil } func (m *Metric) GetHistogram() *Histogram { if m != nil { return m.Histogram } return nil } func (m *Metric) GetTimestampMs() int64 { if m != nil && m.TimestampMs != nil { return *m.TimestampMs } return 0 } type MetricFamily struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` Help *string `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"` Type *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"` Metric []*Metric `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *MetricFamily) Reset() { *m = MetricFamily{} } func (m *MetricFamily) String() string { return proto.CompactTextString(m) } func (*MetricFamily) ProtoMessage() {} func (m *MetricFamily) GetName() string { if m != nil && m.Name != nil { return *m.Name } return "" } func (m *MetricFamily) GetHelp() string { if m != nil && m.Help != nil { return *m.Help } return "" } func (m *MetricFamily) GetType() MetricType { if m != nil && m.Type != nil { return *m.Type } return MetricType_COUNTER } func (m *MetricFamily) GetMetric() []*Metric { if m != nil { return m.Metric } return nil } func init() { proto.RegisterEnum("io.prometheus.client.MetricType", MetricType_name, MetricType_value) } ================================================ FILE: vendor/github.com/prometheus/client_model/metrics.proto ================================================ // Copyright 2013 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto2"; package io.prometheus.client; option java_package = "io.prometheus.client"; message LabelPair { optional string name = 1; optional string value = 2; } enum MetricType { COUNTER = 0; GAUGE = 1; SUMMARY = 2; UNTYPED = 3; HISTOGRAM = 4; } message Gauge { optional double value = 1; } message Counter { optional double value = 1; } message Quantile { optional double quantile = 1; optional double value = 2; } message Summary { optional uint64 sample_count = 1; optional double sample_sum = 2; repeated Quantile quantile = 3; } message Untyped { optional double value = 1; } message Histogram { optional uint64 sample_count = 1; optional double sample_sum = 2; repeated Bucket bucket = 3; // Ordered in increasing order of upper_bound, +Inf bucket is optional. } message Bucket { optional uint64 cumulative_count = 1; // Cumulative in increasing order. optional double upper_bound = 2; // Inclusive. } message Metric { repeated LabelPair label = 1; optional Gauge gauge = 2; optional Counter counter = 3; optional Summary summary = 4; optional Untyped untyped = 5; optional Histogram histogram = 7; optional int64 timestamp_ms = 6; } message MetricFamily { optional string name = 1; optional string help = 2; optional MetricType type = 3; repeated Metric metric = 4; } ================================================ FILE: vendor/github.com/prometheus/client_model/pom.xml ================================================ 4.0.0 io.prometheus.client model 0.0.3-SNAPSHOT org.sonatype.oss oss-parent 7 Prometheus Client Data Model http://github.com/prometheus/client_model Prometheus Client Data Model: Generated Protocol Buffer Assets The Apache Software License, Version 2.0 http://www.apache.org/licenses/LICENSE-2.0.txt repo scm:git:git@github.com:prometheus/client_model.git scm:git:git@github.com:prometheus/client_model.git git@github.com:prometheus/client_model.git mtp Matt T. Proud matt.proud@gmail.com com.google.protobuf protobuf-java 2.5.0 org.apache.maven.plugins maven-javadoc-plugin 2.8 UTF-8 UTF-8 true generate-javadoc-site-report site javadoc attach-javadocs jar maven-compiler-plugin 1.6 1.6 3.1 org.apache.maven.plugins maven-source-plugin 2.2.1 attach-sources jar release-sign-artifacts performRelease true org.apache.maven.plugins maven-gpg-plugin 1.4 sign-artifacts verify sign ================================================ FILE: vendor/github.com/prometheus/client_model/setup.py ================================================ #!/usr/bin/python from setuptools import setup setup( name = 'prometheus_client_model', version = '0.0.1', author = 'Matt T. Proud', author_email = 'matt.proud@gmail.com', description = 'Data model artifacts for the Prometheus client.', license = 'Apache License 2.0', url = 'http://github.com/prometheus/client_model', packages = ['prometheus', 'prometheus/client', 'prometheus/client/model'], package_dir = {'': 'python'}, requires = ['protobuf(==2.4.1)'], platforms = 'Platform Independent', classifiers = ['Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Topic :: Software Development :: Testing', 'Topic :: System :: Monitoring']) ================================================ FILE: vendor/github.com/prometheus/common/.travis.yml ================================================ sudo: false language: go go: - 1.7.5 - tip ================================================ FILE: vendor/github.com/prometheus/common/CONTRIBUTING.md ================================================ # Contributing Prometheus uses GitHub to manage reviews of pull requests. * If you have a trivial fix or improvement, go ahead and create a pull request, addressing (with `@...`) the maintainer of this repository (see [MAINTAINERS.md](MAINTAINERS.md)) in the description of the pull request. * If you plan to do something more involved, first discuss your ideas on our [mailing list](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). This will avoid unnecessary work and surely give you and us a good deal of inspiration. * Relevant coding style guidelines are the [Go Code Review Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) and the _Formatting and style_ section of Peter Bourgon's [Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/#formatting-and-style). ================================================ FILE: vendor/github.com/prometheus/common/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: vendor/github.com/prometheus/common/MAINTAINERS.md ================================================ * Fabian Reinartz ================================================ FILE: vendor/github.com/prometheus/common/NOTICE ================================================ Common libraries shared by Prometheus Go components. Copyright 2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). ================================================ FILE: vendor/github.com/prometheus/common/README.md ================================================ # Common [![Build Status](https://travis-ci.org/prometheus/common.svg)](https://travis-ci.org/prometheus/common) This repository contains Go libraries that are shared across Prometheus components and libraries. * **config**: Common configuration structures * **expfmt**: Decoding and encoding for the exposition format * **log**: A logging wrapper around [logrus](https://github.com/sirupsen/logrus) * **model**: Shared data structures * **route**: A routing wrapper around [httprouter](https://github.com/julienschmidt/httprouter) using `context.Context` * **version**: Version informations and metric ================================================ FILE: vendor/github.com/prometheus/common/expfmt/bench_test.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "bytes" "compress/gzip" "io" "io/ioutil" "testing" "github.com/matttproud/golang_protobuf_extensions/pbutil" dto "github.com/prometheus/client_model/go" ) var parser TextParser // Benchmarks to show how much penalty text format parsing actually inflicts. // // Example results on Linux 3.13.0, Intel(R) Core(TM) i7-4700MQ CPU @ 2.40GHz, go1.4. // // BenchmarkParseText 1000 1188535 ns/op 205085 B/op 6135 allocs/op // BenchmarkParseTextGzip 1000 1376567 ns/op 246224 B/op 6151 allocs/op // BenchmarkParseProto 10000 172790 ns/op 52258 B/op 1160 allocs/op // BenchmarkParseProtoGzip 5000 324021 ns/op 94931 B/op 1211 allocs/op // BenchmarkParseProtoMap 10000 187946 ns/op 58714 B/op 1203 allocs/op // // CONCLUSION: The overhead for the map is negligible. Text format needs ~5x more allocations. // Without compression, it needs ~7x longer, but with compression (the more relevant scenario), // the difference becomes less relevant, only ~4x. // // The test data contains 248 samples. // BenchmarkParseText benchmarks the parsing of a text-format scrape into metric // family DTOs. func BenchmarkParseText(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/text") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { if _, err := parser.TextToMetricFamilies(bytes.NewReader(data)); err != nil { b.Fatal(err) } } } // BenchmarkParseTextGzip benchmarks the parsing of a gzipped text-format scrape // into metric family DTOs. func BenchmarkParseTextGzip(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/text.gz") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { in, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { b.Fatal(err) } if _, err := parser.TextToMetricFamilies(in); err != nil { b.Fatal(err) } } } // BenchmarkParseProto benchmarks the parsing of a protobuf-format scrape into // metric family DTOs. Note that this does not build a map of metric families // (as the text version does), because it is not required for Prometheus // ingestion either. (However, it is required for the text-format parsing, as // the metric family might be sprinkled all over the text, while the // protobuf-format guarantees bundling at one place.) func BenchmarkParseProto(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/protobuf") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { family := &dto.MetricFamily{} in := bytes.NewReader(data) for { family.Reset() if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } b.Fatal(err) } } } } // BenchmarkParseProtoGzip is like BenchmarkParseProto above, but parses gzipped // protobuf format. func BenchmarkParseProtoGzip(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/protobuf.gz") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { family := &dto.MetricFamily{} in, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { b.Fatal(err) } for { family.Reset() if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } b.Fatal(err) } } } } // BenchmarkParseProtoMap is like BenchmarkParseProto but DOES put the parsed // metric family DTOs into a map. This is not happening during Prometheus // ingestion. It is just here to measure the overhead of that map creation and // separate it from the overhead of the text format parsing. func BenchmarkParseProtoMap(b *testing.B) { b.StopTimer() data, err := ioutil.ReadFile("testdata/protobuf") if err != nil { b.Fatal(err) } b.StartTimer() for i := 0; i < b.N; i++ { families := map[string]*dto.MetricFamily{} in := bytes.NewReader(data) for { family := &dto.MetricFamily{} if _, err := pbutil.ReadDelimited(in, family); err != nil { if err == io.EOF { break } b.Fatal(err) } families[family.GetName()] = family } } } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/decode.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "fmt" "io" "math" "mime" "net/http" dto "github.com/prometheus/client_model/go" "github.com/matttproud/golang_protobuf_extensions/pbutil" "github.com/prometheus/common/model" ) // Decoder types decode an input stream into metric families. type Decoder interface { Decode(*dto.MetricFamily) error } // DecodeOptions contains options used by the Decoder and in sample extraction. type DecodeOptions struct { // Timestamp is added to each value from the stream that has no explicit timestamp set. Timestamp model.Time } // ResponseFormat extracts the correct format from a HTTP response header. // If no matching format can be found FormatUnknown is returned. func ResponseFormat(h http.Header) Format { ct := h.Get(hdrContentType) mediatype, params, err := mime.ParseMediaType(ct) if err != nil { return FmtUnknown } const textType = "text/plain" switch mediatype { case ProtoType: if p, ok := params["proto"]; ok && p != ProtoProtocol { return FmtUnknown } if e, ok := params["encoding"]; ok && e != "delimited" { return FmtUnknown } return FmtProtoDelim case textType: if v, ok := params["version"]; ok && v != TextVersion { return FmtUnknown } return FmtText } return FmtUnknown } // NewDecoder returns a new decoder based on the given input format. // If the input format does not imply otherwise, a text format decoder is returned. func NewDecoder(r io.Reader, format Format) Decoder { switch format { case FmtProtoDelim: return &protoDecoder{r: r} } return &textDecoder{r: r} } // protoDecoder implements the Decoder interface for protocol buffers. type protoDecoder struct { r io.Reader } // Decode implements the Decoder interface. func (d *protoDecoder) Decode(v *dto.MetricFamily) error { _, err := pbutil.ReadDelimited(d.r, v) if err != nil { return err } if !model.IsValidMetricName(model.LabelValue(v.GetName())) { return fmt.Errorf("invalid metric name %q", v.GetName()) } for _, m := range v.GetMetric() { if m == nil { continue } for _, l := range m.GetLabel() { if l == nil { continue } if !model.LabelValue(l.GetValue()).IsValid() { return fmt.Errorf("invalid label value %q", l.GetValue()) } if !model.LabelName(l.GetName()).IsValid() { return fmt.Errorf("invalid label name %q", l.GetName()) } } } return nil } // textDecoder implements the Decoder interface for the text protocol. type textDecoder struct { r io.Reader p TextParser fams []*dto.MetricFamily } // Decode implements the Decoder interface. func (d *textDecoder) Decode(v *dto.MetricFamily) error { // TODO(fabxc): Wrap this as a line reader to make streaming safer. if len(d.fams) == 0 { // No cached metric families, read everything and parse metrics. fams, err := d.p.TextToMetricFamilies(d.r) if err != nil { return err } if len(fams) == 0 { return io.EOF } d.fams = make([]*dto.MetricFamily, 0, len(fams)) for _, f := range fams { d.fams = append(d.fams, f) } } *v = *d.fams[0] d.fams = d.fams[1:] return nil } // SampleDecoder wraps a Decoder to extract samples from the metric families // decoded by the wrapped Decoder. type SampleDecoder struct { Dec Decoder Opts *DecodeOptions f dto.MetricFamily } // Decode calls the Decode method of the wrapped Decoder and then extracts the // samples from the decoded MetricFamily into the provided model.Vector. func (sd *SampleDecoder) Decode(s *model.Vector) error { err := sd.Dec.Decode(&sd.f) if err != nil { return err } *s, err = extractSamples(&sd.f, sd.Opts) return err } // ExtractSamples builds a slice of samples from the provided metric // families. If an error occurs during sample extraction, it continues to // extract from the remaining metric families. The returned error is the last // error that has occured. func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) (model.Vector, error) { var ( all model.Vector lastErr error ) for _, f := range fams { some, err := extractSamples(f, o) if err != nil { lastErr = err continue } all = append(all, some...) } return all, lastErr } func extractSamples(f *dto.MetricFamily, o *DecodeOptions) (model.Vector, error) { switch f.GetType() { case dto.MetricType_COUNTER: return extractCounter(o, f), nil case dto.MetricType_GAUGE: return extractGauge(o, f), nil case dto.MetricType_SUMMARY: return extractSummary(o, f), nil case dto.MetricType_UNTYPED: return extractUntyped(o, f), nil case dto.MetricType_HISTOGRAM: return extractHistogram(o, f), nil } return nil, fmt.Errorf("expfmt.extractSamples: unknown metric family type %v", f.GetType()) } func extractCounter(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Counter == nil { continue } lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName()) smpl := &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Counter.GetValue()), } if m.TimestampMs != nil { smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } else { smpl.Timestamp = o.Timestamp } samples = append(samples, smpl) } return samples } func extractGauge(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Gauge == nil { continue } lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName()) smpl := &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Gauge.GetValue()), } if m.TimestampMs != nil { smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } else { smpl.Timestamp = o.Timestamp } samples = append(samples, smpl) } return samples } func extractUntyped(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Untyped == nil { continue } lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName()) smpl := &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Untyped.GetValue()), } if m.TimestampMs != nil { smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } else { smpl.Timestamp = o.Timestamp } samples = append(samples, smpl) } return samples } func extractSummary(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Summary == nil { continue } timestamp := o.Timestamp if m.TimestampMs != nil { timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } for _, q := range m.Summary.Quantile { lset := make(model.LabelSet, len(m.Label)+2) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } // BUG(matt): Update other names to "quantile". lset[model.LabelName(model.QuantileLabel)] = model.LabelValue(fmt.Sprint(q.GetQuantile())) lset[model.MetricNameLabel] = model.LabelValue(f.GetName()) samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(q.GetValue()), Timestamp: timestamp, }) } lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Summary.GetSampleSum()), Timestamp: timestamp, }) lset = make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Summary.GetSampleCount()), Timestamp: timestamp, }) } return samples } func extractHistogram(o *DecodeOptions, f *dto.MetricFamily) model.Vector { samples := make(model.Vector, 0, len(f.Metric)) for _, m := range f.Metric { if m.Histogram == nil { continue } timestamp := o.Timestamp if m.TimestampMs != nil { timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000) } infSeen := false for _, q := range m.Histogram.Bucket { lset := make(model.LabelSet, len(m.Label)+2) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.LabelName(model.BucketLabel)] = model.LabelValue(fmt.Sprint(q.GetUpperBound())) lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket") if math.IsInf(q.GetUpperBound(), +1) { infSeen = true } samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(q.GetCumulativeCount()), Timestamp: timestamp, }) } lset := make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_sum") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Histogram.GetSampleSum()), Timestamp: timestamp, }) lset = make(model.LabelSet, len(m.Label)+1) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_count") count := &model.Sample{ Metric: model.Metric(lset), Value: model.SampleValue(m.Histogram.GetSampleCount()), Timestamp: timestamp, } samples = append(samples, count) if !infSeen { // Append an infinity bucket sample. lset := make(model.LabelSet, len(m.Label)+2) for _, p := range m.Label { lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue()) } lset[model.LabelName(model.BucketLabel)] = model.LabelValue("+Inf") lset[model.MetricNameLabel] = model.LabelValue(f.GetName() + "_bucket") samples = append(samples, &model.Sample{ Metric: model.Metric(lset), Value: count.Value, Timestamp: timestamp, }) } } return samples } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/decode_test.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "io" "net/http" "reflect" "sort" "strings" "testing" "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/model" ) func TestTextDecoder(t *testing.T) { var ( ts = model.Now() in = ` # Only a quite simple scenario with two metric families. # More complicated tests of the parser itself can be found in the text package. # TYPE mf2 counter mf2 3 mf1{label="value1"} -3.14 123456 mf1{label="value2"} 42 mf2 4 ` out = model.Vector{ &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "mf1", "label": "value1", }, Value: -3.14, Timestamp: 123456, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "mf1", "label": "value2", }, Value: 42, Timestamp: ts, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "mf2", }, Value: 3, Timestamp: ts, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "mf2", }, Value: 4, Timestamp: ts, }, } ) dec := &SampleDecoder{ Dec: &textDecoder{r: strings.NewReader(in)}, Opts: &DecodeOptions{ Timestamp: ts, }, } var all model.Vector for { var smpls model.Vector err := dec.Decode(&smpls) if err == io.EOF { break } if err != nil { t.Fatal(err) } all = append(all, smpls...) } sort.Sort(all) sort.Sort(out) if !reflect.DeepEqual(all, out) { t.Fatalf("output does not match") } } func TestProtoDecoder(t *testing.T) { var testTime = model.Now() scenarios := []struct { in string expected model.Vector fail bool }{ { in: "", }, { in: "\x8f\x01\n\rrequest_count\x12\x12Number of requests\x18\x00\"0\n#\n\x0fsome_!abel_name\x12\x10some_label_value\x1a\t\t\x00\x00\x00\x00\x00\x00E\xc0\"6\n)\n\x12another_label_name\x12\x13another_label_value\x1a\t\t\x00\x00\x00\x00\x00\x00U@", fail: true, }, { in: "\x8f\x01\n\rrequest_count\x12\x12Number of requests\x18\x00\"0\n#\n\x0fsome_label_name\x12\x10some_label_value\x1a\t\t\x00\x00\x00\x00\x00\x00E\xc0\"6\n)\n\x12another_label_name\x12\x13another_label_value\x1a\t\t\x00\x00\x00\x00\x00\x00U@", expected: model.Vector{ &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count", "some_label_name": "some_label_value", }, Value: -42, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count", "another_label_name": "another_label_value", }, Value: 84, Timestamp: testTime, }, }, }, { in: "\xb9\x01\n\rrequest_count\x12\x12Number of requests\x18\x02\"O\n#\n\x0fsome_label_name\x12\x10some_label_value\"(\x1a\x12\t\xaeG\xe1z\x14\xae\xef?\x11\x00\x00\x00\x00\x00\x00E\xc0\x1a\x12\t+\x87\x16\xd9\xce\xf7\xef?\x11\x00\x00\x00\x00\x00\x00U\xc0\"A\n)\n\x12another_label_name\x12\x13another_label_value\"\x14\x1a\x12\t\x00\x00\x00\x00\x00\x00\xe0?\x11\x00\x00\x00\x00\x00\x00$@", expected: model.Vector{ &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count_count", "some_label_name": "some_label_value", }, Value: 0, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count_sum", "some_label_name": "some_label_value", }, Value: 0, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count", "some_label_name": "some_label_value", "quantile": "0.99", }, Value: -42, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count", "some_label_name": "some_label_value", "quantile": "0.999", }, Value: -84, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count_count", "another_label_name": "another_label_value", }, Value: 0, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count_sum", "another_label_name": "another_label_value", }, Value: 0, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count", "another_label_name": "another_label_value", "quantile": "0.5", }, Value: 10, Timestamp: testTime, }, }, }, { in: "\x8d\x01\n\x1drequest_duration_microseconds\x12\x15The response latency.\x18\x04\"S:Q\b\x85\x15\x11\xcd\xcc\xccL\x8f\xcb:A\x1a\v\b{\x11\x00\x00\x00\x00\x00\x00Y@\x1a\f\b\x9c\x03\x11\x00\x00\x00\x00\x00\x00^@\x1a\f\b\xd0\x04\x11\x00\x00\x00\x00\x00\x00b@\x1a\f\b\xf4\v\x11\x9a\x99\x99\x99\x99\x99e@\x1a\f\b\x85\x15\x11\x00\x00\x00\x00\x00\x00\xf0\u007f", expected: model.Vector{ &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_bucket", "le": "100", }, Value: 123, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_bucket", "le": "120", }, Value: 412, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_bucket", "le": "144", }, Value: 592, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_bucket", "le": "172.8", }, Value: 1524, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_bucket", "le": "+Inf", }, Value: 2693, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_sum", }, Value: 1756047.3, Timestamp: testTime, }, &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_duration_microseconds_count", }, Value: 2693, Timestamp: testTime, }, }, }, { // The metric type is unset in this protobuf, which needs to be handled // correctly by the decoder. in: "\x1c\n\rrequest_count\"\v\x1a\t\t\x00\x00\x00\x00\x00\x00\xf0?", expected: model.Vector{ &model.Sample{ Metric: model.Metric{ model.MetricNameLabel: "request_count", }, Value: 1, Timestamp: testTime, }, }, }, } for i, scenario := range scenarios { dec := &SampleDecoder{ Dec: &protoDecoder{r: strings.NewReader(scenario.in)}, Opts: &DecodeOptions{ Timestamp: testTime, }, } var all model.Vector for { var smpls model.Vector err := dec.Decode(&smpls) if err == io.EOF { break } if scenario.fail { if err == nil { t.Fatal("Expected error but got none") } break } if err != nil { t.Fatal(err) } all = append(all, smpls...) } sort.Sort(all) sort.Sort(scenario.expected) if !reflect.DeepEqual(all, scenario.expected) { t.Fatalf("%d. output does not match, want: %#v, got %#v", i, scenario.expected, all) } } } func testDiscriminatorHTTPHeader(t testing.TB) { var scenarios = []struct { input map[string]string output Format err error }{ { input: map[string]string{"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="delimited"`}, output: FmtProtoDelim, }, { input: map[string]string{"Content-Type": `application/vnd.google.protobuf; proto="illegal"; encoding="delimited"`}, output: FmtUnknown, }, { input: map[string]string{"Content-Type": `application/vnd.google.protobuf; proto="io.prometheus.client.MetricFamily"; encoding="illegal"`}, output: FmtUnknown, }, { input: map[string]string{"Content-Type": `text/plain; version=0.0.4`}, output: FmtText, }, { input: map[string]string{"Content-Type": `text/plain`}, output: FmtText, }, { input: map[string]string{"Content-Type": `text/plain; version=0.0.3`}, output: FmtUnknown, }, } for i, scenario := range scenarios { var header http.Header if len(scenario.input) > 0 { header = http.Header{} } for key, value := range scenario.input { header.Add(key, value) } actual := ResponseFormat(header) if scenario.output != actual { t.Errorf("%d. expected %s, got %s", i, scenario.output, actual) } } } func TestDiscriminatorHTTPHeader(t *testing.T) { testDiscriminatorHTTPHeader(t) } func BenchmarkDiscriminatorHTTPHeader(b *testing.B) { for i := 0; i < b.N; i++ { testDiscriminatorHTTPHeader(b) } } func TestExtractSamples(t *testing.T) { var ( goodMetricFamily1 = &dto.MetricFamily{ Name: proto.String("foo"), Help: proto.String("Help for foo."), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Counter: &dto.Counter{ Value: proto.Float64(4711), }, }, }, } goodMetricFamily2 = &dto.MetricFamily{ Name: proto.String("bar"), Help: proto.String("Help for bar."), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Gauge: &dto.Gauge{ Value: proto.Float64(3.14), }, }, }, } badMetricFamily = &dto.MetricFamily{ Name: proto.String("bad"), Help: proto.String("Help for bad."), Type: dto.MetricType(42).Enum(), Metric: []*dto.Metric{ &dto.Metric{ Gauge: &dto.Gauge{ Value: proto.Float64(2.7), }, }, }, } opts = &DecodeOptions{ Timestamp: 42, } ) got, err := ExtractSamples(opts, goodMetricFamily1, goodMetricFamily2) if err != nil { t.Error("Unexpected error from ExtractSamples:", err) } want := model.Vector{ &model.Sample{Metric: model.Metric{model.MetricNameLabel: "foo"}, Value: 4711, Timestamp: 42}, &model.Sample{Metric: model.Metric{model.MetricNameLabel: "bar"}, Value: 3.14, Timestamp: 42}, } if !reflect.DeepEqual(got, want) { t.Errorf("unexpected samples extracted, got: %v, want: %v", got, want) } got, err = ExtractSamples(opts, goodMetricFamily1, badMetricFamily, goodMetricFamily2) if err == nil { t.Error("Expected error from ExtractSamples") } if !reflect.DeepEqual(got, want) { t.Errorf("unexpected samples extracted, got: %v, want: %v", got, want) } } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/encode.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "fmt" "io" "net/http" "github.com/golang/protobuf/proto" "github.com/matttproud/golang_protobuf_extensions/pbutil" "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" dto "github.com/prometheus/client_model/go" ) // Encoder types encode metric families into an underlying wire protocol. type Encoder interface { Encode(*dto.MetricFamily) error } type encoder func(*dto.MetricFamily) error func (e encoder) Encode(v *dto.MetricFamily) error { return e(v) } // Negotiate returns the Content-Type based on the given Accept header. // If no appropriate accepted type is found, FmtText is returned. func Negotiate(h http.Header) Format { for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) { // Check for protocol buffer if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol { switch ac.Params["encoding"] { case "delimited": return FmtProtoDelim case "text": return FmtProtoText case "compact-text": return FmtProtoCompact } } // Check for text format. ver := ac.Params["version"] if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") { return FmtText } } return FmtText } // NewEncoder returns a new encoder based on content type negotiation. func NewEncoder(w io.Writer, format Format) Encoder { switch format { case FmtProtoDelim: return encoder(func(v *dto.MetricFamily) error { _, err := pbutil.WriteDelimited(w, v) return err }) case FmtProtoCompact: return encoder(func(v *dto.MetricFamily) error { _, err := fmt.Fprintln(w, v.String()) return err }) case FmtProtoText: return encoder(func(v *dto.MetricFamily) error { _, err := fmt.Fprintln(w, proto.MarshalTextString(v)) return err }) case FmtText: return encoder(func(v *dto.MetricFamily) error { _, err := MetricFamilyToText(w, v) return err }) } panic("expfmt.NewEncoder: unknown format") } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/expfmt.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package expfmt contains tools for reading and writing Prometheus metrics. package expfmt // Format specifies the HTTP content type of the different wire protocols. type Format string // Constants to assemble the Content-Type values for the different wire protocols. const ( TextVersion = "0.0.4" ProtoType = `application/vnd.google.protobuf` ProtoProtocol = `io.prometheus.client.MetricFamily` ProtoFmt = ProtoType + "; proto=" + ProtoProtocol + ";" // The Content-Type values for the different wire protocols. FmtUnknown Format = `` FmtText Format = `text/plain; version=` + TextVersion FmtProtoDelim Format = ProtoFmt + ` encoding=delimited` FmtProtoText Format = ProtoFmt + ` encoding=text` FmtProtoCompact Format = ProtoFmt + ` encoding=compact-text` ) const ( hdrContentType = "Content-Type" hdrAccept = "Accept" ) ================================================ FILE: vendor/github.com/prometheus/common/expfmt/fuzz.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Build only when actually fuzzing // +build gofuzz package expfmt import "bytes" // Fuzz text metric parser with with github.com/dvyukov/go-fuzz: // // go-fuzz-build github.com/prometheus/common/expfmt // go-fuzz -bin expfmt-fuzz.zip -workdir fuzz // // Further input samples should go in the folder fuzz/corpus. func Fuzz(in []byte) int { parser := TextParser{} _, err := parser.TextToMetricFamilies(bytes.NewReader(in)) if err != nil { return 0 } return 1 } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/text_create.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "fmt" "io" "math" "strings" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/model" ) // MetricFamilyToText converts a MetricFamily proto message into text format and // writes the resulting lines to 'out'. It returns the number of bytes written // and any error encountered. The output will have the same order as the input, // no further sorting is performed. Furthermore, this function assumes the input // is already sanitized and does not perform any sanity checks. If the input // contains duplicate metrics or invalid metric or label names, the conversion // will result in invalid text format output. // // This method fulfills the type 'prometheus.encoder'. func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (int, error) { var written int // Fail-fast checks. if len(in.Metric) == 0 { return written, fmt.Errorf("MetricFamily has no metrics: %s", in) } name := in.GetName() if name == "" { return written, fmt.Errorf("MetricFamily has no name: %s", in) } // Comments, first HELP, then TYPE. if in.Help != nil { n, err := fmt.Fprintf( out, "# HELP %s %s\n", name, escapeString(*in.Help, false), ) written += n if err != nil { return written, err } } metricType := in.GetType() n, err := fmt.Fprintf( out, "# TYPE %s %s\n", name, strings.ToLower(metricType.String()), ) written += n if err != nil { return written, err } // Finally the samples, one line for each. for _, metric := range in.Metric { switch metricType { case dto.MetricType_COUNTER: if metric.Counter == nil { return written, fmt.Errorf( "expected counter in metric %s %s", name, metric, ) } n, err = writeSample( name, metric, "", "", metric.Counter.GetValue(), out, ) case dto.MetricType_GAUGE: if metric.Gauge == nil { return written, fmt.Errorf( "expected gauge in metric %s %s", name, metric, ) } n, err = writeSample( name, metric, "", "", metric.Gauge.GetValue(), out, ) case dto.MetricType_UNTYPED: if metric.Untyped == nil { return written, fmt.Errorf( "expected untyped in metric %s %s", name, metric, ) } n, err = writeSample( name, metric, "", "", metric.Untyped.GetValue(), out, ) case dto.MetricType_SUMMARY: if metric.Summary == nil { return written, fmt.Errorf( "expected summary in metric %s %s", name, metric, ) } for _, q := range metric.Summary.Quantile { n, err = writeSample( name, metric, model.QuantileLabel, fmt.Sprint(q.GetQuantile()), q.GetValue(), out, ) written += n if err != nil { return written, err } } n, err = writeSample( name+"_sum", metric, "", "", metric.Summary.GetSampleSum(), out, ) if err != nil { return written, err } written += n n, err = writeSample( name+"_count", metric, "", "", float64(metric.Summary.GetSampleCount()), out, ) case dto.MetricType_HISTOGRAM: if metric.Histogram == nil { return written, fmt.Errorf( "expected histogram in metric %s %s", name, metric, ) } infSeen := false for _, q := range metric.Histogram.Bucket { n, err = writeSample( name+"_bucket", metric, model.BucketLabel, fmt.Sprint(q.GetUpperBound()), float64(q.GetCumulativeCount()), out, ) written += n if err != nil { return written, err } if math.IsInf(q.GetUpperBound(), +1) { infSeen = true } } if !infSeen { n, err = writeSample( name+"_bucket", metric, model.BucketLabel, "+Inf", float64(metric.Histogram.GetSampleCount()), out, ) if err != nil { return written, err } written += n } n, err = writeSample( name+"_sum", metric, "", "", metric.Histogram.GetSampleSum(), out, ) if err != nil { return written, err } written += n n, err = writeSample( name+"_count", metric, "", "", float64(metric.Histogram.GetSampleCount()), out, ) default: return written, fmt.Errorf( "unexpected type in metric %s %s", name, metric, ) } written += n if err != nil { return written, err } } return written, nil } // writeSample writes a single sample in text format to out, given the metric // name, the metric proto message itself, optionally an additional label name // and value (use empty strings if not required), and the value. The function // returns the number of bytes written and any error encountered. func writeSample( name string, metric *dto.Metric, additionalLabelName, additionalLabelValue string, value float64, out io.Writer, ) (int, error) { var written int n, err := fmt.Fprint(out, name) written += n if err != nil { return written, err } n, err = labelPairsToText( metric.Label, additionalLabelName, additionalLabelValue, out, ) written += n if err != nil { return written, err } n, err = fmt.Fprintf(out, " %v", value) written += n if err != nil { return written, err } if metric.TimestampMs != nil { n, err = fmt.Fprintf(out, " %v", *metric.TimestampMs) written += n if err != nil { return written, err } } n, err = out.Write([]byte{'\n'}) written += n if err != nil { return written, err } return written, nil } // labelPairsToText converts a slice of LabelPair proto messages plus the // explicitly given additional label pair into text formatted as required by the // text format and writes it to 'out'. An empty slice in combination with an // empty string 'additionalLabelName' results in nothing being // written. Otherwise, the label pairs are written, escaped as required by the // text format, and enclosed in '{...}'. The function returns the number of // bytes written and any error encountered. func labelPairsToText( in []*dto.LabelPair, additionalLabelName, additionalLabelValue string, out io.Writer, ) (int, error) { if len(in) == 0 && additionalLabelName == "" { return 0, nil } var written int separator := '{' for _, lp := range in { n, err := fmt.Fprintf( out, `%c%s="%s"`, separator, lp.GetName(), escapeString(lp.GetValue(), true), ) written += n if err != nil { return written, err } separator = ',' } if additionalLabelName != "" { n, err := fmt.Fprintf( out, `%c%s="%s"`, separator, additionalLabelName, escapeString(additionalLabelValue, true), ) written += n if err != nil { return written, err } } n, err := out.Write([]byte{'}'}) written += n if err != nil { return written, err } return written, nil } var ( escape = strings.NewReplacer("\\", `\\`, "\n", `\n`) escapeWithDoubleQuote = strings.NewReplacer("\\", `\\`, "\n", `\n`, "\"", `\"`) ) // escapeString replaces '\' by '\\', new line character by '\n', and - if // includeDoubleQuote is true - '"' by '\"'. func escapeString(v string, includeDoubleQuote bool) string { if includeDoubleQuote { return escapeWithDoubleQuote.Replace(v) } return escape.Replace(v) } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/text_create_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "bytes" "math" "strings" "testing" "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) func testCreate(t testing.TB) { var scenarios = []struct { in *dto.MetricFamily out string }{ // 0: Counter, NaN as value, timestamp given. { in: &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val1"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(math.NaN()), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(.23), }, TimestampMs: proto.Int64(1234567890), }, }, }, out: `# HELP name two-line\n doc str\\ing # TYPE name counter name{labelname="val1",basename="basevalue"} NaN name{labelname="val2",basename="basevalue"} 0.23 1234567890 `, }, // 1: Gauge, some escaping required, +Inf as value, multi-byte characters in label values. { in: &dto.MetricFamily{ Name: proto.String("gauge_name"), Help: proto.String("gauge\ndoc\nstr\"ing"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("val with\nnew line"), }, &dto.LabelPair{ Name: proto.String("name_2"), Value: proto.String("val with \\backslash and \"quotes\""), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(math.Inf(+1)), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("Björn"), }, &dto.LabelPair{ Name: proto.String("name_2"), Value: proto.String("佖佥"), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(3.14E42), }, }, }, }, out: `# HELP gauge_name gauge\ndoc\nstr"ing # TYPE gauge_name gauge gauge_name{name_1="val with\nnew line",name_2="val with \\backslash and \"quotes\""} +Inf gauge_name{name_1="Björn",name_2="佖佥"} 3.14e+42 `, }, // 2: Untyped, no help, one sample with no labels and -Inf as value, another sample with one label. { in: &dto.MetricFamily{ Name: proto.String("untyped_name"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("value 1"), }, }, Untyped: &dto.Untyped{ Value: proto.Float64(-1.23e-45), }, }, }, }, out: `# TYPE untyped_name untyped untyped_name -Inf untyped_name{name_1="value 1"} -1.23e-45 `, }, // 3: Summary. { in: &dto.MetricFamily{ Name: proto.String("summary_name"), Help: proto.String("summary docstring"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Summary: &dto.Summary{ SampleCount: proto.Uint64(42), SampleSum: proto.Float64(-3.4567), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.5), Value: proto.Float64(-1.23), }, &dto.Quantile{ Quantile: proto.Float64(0.9), Value: proto.Float64(.2342354), }, &dto.Quantile{ Quantile: proto.Float64(0.99), Value: proto.Float64(0), }, }, }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("name_1"), Value: proto.String("value 1"), }, &dto.LabelPair{ Name: proto.String("name_2"), Value: proto.String("value 2"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(4711), SampleSum: proto.Float64(2010.1971), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.5), Value: proto.Float64(1), }, &dto.Quantile{ Quantile: proto.Float64(0.9), Value: proto.Float64(2), }, &dto.Quantile{ Quantile: proto.Float64(0.99), Value: proto.Float64(3), }, }, }, }, }, }, out: `# HELP summary_name summary docstring # TYPE summary_name summary summary_name{quantile="0.5"} -1.23 summary_name{quantile="0.9"} 0.2342354 summary_name{quantile="0.99"} 0 summary_name_sum -3.4567 summary_name_count 42 summary_name{name_1="value 1",name_2="value 2",quantile="0.5"} 1 summary_name{name_1="value 1",name_2="value 2",quantile="0.9"} 2 summary_name{name_1="value 1",name_2="value 2",quantile="0.99"} 3 summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 `, }, // 4: Histogram { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ &dto.Bucket{ UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, &dto.Bucket{ UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, &dto.Bucket{ UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, &dto.Bucket{ UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, &dto.Bucket{ UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, }, }, }, }, }, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100"} 123 request_duration_microseconds_bucket{le="120"} 412 request_duration_microseconds_bucket{le="144"} 592 request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, // 5: Histogram with missing +Inf bucket. { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ &dto.Bucket{ UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, &dto.Bucket{ UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, &dto.Bucket{ UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, &dto.Bucket{ UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, }, }, }, }, }, out: `# HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100"} 123 request_duration_microseconds_bucket{le="120"} 412 request_duration_microseconds_bucket{le="144"} 592 request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, // 6: No metric type, should result in default type Counter. { in: &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("doc string"), Metric: []*dto.Metric{ &dto.Metric{ Counter: &dto.Counter{ Value: proto.Float64(math.Inf(-1)), }, }, }, }, out: `# HELP name doc string # TYPE name counter name -Inf `, }, } for i, scenario := range scenarios { out := bytes.NewBuffer(make([]byte, 0, len(scenario.out))) n, err := MetricFamilyToText(out, scenario.in) if err != nil { t.Errorf("%d. error: %s", i, err) continue } if expected, got := len(scenario.out), n; expected != got { t.Errorf( "%d. expected %d bytes written, got %d", i, expected, got, ) } if expected, got := scenario.out, out.String(); expected != got { t.Errorf( "%d. expected out=%q, got %q", i, expected, got, ) } } } func TestCreate(t *testing.T) { testCreate(t) } func BenchmarkCreate(b *testing.B) { for i := 0; i < b.N; i++ { testCreate(b) } } func testCreateError(t testing.TB) { var scenarios = []struct { in *dto.MetricFamily err string }{ // 0: No metric. { in: &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("doc string"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{}, }, err: "MetricFamily has no metrics", }, // 1: No metric name. { in: &dto.MetricFamily{ Help: proto.String("doc string"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, }, }, }, err: "MetricFamily has no name", }, // 2: Wrong type. { in: &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("doc string"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, }, }, }, err: "expected counter in metric", }, } for i, scenario := range scenarios { var out bytes.Buffer _, err := MetricFamilyToText(&out, scenario.in) if err == nil { t.Errorf("%d. expected error, got nil", i) continue } if expected, got := scenario.err, err.Error(); strings.Index(got, expected) != 0 { t.Errorf( "%d. expected error starting with %q, got %q", i, expected, got, ) } } } func TestCreateError(t *testing.T) { testCreateError(t) } func BenchmarkCreateError(b *testing.B) { for i := 0; i < b.N; i++ { testCreateError(b) } } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/text_parse.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "bufio" "bytes" "fmt" "io" "math" "strconv" "strings" dto "github.com/prometheus/client_model/go" "github.com/golang/protobuf/proto" "github.com/prometheus/common/model" ) // A stateFn is a function that represents a state in a state machine. By // executing it, the state is progressed to the next state. The stateFn returns // another stateFn, which represents the new state. The end state is represented // by nil. type stateFn func() stateFn // ParseError signals errors while parsing the simple and flat text-based // exchange format. type ParseError struct { Line int Msg string } // Error implements the error interface. func (e ParseError) Error() string { return fmt.Sprintf("text format parsing error in line %d: %s", e.Line, e.Msg) } // TextParser is used to parse the simple and flat text-based exchange format. Its // zero value is ready to use. type TextParser struct { metricFamiliesByName map[string]*dto.MetricFamily buf *bufio.Reader // Where the parsed input is read through. err error // Most recent error. lineCount int // Tracks the line count for error messages. currentByte byte // The most recent byte read. currentToken bytes.Buffer // Re-used each time a token has to be gathered from multiple bytes. currentMF *dto.MetricFamily currentMetric *dto.Metric currentLabelPair *dto.LabelPair // The remaining member variables are only used for summaries/histograms. currentLabels map[string]string // All labels including '__name__' but excluding 'quantile'/'le' // Summary specific. summaries map[uint64]*dto.Metric // Key is created with LabelsToSignature. currentQuantile float64 // Histogram specific. histograms map[uint64]*dto.Metric // Key is created with LabelsToSignature. currentBucket float64 // These tell us if the currently processed line ends on '_count' or // '_sum' respectively and belong to a summary/histogram, representing the sample // count and sum of that summary/histogram. currentIsSummaryCount, currentIsSummarySum bool currentIsHistogramCount, currentIsHistogramSum bool } // TextToMetricFamilies reads 'in' as the simple and flat text-based exchange // format and creates MetricFamily proto messages. It returns the MetricFamily // proto messages in a map where the metric names are the keys, along with any // error encountered. // // If the input contains duplicate metrics (i.e. lines with the same metric name // and exactly the same label set), the resulting MetricFamily will contain // duplicate Metric proto messages. Similar is true for duplicate label // names. Checks for duplicates have to be performed separately, if required. // Also note that neither the metrics within each MetricFamily are sorted nor // the label pairs within each Metric. Sorting is not required for the most // frequent use of this method, which is sample ingestion in the Prometheus // server. However, for presentation purposes, you might want to sort the // metrics, and in some cases, you must sort the labels, e.g. for consumption by // the metric family injection hook of the Prometheus registry. // // Summaries and histograms are rather special beasts. You would probably not // use them in the simple text format anyway. This method can deal with // summaries and histograms if they are presented in exactly the way the // text.Create function creates them. // // This method must not be called concurrently. If you want to parse different // input concurrently, instantiate a separate Parser for each goroutine. func (p *TextParser) TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) { p.reset(in) for nextState := p.startOfLine; nextState != nil; nextState = nextState() { // Magic happens here... } // Get rid of empty metric families. for k, mf := range p.metricFamiliesByName { if len(mf.GetMetric()) == 0 { delete(p.metricFamiliesByName, k) } } // If p.err is io.EOF now, we have run into a premature end of the input // stream. Turn this error into something nicer and more // meaningful. (io.EOF is often used as a signal for the legitimate end // of an input stream.) if p.err == io.EOF { p.parseError("unexpected end of input stream") } return p.metricFamiliesByName, p.err } func (p *TextParser) reset(in io.Reader) { p.metricFamiliesByName = map[string]*dto.MetricFamily{} if p.buf == nil { p.buf = bufio.NewReader(in) } else { p.buf.Reset(in) } p.err = nil p.lineCount = 0 if p.summaries == nil || len(p.summaries) > 0 { p.summaries = map[uint64]*dto.Metric{} } if p.histograms == nil || len(p.histograms) > 0 { p.histograms = map[uint64]*dto.Metric{} } p.currentQuantile = math.NaN() p.currentBucket = math.NaN() } // startOfLine represents the state where the next byte read from p.buf is the // start of a line (or whitespace leading up to it). func (p *TextParser) startOfLine() stateFn { p.lineCount++ if p.skipBlankTab(); p.err != nil { // End of input reached. This is the only case where // that is not an error but a signal that we are done. p.err = nil return nil } switch p.currentByte { case '#': return p.startComment case '\n': return p.startOfLine // Empty line, start the next one. } return p.readingMetricName } // startComment represents the state where the next byte read from p.buf is the // start of a comment (or whitespace leading up to it). func (p *TextParser) startComment() stateFn { if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.currentByte == '\n' { return p.startOfLine } if p.readTokenUntilWhitespace(); p.err != nil { return nil // Unexpected end of input. } // If we have hit the end of line already, there is nothing left // to do. This is not considered a syntax error. if p.currentByte == '\n' { return p.startOfLine } keyword := p.currentToken.String() if keyword != "HELP" && keyword != "TYPE" { // Generic comment, ignore by fast forwarding to end of line. for p.currentByte != '\n' { if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil { return nil // Unexpected end of input. } } return p.startOfLine } // There is something. Next has to be a metric name. if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.readTokenAsMetricName(); p.err != nil { return nil // Unexpected end of input. } if p.currentByte == '\n' { // At the end of the line already. // Again, this is not considered a syntax error. return p.startOfLine } if !isBlankOrTab(p.currentByte) { p.parseError("invalid metric name in comment") return nil } p.setOrCreateCurrentMF() if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.currentByte == '\n' { // At the end of the line already. // Again, this is not considered a syntax error. return p.startOfLine } switch keyword { case "HELP": return p.readingHelp case "TYPE": return p.readingType } panic(fmt.Sprintf("code error: unexpected keyword %q", keyword)) } // readingMetricName represents the state where the last byte read (now in // p.currentByte) is the first byte of a metric name. func (p *TextParser) readingMetricName() stateFn { if p.readTokenAsMetricName(); p.err != nil { return nil } if p.currentToken.Len() == 0 { p.parseError("invalid metric name") return nil } p.setOrCreateCurrentMF() // Now is the time to fix the type if it hasn't happened yet. if p.currentMF.Type == nil { p.currentMF.Type = dto.MetricType_UNTYPED.Enum() } p.currentMetric = &dto.Metric{} // Do not append the newly created currentMetric to // currentMF.Metric right now. First wait if this is a summary, // and the metric exists already, which we can only know after // having read all the labels. if p.skipBlankTabIfCurrentBlankTab(); p.err != nil { return nil // Unexpected end of input. } return p.readingLabels } // readingLabels represents the state where the last byte read (now in // p.currentByte) is either the first byte of the label set (i.e. a '{'), or the // first byte of the value (otherwise). func (p *TextParser) readingLabels() stateFn { // Summaries/histograms are special. We have to reset the // currentLabels map, currentQuantile and currentBucket before starting to // read labels. if p.currentMF.GetType() == dto.MetricType_SUMMARY || p.currentMF.GetType() == dto.MetricType_HISTOGRAM { p.currentLabels = map[string]string{} p.currentLabels[string(model.MetricNameLabel)] = p.currentMF.GetName() p.currentQuantile = math.NaN() p.currentBucket = math.NaN() } if p.currentByte != '{' { return p.readingValue } return p.startLabelName } // startLabelName represents the state where the next byte read from p.buf is // the start of a label name (or whitespace leading up to it). func (p *TextParser) startLabelName() stateFn { if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.currentByte == '}' { if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } return p.readingValue } if p.readTokenAsLabelName(); p.err != nil { return nil // Unexpected end of input. } if p.currentToken.Len() == 0 { p.parseError(fmt.Sprintf("invalid label name for metric %q", p.currentMF.GetName())) return nil } p.currentLabelPair = &dto.LabelPair{Name: proto.String(p.currentToken.String())} if p.currentLabelPair.GetName() == string(model.MetricNameLabel) { p.parseError(fmt.Sprintf("label name %q is reserved", model.MetricNameLabel)) return nil } // Special summary/histogram treatment. Don't add 'quantile' and 'le' // labels to 'real' labels. if !(p.currentMF.GetType() == dto.MetricType_SUMMARY && p.currentLabelPair.GetName() == model.QuantileLabel) && !(p.currentMF.GetType() == dto.MetricType_HISTOGRAM && p.currentLabelPair.GetName() == model.BucketLabel) { p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPair) } if p.skipBlankTabIfCurrentBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.currentByte != '=' { p.parseError(fmt.Sprintf("expected '=' after label name, found %q", p.currentByte)) return nil } return p.startLabelValue } // startLabelValue represents the state where the next byte read from p.buf is // the start of a (quoted) label value (or whitespace leading up to it). func (p *TextParser) startLabelValue() stateFn { if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.currentByte != '"' { p.parseError(fmt.Sprintf("expected '\"' at start of label value, found %q", p.currentByte)) return nil } if p.readTokenAsLabelValue(); p.err != nil { return nil } if !model.LabelValue(p.currentToken.String()).IsValid() { p.parseError(fmt.Sprintf("invalid label value %q", p.currentToken.String())) return nil } p.currentLabelPair.Value = proto.String(p.currentToken.String()) // Special treatment of summaries: // - Quantile labels are special, will result in dto.Quantile later. // - Other labels have to be added to currentLabels for signature calculation. if p.currentMF.GetType() == dto.MetricType_SUMMARY { if p.currentLabelPair.GetName() == model.QuantileLabel { if p.currentQuantile, p.err = strconv.ParseFloat(p.currentLabelPair.GetValue(), 64); p.err != nil { // Create a more helpful error message. p.parseError(fmt.Sprintf("expected float as value for 'quantile' label, got %q", p.currentLabelPair.GetValue())) return nil } } else { p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue() } } // Similar special treatment of histograms. if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { if p.currentLabelPair.GetName() == model.BucketLabel { if p.currentBucket, p.err = strconv.ParseFloat(p.currentLabelPair.GetValue(), 64); p.err != nil { // Create a more helpful error message. p.parseError(fmt.Sprintf("expected float as value for 'le' label, got %q", p.currentLabelPair.GetValue())) return nil } } else { p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue() } } if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } switch p.currentByte { case ',': return p.startLabelName case '}': if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } return p.readingValue default: p.parseError(fmt.Sprintf("unexpected end of label value %q", p.currentLabelPair.Value)) return nil } } // readingValue represents the state where the last byte read (now in // p.currentByte) is the first byte of the sample value (i.e. a float). func (p *TextParser) readingValue() stateFn { // When we are here, we have read all the labels, so for the // special case of a summary/histogram, we can finally find out // if the metric already exists. if p.currentMF.GetType() == dto.MetricType_SUMMARY { signature := model.LabelsToSignature(p.currentLabels) if summary := p.summaries[signature]; summary != nil { p.currentMetric = summary } else { p.summaries[signature] = p.currentMetric p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } } else if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { signature := model.LabelsToSignature(p.currentLabels) if histogram := p.histograms[signature]; histogram != nil { p.currentMetric = histogram } else { p.histograms[signature] = p.currentMetric p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } } else { p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) } if p.readTokenUntilWhitespace(); p.err != nil { return nil // Unexpected end of input. } value, err := strconv.ParseFloat(p.currentToken.String(), 64) if err != nil { // Create a more helpful error message. p.parseError(fmt.Sprintf("expected float as value, got %q", p.currentToken.String())) return nil } switch p.currentMF.GetType() { case dto.MetricType_COUNTER: p.currentMetric.Counter = &dto.Counter{Value: proto.Float64(value)} case dto.MetricType_GAUGE: p.currentMetric.Gauge = &dto.Gauge{Value: proto.Float64(value)} case dto.MetricType_UNTYPED: p.currentMetric.Untyped = &dto.Untyped{Value: proto.Float64(value)} case dto.MetricType_SUMMARY: // *sigh* if p.currentMetric.Summary == nil { p.currentMetric.Summary = &dto.Summary{} } switch { case p.currentIsSummaryCount: p.currentMetric.Summary.SampleCount = proto.Uint64(uint64(value)) case p.currentIsSummarySum: p.currentMetric.Summary.SampleSum = proto.Float64(value) case !math.IsNaN(p.currentQuantile): p.currentMetric.Summary.Quantile = append( p.currentMetric.Summary.Quantile, &dto.Quantile{ Quantile: proto.Float64(p.currentQuantile), Value: proto.Float64(value), }, ) } case dto.MetricType_HISTOGRAM: // *sigh* if p.currentMetric.Histogram == nil { p.currentMetric.Histogram = &dto.Histogram{} } switch { case p.currentIsHistogramCount: p.currentMetric.Histogram.SampleCount = proto.Uint64(uint64(value)) case p.currentIsHistogramSum: p.currentMetric.Histogram.SampleSum = proto.Float64(value) case !math.IsNaN(p.currentBucket): p.currentMetric.Histogram.Bucket = append( p.currentMetric.Histogram.Bucket, &dto.Bucket{ UpperBound: proto.Float64(p.currentBucket), CumulativeCount: proto.Uint64(uint64(value)), }, ) } default: p.err = fmt.Errorf("unexpected type for metric name %q", p.currentMF.GetName()) } if p.currentByte == '\n' { return p.startOfLine } return p.startTimestamp } // startTimestamp represents the state where the next byte read from p.buf is // the start of the timestamp (or whitespace leading up to it). func (p *TextParser) startTimestamp() stateFn { if p.skipBlankTab(); p.err != nil { return nil // Unexpected end of input. } if p.readTokenUntilWhitespace(); p.err != nil { return nil // Unexpected end of input. } timestamp, err := strconv.ParseInt(p.currentToken.String(), 10, 64) if err != nil { // Create a more helpful error message. p.parseError(fmt.Sprintf("expected integer as timestamp, got %q", p.currentToken.String())) return nil } p.currentMetric.TimestampMs = proto.Int64(timestamp) if p.readTokenUntilNewline(false); p.err != nil { return nil // Unexpected end of input. } if p.currentToken.Len() > 0 { p.parseError(fmt.Sprintf("spurious string after timestamp: %q", p.currentToken.String())) return nil } return p.startOfLine } // readingHelp represents the state where the last byte read (now in // p.currentByte) is the first byte of the docstring after 'HELP'. func (p *TextParser) readingHelp() stateFn { if p.currentMF.Help != nil { p.parseError(fmt.Sprintf("second HELP line for metric name %q", p.currentMF.GetName())) return nil } // Rest of line is the docstring. if p.readTokenUntilNewline(true); p.err != nil { return nil // Unexpected end of input. } p.currentMF.Help = proto.String(p.currentToken.String()) return p.startOfLine } // readingType represents the state where the last byte read (now in // p.currentByte) is the first byte of the type hint after 'HELP'. func (p *TextParser) readingType() stateFn { if p.currentMF.Type != nil { p.parseError(fmt.Sprintf("second TYPE line for metric name %q, or TYPE reported after samples", p.currentMF.GetName())) return nil } // Rest of line is the type. if p.readTokenUntilNewline(false); p.err != nil { return nil // Unexpected end of input. } metricType, ok := dto.MetricType_value[strings.ToUpper(p.currentToken.String())] if !ok { p.parseError(fmt.Sprintf("unknown metric type %q", p.currentToken.String())) return nil } p.currentMF.Type = dto.MetricType(metricType).Enum() return p.startOfLine } // parseError sets p.err to a ParseError at the current line with the given // message. func (p *TextParser) parseError(msg string) { p.err = ParseError{ Line: p.lineCount, Msg: msg, } } // skipBlankTab reads (and discards) bytes from p.buf until it encounters a byte // that is neither ' ' nor '\t'. That byte is left in p.currentByte. func (p *TextParser) skipBlankTab() { for { if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil || !isBlankOrTab(p.currentByte) { return } } } // skipBlankTabIfCurrentBlankTab works exactly as skipBlankTab but doesn't do // anything if p.currentByte is neither ' ' nor '\t'. func (p *TextParser) skipBlankTabIfCurrentBlankTab() { if isBlankOrTab(p.currentByte) { p.skipBlankTab() } } // readTokenUntilWhitespace copies bytes from p.buf into p.currentToken. The // first byte considered is the byte already read (now in p.currentByte). The // first whitespace byte encountered is still copied into p.currentByte, but not // into p.currentToken. func (p *TextParser) readTokenUntilWhitespace() { p.currentToken.Reset() for p.err == nil && !isBlankOrTab(p.currentByte) && p.currentByte != '\n' { p.currentToken.WriteByte(p.currentByte) p.currentByte, p.err = p.buf.ReadByte() } } // readTokenUntilNewline copies bytes from p.buf into p.currentToken. The first // byte considered is the byte already read (now in p.currentByte). The first // newline byte encountered is still copied into p.currentByte, but not into // p.currentToken. If recognizeEscapeSequence is true, two escape sequences are // recognized: '\\' tranlates into '\', and '\n' into a line-feed character. All // other escape sequences are invalid and cause an error. func (p *TextParser) readTokenUntilNewline(recognizeEscapeSequence bool) { p.currentToken.Reset() escaped := false for p.err == nil { if recognizeEscapeSequence && escaped { switch p.currentByte { case '\\': p.currentToken.WriteByte(p.currentByte) case 'n': p.currentToken.WriteByte('\n') default: p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte)) return } escaped = false } else { switch p.currentByte { case '\n': return case '\\': escaped = true default: p.currentToken.WriteByte(p.currentByte) } } p.currentByte, p.err = p.buf.ReadByte() } } // readTokenAsMetricName copies a metric name from p.buf into p.currentToken. // The first byte considered is the byte already read (now in p.currentByte). // The first byte not part of a metric name is still copied into p.currentByte, // but not into p.currentToken. func (p *TextParser) readTokenAsMetricName() { p.currentToken.Reset() if !isValidMetricNameStart(p.currentByte) { return } for { p.currentToken.WriteByte(p.currentByte) p.currentByte, p.err = p.buf.ReadByte() if p.err != nil || !isValidMetricNameContinuation(p.currentByte) { return } } } // readTokenAsLabelName copies a label name from p.buf into p.currentToken. // The first byte considered is the byte already read (now in p.currentByte). // The first byte not part of a label name is still copied into p.currentByte, // but not into p.currentToken. func (p *TextParser) readTokenAsLabelName() { p.currentToken.Reset() if !isValidLabelNameStart(p.currentByte) { return } for { p.currentToken.WriteByte(p.currentByte) p.currentByte, p.err = p.buf.ReadByte() if p.err != nil || !isValidLabelNameContinuation(p.currentByte) { return } } } // readTokenAsLabelValue copies a label value from p.buf into p.currentToken. // In contrast to the other 'readTokenAs...' functions, which start with the // last read byte in p.currentByte, this method ignores p.currentByte and starts // with reading a new byte from p.buf. The first byte not part of a label value // is still copied into p.currentByte, but not into p.currentToken. func (p *TextParser) readTokenAsLabelValue() { p.currentToken.Reset() escaped := false for { if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil { return } if escaped { switch p.currentByte { case '"', '\\': p.currentToken.WriteByte(p.currentByte) case 'n': p.currentToken.WriteByte('\n') default: p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte)) return } escaped = false continue } switch p.currentByte { case '"': return case '\n': p.parseError(fmt.Sprintf("label value %q contains unescaped new-line", p.currentToken.String())) return case '\\': escaped = true default: p.currentToken.WriteByte(p.currentByte) } } } func (p *TextParser) setOrCreateCurrentMF() { p.currentIsSummaryCount = false p.currentIsSummarySum = false p.currentIsHistogramCount = false p.currentIsHistogramSum = false name := p.currentToken.String() if p.currentMF = p.metricFamiliesByName[name]; p.currentMF != nil { return } // Try out if this is a _sum or _count for a summary/histogram. summaryName := summaryMetricName(name) if p.currentMF = p.metricFamiliesByName[summaryName]; p.currentMF != nil { if p.currentMF.GetType() == dto.MetricType_SUMMARY { if isCount(name) { p.currentIsSummaryCount = true } if isSum(name) { p.currentIsSummarySum = true } return } } histogramName := histogramMetricName(name) if p.currentMF = p.metricFamiliesByName[histogramName]; p.currentMF != nil { if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { if isCount(name) { p.currentIsHistogramCount = true } if isSum(name) { p.currentIsHistogramSum = true } return } } p.currentMF = &dto.MetricFamily{Name: proto.String(name)} p.metricFamiliesByName[name] = p.currentMF } func isValidLabelNameStart(b byte) bool { return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' } func isValidLabelNameContinuation(b byte) bool { return isValidLabelNameStart(b) || (b >= '0' && b <= '9') } func isValidMetricNameStart(b byte) bool { return isValidLabelNameStart(b) || b == ':' } func isValidMetricNameContinuation(b byte) bool { return isValidLabelNameContinuation(b) || b == ':' } func isBlankOrTab(b byte) bool { return b == ' ' || b == '\t' } func isCount(name string) bool { return len(name) > 6 && name[len(name)-6:] == "_count" } func isSum(name string) bool { return len(name) > 4 && name[len(name)-4:] == "_sum" } func isBucket(name string) bool { return len(name) > 7 && name[len(name)-7:] == "_bucket" } func summaryMetricName(name string) string { switch { case isCount(name): return name[:len(name)-6] case isSum(name): return name[:len(name)-4] default: return name } } func histogramMetricName(name string) string { switch { case isCount(name): return name[:len(name)-6] case isSum(name): return name[:len(name)-4] case isBucket(name): return name[:len(name)-7] default: return name } } ================================================ FILE: vendor/github.com/prometheus/common/expfmt/text_parse_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package expfmt import ( "math" "strings" "testing" "github.com/golang/protobuf/proto" dto "github.com/prometheus/client_model/go" ) func testTextParse(t testing.TB) { var scenarios = []struct { in string out []*dto.MetricFamily }{ // 0: Empty lines as input. { in: ` `, out: []*dto.MetricFamily{}, }, // 1: Minimal case. { in: ` minimal_metric 1.234 another_metric -3e3 103948 # Even that: no_labels{} 3 # HELP line for non-existing metric will be ignored. `, out: []*dto.MetricFamily{ &dto.MetricFamily{ Name: proto.String("minimal_metric"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(1.234), }, }, }, }, &dto.MetricFamily{ Name: proto.String("another_metric"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(-3e3), }, TimestampMs: proto.Int64(103948), }, }, }, &dto.MetricFamily{ Name: proto.String("no_labels"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(3), }, }, }, }, }, }, // 2: Counters & gauges, docstrings, various whitespace, escape sequences. { in: ` # A normal comment. # # TYPE name counter name{labelname="val1",basename="basevalue"} NaN name {labelname="val2",basename="base\"v\\al\nue"} 0.23 1234567890 # HELP name two-line\n doc str\\ing # HELP name2 doc str"ing 2 # TYPE name2 gauge name2{labelname="val2" ,basename = "basevalue2" } +Inf 54321 name2{ labelname = "val1" , }-Inf `, out: []*dto.MetricFamily{ &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val1"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(math.NaN()), }, }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("base\"v\\al\nue"), }, }, Counter: &dto.Counter{ Value: proto.Float64(.23), }, TimestampMs: proto.Int64(1234567890), }, }, }, &dto.MetricFamily{ Name: proto.String("name2"), Help: proto.String("doc str\"ing 2"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("basename"), Value: proto.String("basevalue2"), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(math.Inf(+1)), }, TimestampMs: proto.Int64(54321), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("labelname"), Value: proto.String("val1"), }, }, Gauge: &dto.Gauge{ Value: proto.Float64(math.Inf(-1)), }, }, }, }, }, }, // 3: The evil summary, mixed with other types and funny comments. { in: ` # TYPE my_summary summary my_summary{n1="val1",quantile="0.5"} 110 decoy -1 -2 my_summary{n1="val1",quantile="0.9"} 140 1 my_summary_count{n1="val1"} 42 # Latest timestamp wins in case of a summary. my_summary_sum{n1="val1"} 4711 2 fake_sum{n1="val1"} 2001 # TYPE another_summary summary another_summary_count{n2="val2",n1="val1"} 20 my_summary_count{n2="val2",n1="val1"} 5 5 another_summary{n1="val1",n2="val2",quantile=".3"} -1.2 my_summary_sum{n1="val2"} 08 15 my_summary{n1="val3", quantile="0.2"} 4711 my_summary{n1="val1",n2="val2",quantile="-12.34",} NaN # some # funny comments # HELP # HELP # HELP my_summary # HELP my_summary `, out: []*dto.MetricFamily{ &dto.MetricFamily{ Name: proto.String("fake_sum"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Untyped: &dto.Untyped{ Value: proto.Float64(2001), }, }, }, }, &dto.MetricFamily{ Name: proto.String("decoy"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Untyped: &dto.Untyped{ Value: proto.Float64(-1), }, TimestampMs: proto.Int64(-2), }, }, }, &dto.MetricFamily{ Name: proto.String("my_summary"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(42), SampleSum: proto.Float64(4711), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.5), Value: proto.Float64(110), }, &dto.Quantile{ Quantile: proto.Float64(0.9), Value: proto.Float64(140), }, }, }, TimestampMs: proto.Int64(2), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n2"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(5), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(-12.34), Value: proto.Float64(math.NaN()), }, }, }, TimestampMs: proto.Int64(5), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val2"), }, }, Summary: &dto.Summary{ SampleSum: proto.Float64(8), }, TimestampMs: proto.Int64(15), }, &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val3"), }, }, Summary: &dto.Summary{ Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.2), Value: proto.Float64(4711), }, }, }, }, }, }, &dto.MetricFamily{ Name: proto.String("another_summary"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Label: []*dto.LabelPair{ &dto.LabelPair{ Name: proto.String("n2"), Value: proto.String("val2"), }, &dto.LabelPair{ Name: proto.String("n1"), Value: proto.String("val1"), }, }, Summary: &dto.Summary{ SampleCount: proto.Uint64(20), Quantile: []*dto.Quantile{ &dto.Quantile{ Quantile: proto.Float64(0.3), Value: proto.Float64(-1.2), }, }, }, }, }, }, }, }, // 4: The histogram. { in: ` # HELP request_duration_microseconds The response latency. # TYPE request_duration_microseconds histogram request_duration_microseconds_bucket{le="100"} 123 request_duration_microseconds_bucket{le="120"} 412 request_duration_microseconds_bucket{le="144"} 592 request_duration_microseconds_bucket{le="172.8"} 1524 request_duration_microseconds_bucket{le="+Inf"} 2693 request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, out: []*dto.MetricFamily{ { Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ &dto.Metric{ Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ &dto.Bucket{ UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, &dto.Bucket{ UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, &dto.Bucket{ UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, &dto.Bucket{ UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, &dto.Bucket{ UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, }, }, }, }, }, }, }, } for i, scenario := range scenarios { out, err := parser.TextToMetricFamilies(strings.NewReader(scenario.in)) if err != nil { t.Errorf("%d. error: %s", i, err) continue } if expected, got := len(scenario.out), len(out); expected != got { t.Errorf( "%d. expected %d MetricFamilies, got %d", i, expected, got, ) } for _, expected := range scenario.out { got, ok := out[expected.GetName()] if !ok { t.Errorf( "%d. expected MetricFamily %q, found none", i, expected.GetName(), ) continue } if expected.String() != got.String() { t.Errorf( "%d. expected MetricFamily %s, got %s", i, expected, got, ) } } } } func TestTextParse(t *testing.T) { testTextParse(t) } func BenchmarkTextParse(b *testing.B) { for i := 0; i < b.N; i++ { testTextParse(b) } } func testTextParseError(t testing.TB) { var scenarios = []struct { in string err string }{ // 0: No new-line at end of input. { in: ` bla 3.14 blubber 42`, err: "text format parsing error in line 3: unexpected end of input stream", }, // 1: Invalid escape sequence in label value. { in: `metric{label="\t"} 3.14`, err: "text format parsing error in line 1: invalid escape sequence", }, // 2: Newline in label value. { in: ` metric{label="new line"} 3.14 `, err: `text format parsing error in line 2: label value "new" contains unescaped new-line`, }, // 3: { in: `metric{@="bla"} 3.14`, err: "text format parsing error in line 1: invalid label name for metric", }, // 4: { in: `metric{__name__="bla"} 3.14`, err: `text format parsing error in line 1: label name "__name__" is reserved`, }, // 5: { in: `metric{label+="bla"} 3.14`, err: "text format parsing error in line 1: expected '=' after label name", }, // 6: { in: `metric{label=bla} 3.14`, err: "text format parsing error in line 1: expected '\"' at start of label value", }, // 7: { in: ` # TYPE metric summary metric{quantile="bla"} 3.14 `, err: "text format parsing error in line 3: expected float as value for 'quantile' label", }, // 8: { in: `metric{label="bla"+} 3.14`, err: "text format parsing error in line 1: unexpected end of label value", }, // 9: { in: `metric{label="bla"} 3.14 2.72 `, err: "text format parsing error in line 1: expected integer as timestamp", }, // 10: { in: `metric{label="bla"} 3.14 2 3 `, err: "text format parsing error in line 1: spurious string after timestamp", }, // 11: { in: `metric{label="bla"} blubb `, err: "text format parsing error in line 1: expected float as value", }, // 12: { in: ` # HELP metric one # HELP metric two `, err: "text format parsing error in line 3: second HELP line for metric name", }, // 13: { in: ` # TYPE metric counter # TYPE metric untyped `, err: `text format parsing error in line 3: second TYPE line for metric name "metric", or TYPE reported after samples`, }, // 14: { in: ` metric 4.12 # TYPE metric counter `, err: `text format parsing error in line 3: second TYPE line for metric name "metric", or TYPE reported after samples`, }, // 14: { in: ` # TYPE metric bla `, err: "text format parsing error in line 2: unknown metric type", }, // 15: { in: ` # TYPE met-ric `, err: "text format parsing error in line 2: invalid metric name in comment", }, // 16: { in: `@invalidmetric{label="bla"} 3.14 2`, err: "text format parsing error in line 1: invalid metric name", }, // 17: { in: `{label="bla"} 3.14 2`, err: "text format parsing error in line 1: invalid metric name", }, // 18: { in: ` # TYPE metric histogram metric_bucket{le="bla"} 3.14 `, err: "text format parsing error in line 3: expected float as value for 'le' label", }, // 19: Invalid UTF-8 in label value. { in: "metric{l=\"\xbd\"} 3.14\n", err: "text format parsing error in line 1: invalid label value \"\\xbd\"", }, } for i, scenario := range scenarios { _, err := parser.TextToMetricFamilies(strings.NewReader(scenario.in)) if err == nil { t.Errorf("%d. expected error, got nil", i) continue } if expected, got := scenario.err, err.Error(); strings.Index(got, expected) != 0 { t.Errorf( "%d. expected error starting with %q, got %q", i, expected, got, ) } } } func TestTextParseError(t *testing.T) { testTextParseError(t) } func BenchmarkParseError(b *testing.B) { for i := 0; i < b.N; i++ { testTextParseError(b) } } ================================================ FILE: vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/README.txt ================================================ PACKAGE package goautoneg import "bitbucket.org/ww/goautoneg" HTTP Content-Type Autonegotiation. The functions in this package implement the behaviour specified in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html Copyright (c) 2011, Open Knowledge Foundation Ltd. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of the Open Knowledge Foundation Ltd. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. FUNCTIONS func Negotiate(header string, alternatives []string) (content_type string) Negotiate the most appropriate content_type given the accept header and a list of alternatives. func ParseAccept(header string) (accept []Accept) Parse an Accept Header string returning a sorted list of clauses TYPES type Accept struct { Type, SubType string Q float32 Params map[string]string } Structure to represent a clause in an HTTP Accept Header SUBDIRECTORIES .hg ================================================ FILE: vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go ================================================ /* HTTP Content-Type Autonegotiation. The functions in this package implement the behaviour specified in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html Copyright (c) 2011, Open Knowledge Foundation Ltd. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of the Open Knowledge Foundation Ltd. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package goautoneg import ( "sort" "strconv" "strings" ) // Structure to represent a clause in an HTTP Accept Header type Accept struct { Type, SubType string Q float64 Params map[string]string } // For internal use, so that we can use the sort interface type accept_slice []Accept func (accept accept_slice) Len() int { slice := []Accept(accept) return len(slice) } func (accept accept_slice) Less(i, j int) bool { slice := []Accept(accept) ai, aj := slice[i], slice[j] if ai.Q > aj.Q { return true } if ai.Type != "*" && aj.Type == "*" { return true } if ai.SubType != "*" && aj.SubType == "*" { return true } return false } func (accept accept_slice) Swap(i, j int) { slice := []Accept(accept) slice[i], slice[j] = slice[j], slice[i] } // Parse an Accept Header string returning a sorted list // of clauses func ParseAccept(header string) (accept []Accept) { parts := strings.Split(header, ",") accept = make([]Accept, 0, len(parts)) for _, part := range parts { part := strings.Trim(part, " ") a := Accept{} a.Params = make(map[string]string) a.Q = 1.0 mrp := strings.Split(part, ";") media_range := mrp[0] sp := strings.Split(media_range, "/") a.Type = strings.Trim(sp[0], " ") switch { case len(sp) == 1 && a.Type == "*": a.SubType = "*" case len(sp) == 2: a.SubType = strings.Trim(sp[1], " ") default: continue } if len(mrp) == 1 { accept = append(accept, a) continue } for _, param := range mrp[1:] { sp := strings.SplitN(param, "=", 2) if len(sp) != 2 { continue } token := strings.Trim(sp[0], " ") if token == "q" { a.Q, _ = strconv.ParseFloat(sp[1], 32) } else { a.Params[token] = strings.Trim(sp[1], " ") } } accept = append(accept, a) } slice := accept_slice(accept) sort.Sort(slice) return } // Negotiate the most appropriate content_type given the accept header // and a list of alternatives. func Negotiate(header string, alternatives []string) (content_type string) { asp := make([][]string, 0, len(alternatives)) for _, ctype := range alternatives { asp = append(asp, strings.SplitN(ctype, "/", 2)) } for _, clause := range ParseAccept(header) { for i, ctsp := range asp { if clause.Type == ctsp[0] && clause.SubType == ctsp[1] { content_type = alternatives[i] return } if clause.Type == ctsp[0] && clause.SubType == "*" { content_type = alternatives[i] return } if clause.Type == "*" && clause.SubType == "*" { content_type = alternatives[i] return } } } return } ================================================ FILE: vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg_test.go ================================================ package goautoneg import ( "testing" ) var chrome = "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5" func TestParseAccept(t *testing.T) { alternatives := []string{"text/html", "image/png"} content_type := Negotiate(chrome, alternatives) if content_type != "image/png" { t.Errorf("got %s expected image/png", content_type) } alternatives = []string{"text/html", "text/plain", "text/n3"} content_type = Negotiate(chrome, alternatives) if content_type != "text/html" { t.Errorf("got %s expected text/html", content_type) } alternatives = []string{"text/n3", "text/plain"} content_type = Negotiate(chrome, alternatives) if content_type != "text/plain" { t.Errorf("got %s expected text/plain", content_type) } alternatives = []string{"text/n3", "application/rdf+xml"} content_type = Negotiate(chrome, alternatives) if content_type != "text/n3" { t.Errorf("got %s expected text/n3", content_type) } } ================================================ FILE: vendor/github.com/prometheus/common/model/alert.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "time" ) type AlertStatus string const ( AlertFiring AlertStatus = "firing" AlertResolved AlertStatus = "resolved" ) // Alert is a generic representation of an alert in the Prometheus eco-system. type Alert struct { // Label value pairs for purpose of aggregation, matching, and disposition // dispatching. This must minimally include an "alertname" label. Labels LabelSet `json:"labels"` // Extra key/value information which does not define alert identity. Annotations LabelSet `json:"annotations"` // The known time range for this alert. Both ends are optional. StartsAt time.Time `json:"startsAt,omitempty"` EndsAt time.Time `json:"endsAt,omitempty"` GeneratorURL string `json:"generatorURL"` } // Name returns the name of the alert. It is equivalent to the "alertname" label. func (a *Alert) Name() string { return string(a.Labels[AlertNameLabel]) } // Fingerprint returns a unique hash for the alert. It is equivalent to // the fingerprint of the alert's label set. func (a *Alert) Fingerprint() Fingerprint { return a.Labels.Fingerprint() } func (a *Alert) String() string { s := fmt.Sprintf("%s[%s]", a.Name(), a.Fingerprint().String()[:7]) if a.Resolved() { return s + "[resolved]" } return s + "[active]" } // Resolved returns true iff the activity interval ended in the past. func (a *Alert) Resolved() bool { return a.ResolvedAt(time.Now()) } // ResolvedAt returns true off the activity interval ended before // the given timestamp. func (a *Alert) ResolvedAt(ts time.Time) bool { if a.EndsAt.IsZero() { return false } return !a.EndsAt.After(ts) } // Status returns the status of the alert. func (a *Alert) Status() AlertStatus { if a.Resolved() { return AlertResolved } return AlertFiring } // Validate checks whether the alert data is inconsistent. func (a *Alert) Validate() error { if a.StartsAt.IsZero() { return fmt.Errorf("start time missing") } if !a.EndsAt.IsZero() && a.EndsAt.Before(a.StartsAt) { return fmt.Errorf("start time must be before end time") } if err := a.Labels.Validate(); err != nil { return fmt.Errorf("invalid label set: %s", err) } if len(a.Labels) == 0 { return fmt.Errorf("at least one label pair required") } if err := a.Annotations.Validate(); err != nil { return fmt.Errorf("invalid annotations: %s", err) } return nil } // Alert is a list of alerts that can be sorted in chronological order. type Alerts []*Alert func (as Alerts) Len() int { return len(as) } func (as Alerts) Swap(i, j int) { as[i], as[j] = as[j], as[i] } func (as Alerts) Less(i, j int) bool { if as[i].StartsAt.Before(as[j].StartsAt) { return true } if as[i].EndsAt.Before(as[j].EndsAt) { return true } return as[i].Fingerprint() < as[j].Fingerprint() } // HasFiring returns true iff one of the alerts is not resolved. func (as Alerts) HasFiring() bool { for _, a := range as { if !a.Resolved() { return true } } return false } // Status returns StatusFiring iff at least one of the alerts is firing. func (as Alerts) Status() AlertStatus { if as.HasFiring() { return AlertFiring } return AlertResolved } ================================================ FILE: vendor/github.com/prometheus/common/model/alert_test.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "strings" "testing" "time" ) func TestAlertValidate(t *testing.T) { ts := time.Now() var cases = []struct { alert *Alert err string }{ { alert: &Alert{ Labels: LabelSet{"a": "b"}, StartsAt: ts, }, }, { alert: &Alert{ Labels: LabelSet{"a": "b"}, }, err: "start time missing", }, { alert: &Alert{ Labels: LabelSet{"a": "b"}, StartsAt: ts, EndsAt: ts, }, }, { alert: &Alert{ Labels: LabelSet{"a": "b"}, StartsAt: ts, EndsAt: ts.Add(1 * time.Minute), }, }, { alert: &Alert{ Labels: LabelSet{"a": "b"}, StartsAt: ts, EndsAt: ts.Add(-1 * time.Minute), }, err: "start time must be before end time", }, { alert: &Alert{ StartsAt: ts, }, err: "at least one label pair required", }, { alert: &Alert{ Labels: LabelSet{"a": "b", "!bad": "label"}, StartsAt: ts, }, err: "invalid label set: invalid name", }, { alert: &Alert{ Labels: LabelSet{"a": "b", "bad": "\xfflabel"}, StartsAt: ts, }, err: "invalid label set: invalid value", }, { alert: &Alert{ Labels: LabelSet{"a": "b"}, Annotations: LabelSet{"!bad": "label"}, StartsAt: ts, }, err: "invalid annotations: invalid name", }, { alert: &Alert{ Labels: LabelSet{"a": "b"}, Annotations: LabelSet{"bad": "\xfflabel"}, StartsAt: ts, }, err: "invalid annotations: invalid value", }, } for i, c := range cases { err := c.alert.Validate() if err == nil { if c.err == "" { continue } t.Errorf("%d. Expected error %q but got none", i, c.err) continue } if c.err == "" && err != nil { t.Errorf("%d. Expected no error but got %q", i, err) continue } if !strings.Contains(err.Error(), c.err) { t.Errorf("%d. Expected error to contain %q but got %q", i, c.err, err) } } } ================================================ FILE: vendor/github.com/prometheus/common/model/fingerprinting.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "strconv" ) // Fingerprint provides a hash-capable representation of a Metric. // For our purposes, FNV-1A 64-bit is used. type Fingerprint uint64 // FingerprintFromString transforms a string representation into a Fingerprint. func FingerprintFromString(s string) (Fingerprint, error) { num, err := strconv.ParseUint(s, 16, 64) return Fingerprint(num), err } // ParseFingerprint parses the input string into a fingerprint. func ParseFingerprint(s string) (Fingerprint, error) { num, err := strconv.ParseUint(s, 16, 64) if err != nil { return 0, err } return Fingerprint(num), nil } func (f Fingerprint) String() string { return fmt.Sprintf("%016x", uint64(f)) } // Fingerprints represents a collection of Fingerprint subject to a given // natural sorting scheme. It implements sort.Interface. type Fingerprints []Fingerprint // Len implements sort.Interface. func (f Fingerprints) Len() int { return len(f) } // Less implements sort.Interface. func (f Fingerprints) Less(i, j int) bool { return f[i] < f[j] } // Swap implements sort.Interface. func (f Fingerprints) Swap(i, j int) { f[i], f[j] = f[j], f[i] } // FingerprintSet is a set of Fingerprints. type FingerprintSet map[Fingerprint]struct{} // Equal returns true if both sets contain the same elements (and not more). func (s FingerprintSet) Equal(o FingerprintSet) bool { if len(s) != len(o) { return false } for k := range s { if _, ok := o[k]; !ok { return false } } return true } // Intersection returns the elements contained in both sets. func (s FingerprintSet) Intersection(o FingerprintSet) FingerprintSet { myLength, otherLength := len(s), len(o) if myLength == 0 || otherLength == 0 { return FingerprintSet{} } subSet := s superSet := o if otherLength < myLength { subSet = o superSet = s } out := FingerprintSet{} for k := range subSet { if _, ok := superSet[k]; ok { out[k] = struct{}{} } } return out } ================================================ FILE: vendor/github.com/prometheus/common/model/fnv.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model // Inline and byte-free variant of hash/fnv's fnv64a. const ( offset64 = 14695981039346656037 prime64 = 1099511628211 ) // hashNew initializies a new fnv64a hash value. func hashNew() uint64 { return offset64 } // hashAdd adds a string to a fnv64a hash value, returning the updated hash. func hashAdd(h uint64, s string) uint64 { for i := 0; i < len(s); i++ { h ^= uint64(s[i]) h *= prime64 } return h } // hashAddByte adds a byte to a fnv64a hash value, returning the updated hash. func hashAddByte(h uint64, b byte) uint64 { h ^= uint64(b) h *= prime64 return h } ================================================ FILE: vendor/github.com/prometheus/common/model/labels.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "fmt" "regexp" "strings" "unicode/utf8" ) const ( // AlertNameLabel is the name of the label containing the an alert's name. AlertNameLabel = "alertname" // ExportedLabelPrefix is the prefix to prepend to the label names present in // exported metrics if a label of the same name is added by the server. ExportedLabelPrefix = "exported_" // MetricNameLabel is the label name indicating the metric name of a // timeseries. MetricNameLabel = "__name__" // SchemeLabel is the name of the label that holds the scheme on which to // scrape a target. SchemeLabel = "__scheme__" // AddressLabel is the name of the label that holds the address of // a scrape target. AddressLabel = "__address__" // MetricsPathLabel is the name of the label that holds the path on which to // scrape a target. MetricsPathLabel = "__metrics_path__" // ReservedLabelPrefix is a prefix which is not legal in user-supplied // label names. ReservedLabelPrefix = "__" // MetaLabelPrefix is a prefix for labels that provide meta information. // Labels with this prefix are used for intermediate label processing and // will not be attached to time series. MetaLabelPrefix = "__meta_" // TmpLabelPrefix is a prefix for temporary labels as part of relabelling. // Labels with this prefix are used for intermediate label processing and // will not be attached to time series. This is reserved for use in // Prometheus configuration files by users. TmpLabelPrefix = "__tmp_" // ParamLabelPrefix is a prefix for labels that provide URL parameters // used to scrape a target. ParamLabelPrefix = "__param_" // JobLabel is the label name indicating the job from which a timeseries // was scraped. JobLabel = "job" // InstanceLabel is the label name used for the instance label. InstanceLabel = "instance" // BucketLabel is used for the label that defines the upper bound of a // bucket of a histogram ("le" -> "less or equal"). BucketLabel = "le" // QuantileLabel is used for the label that defines the quantile in a // summary. QuantileLabel = "quantile" ) // LabelNameRE is a regular expression matching valid label names. Note that the // IsValid method of LabelName performs the same check but faster than a match // with this regular expression. var LabelNameRE = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") // A LabelName is a key for a LabelSet or Metric. It has a value associated // therewith. type LabelName string // IsValid is true iff the label name matches the pattern of LabelNameRE. This // method, however, does not use LabelNameRE for the check but a much faster // hardcoded implementation. func (ln LabelName) IsValid() bool { if len(ln) == 0 { return false } for i, b := range ln { if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0)) { return false } } return true } // UnmarshalYAML implements the yaml.Unmarshaler interface. func (ln *LabelName) UnmarshalYAML(unmarshal func(interface{}) error) error { var s string if err := unmarshal(&s); err != nil { return err } if !LabelName(s).IsValid() { return fmt.Errorf("%q is not a valid label name", s) } *ln = LabelName(s) return nil } // UnmarshalJSON implements the json.Unmarshaler interface. func (ln *LabelName) UnmarshalJSON(b []byte) error { var s string if err := json.Unmarshal(b, &s); err != nil { return err } if !LabelName(s).IsValid() { return fmt.Errorf("%q is not a valid label name", s) } *ln = LabelName(s) return nil } // LabelNames is a sortable LabelName slice. In implements sort.Interface. type LabelNames []LabelName func (l LabelNames) Len() int { return len(l) } func (l LabelNames) Less(i, j int) bool { return l[i] < l[j] } func (l LabelNames) Swap(i, j int) { l[i], l[j] = l[j], l[i] } func (l LabelNames) String() string { labelStrings := make([]string, 0, len(l)) for _, label := range l { labelStrings = append(labelStrings, string(label)) } return strings.Join(labelStrings, ", ") } // A LabelValue is an associated value for a LabelName. type LabelValue string // IsValid returns true iff the string is a valid UTF8. func (lv LabelValue) IsValid() bool { return utf8.ValidString(string(lv)) } // LabelValues is a sortable LabelValue slice. It implements sort.Interface. type LabelValues []LabelValue func (l LabelValues) Len() int { return len(l) } func (l LabelValues) Less(i, j int) bool { return string(l[i]) < string(l[j]) } func (l LabelValues) Swap(i, j int) { l[i], l[j] = l[j], l[i] } // LabelPair pairs a name with a value. type LabelPair struct { Name LabelName Value LabelValue } // LabelPairs is a sortable slice of LabelPair pointers. It implements // sort.Interface. type LabelPairs []*LabelPair func (l LabelPairs) Len() int { return len(l) } func (l LabelPairs) Less(i, j int) bool { switch { case l[i].Name > l[j].Name: return false case l[i].Name < l[j].Name: return true case l[i].Value > l[j].Value: return false case l[i].Value < l[j].Value: return true default: return false } } func (l LabelPairs) Swap(i, j int) { l[i], l[j] = l[j], l[i] } ================================================ FILE: vendor/github.com/prometheus/common/model/labels_test.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "sort" "testing" ) func testLabelNames(t testing.TB) { var scenarios = []struct { in LabelNames out LabelNames }{ { in: LabelNames{"ZZZ", "zzz"}, out: LabelNames{"ZZZ", "zzz"}, }, { in: LabelNames{"aaa", "AAA"}, out: LabelNames{"AAA", "aaa"}, }, } for i, scenario := range scenarios { sort.Sort(scenario.in) for j, expected := range scenario.out { if expected != scenario.in[j] { t.Errorf("%d.%d expected %s, got %s", i, j, expected, scenario.in[j]) } } } } func TestLabelNames(t *testing.T) { testLabelNames(t) } func BenchmarkLabelNames(b *testing.B) { for i := 0; i < b.N; i++ { testLabelNames(b) } } func testLabelValues(t testing.TB) { var scenarios = []struct { in LabelValues out LabelValues }{ { in: LabelValues{"ZZZ", "zzz"}, out: LabelValues{"ZZZ", "zzz"}, }, { in: LabelValues{"aaa", "AAA"}, out: LabelValues{"AAA", "aaa"}, }, } for i, scenario := range scenarios { sort.Sort(scenario.in) for j, expected := range scenario.out { if expected != scenario.in[j] { t.Errorf("%d.%d expected %s, got %s", i, j, expected, scenario.in[j]) } } } } func TestLabelValues(t *testing.T) { testLabelValues(t) } func BenchmarkLabelValues(b *testing.B) { for i := 0; i < b.N; i++ { testLabelValues(b) } } func TestLabelNameIsValid(t *testing.T) { var scenarios = []struct { ln LabelName valid bool }{ { ln: "Avalid_23name", valid: true, }, { ln: "_Avalid_23name", valid: true, }, { ln: "1valid_23name", valid: false, }, { ln: "avalid_23name", valid: true, }, { ln: "Ava:lid_23name", valid: false, }, { ln: "a lid_23name", valid: false, }, { ln: ":leading_colon", valid: false, }, { ln: "colon:in:the:middle", valid: false, }, } for _, s := range scenarios { if s.ln.IsValid() != s.valid { t.Errorf("Expected %v for %q using IsValid method", s.valid, s.ln) } if LabelNameRE.MatchString(string(s.ln)) != s.valid { t.Errorf("Expected %v for %q using regexp match", s.valid, s.ln) } } } ================================================ FILE: vendor/github.com/prometheus/common/model/labelset.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "fmt" "sort" "strings" ) // A LabelSet is a collection of LabelName and LabelValue pairs. The LabelSet // may be fully-qualified down to the point where it may resolve to a single // Metric in the data store or not. All operations that occur within the realm // of a LabelSet can emit a vector of Metric entities to which the LabelSet may // match. type LabelSet map[LabelName]LabelValue // Validate checks whether all names and values in the label set // are valid. func (ls LabelSet) Validate() error { for ln, lv := range ls { if !ln.IsValid() { return fmt.Errorf("invalid name %q", ln) } if !lv.IsValid() { return fmt.Errorf("invalid value %q", lv) } } return nil } // Equal returns true iff both label sets have exactly the same key/value pairs. func (ls LabelSet) Equal(o LabelSet) bool { if len(ls) != len(o) { return false } for ln, lv := range ls { olv, ok := o[ln] if !ok { return false } if olv != lv { return false } } return true } // Before compares the metrics, using the following criteria: // // If m has fewer labels than o, it is before o. If it has more, it is not. // // If the number of labels is the same, the superset of all label names is // sorted alphanumerically. The first differing label pair found in that order // determines the outcome: If the label does not exist at all in m, then m is // before o, and vice versa. Otherwise the label value is compared // alphanumerically. // // If m and o are equal, the method returns false. func (ls LabelSet) Before(o LabelSet) bool { if len(ls) < len(o) { return true } if len(ls) > len(o) { return false } lns := make(LabelNames, 0, len(ls)+len(o)) for ln := range ls { lns = append(lns, ln) } for ln := range o { lns = append(lns, ln) } // It's probably not worth it to de-dup lns. sort.Sort(lns) for _, ln := range lns { mlv, ok := ls[ln] if !ok { return true } olv, ok := o[ln] if !ok { return false } if mlv < olv { return true } if mlv > olv { return false } } return false } // Clone returns a copy of the label set. func (ls LabelSet) Clone() LabelSet { lsn := make(LabelSet, len(ls)) for ln, lv := range ls { lsn[ln] = lv } return lsn } // Merge is a helper function to non-destructively merge two label sets. func (l LabelSet) Merge(other LabelSet) LabelSet { result := make(LabelSet, len(l)) for k, v := range l { result[k] = v } for k, v := range other { result[k] = v } return result } func (l LabelSet) String() string { lstrs := make([]string, 0, len(l)) for l, v := range l { lstrs = append(lstrs, fmt.Sprintf("%s=%q", l, v)) } sort.Strings(lstrs) return fmt.Sprintf("{%s}", strings.Join(lstrs, ", ")) } // Fingerprint returns the LabelSet's fingerprint. func (ls LabelSet) Fingerprint() Fingerprint { return labelSetToFingerprint(ls) } // FastFingerprint returns the LabelSet's Fingerprint calculated by a faster hashing // algorithm, which is, however, more susceptible to hash collisions. func (ls LabelSet) FastFingerprint() Fingerprint { return labelSetToFastFingerprint(ls) } // UnmarshalJSON implements the json.Unmarshaler interface. func (l *LabelSet) UnmarshalJSON(b []byte) error { var m map[LabelName]LabelValue if err := json.Unmarshal(b, &m); err != nil { return err } // encoding/json only unmarshals maps of the form map[string]T. It treats // LabelName as a string and does not call its UnmarshalJSON method. // Thus, we have to replicate the behavior here. for ln := range m { if !ln.IsValid() { return fmt.Errorf("%q is not a valid label name", ln) } } *l = LabelSet(m) return nil } ================================================ FILE: vendor/github.com/prometheus/common/model/metric.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "regexp" "sort" "strings" ) var ( separator = []byte{0} // MetricNameRE is a regular expression matching valid metric // names. Note that the IsValidMetricName function performs the same // check but faster than a match with this regular expression. MetricNameRE = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*$`) ) // A Metric is similar to a LabelSet, but the key difference is that a Metric is // a singleton and refers to one and only one stream of samples. type Metric LabelSet // Equal compares the metrics. func (m Metric) Equal(o Metric) bool { return LabelSet(m).Equal(LabelSet(o)) } // Before compares the metrics' underlying label sets. func (m Metric) Before(o Metric) bool { return LabelSet(m).Before(LabelSet(o)) } // Clone returns a copy of the Metric. func (m Metric) Clone() Metric { clone := make(Metric, len(m)) for k, v := range m { clone[k] = v } return clone } func (m Metric) String() string { metricName, hasName := m[MetricNameLabel] numLabels := len(m) - 1 if !hasName { numLabels = len(m) } labelStrings := make([]string, 0, numLabels) for label, value := range m { if label != MetricNameLabel { labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value)) } } switch numLabels { case 0: if hasName { return string(metricName) } return "{}" default: sort.Strings(labelStrings) return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ", ")) } } // Fingerprint returns a Metric's Fingerprint. func (m Metric) Fingerprint() Fingerprint { return LabelSet(m).Fingerprint() } // FastFingerprint returns a Metric's Fingerprint calculated by a faster hashing // algorithm, which is, however, more susceptible to hash collisions. func (m Metric) FastFingerprint() Fingerprint { return LabelSet(m).FastFingerprint() } // IsValidMetricName returns true iff name matches the pattern of MetricNameRE. // This function, however, does not use MetricNameRE for the check but a much // faster hardcoded implementation. func IsValidMetricName(n LabelValue) bool { if len(n) == 0 { return false } for i, b := range n { if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || b == ':' || (b >= '0' && b <= '9' && i > 0)) { return false } } return true } ================================================ FILE: vendor/github.com/prometheus/common/model/metric_test.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import "testing" func testMetric(t testing.TB) { var scenarios = []struct { input LabelSet fingerprint Fingerprint fastFingerprint Fingerprint }{ { input: LabelSet{}, fingerprint: 14695981039346656037, fastFingerprint: 14695981039346656037, }, { input: LabelSet{ "first_name": "electro", "occupation": "robot", "manufacturer": "westinghouse", }, fingerprint: 5911716720268894962, fastFingerprint: 11310079640881077873, }, { input: LabelSet{ "x": "y", }, fingerprint: 8241431561484471700, fastFingerprint: 13948396922932177635, }, { input: LabelSet{ "a": "bb", "b": "c", }, fingerprint: 3016285359649981711, fastFingerprint: 3198632812309449502, }, { input: LabelSet{ "a": "b", "bb": "c", }, fingerprint: 7122421792099404749, fastFingerprint: 5774953389407657638, }, } for i, scenario := range scenarios { input := Metric(scenario.input) if scenario.fingerprint != input.Fingerprint() { t.Errorf("%d. expected %d, got %d", i, scenario.fingerprint, input.Fingerprint()) } if scenario.fastFingerprint != input.FastFingerprint() { t.Errorf("%d. expected %d, got %d", i, scenario.fastFingerprint, input.FastFingerprint()) } } } func TestMetric(t *testing.T) { testMetric(t) } func BenchmarkMetric(b *testing.B) { for i := 0; i < b.N; i++ { testMetric(b) } } func TestMetricNameIsValid(t *testing.T) { var scenarios = []struct { mn LabelValue valid bool }{ { mn: "Avalid_23name", valid: true, }, { mn: "_Avalid_23name", valid: true, }, { mn: "1valid_23name", valid: false, }, { mn: "avalid_23name", valid: true, }, { mn: "Ava:lid_23name", valid: true, }, { mn: "a lid_23name", valid: false, }, { mn: ":leading_colon", valid: true, }, { mn: "colon:in:the:middle", valid: true, }, } for _, s := range scenarios { if IsValidMetricName(s.mn) != s.valid { t.Errorf("Expected %v for %q using IsValidMetricName function", s.valid, s.mn) } if MetricNameRE.MatchString(string(s.mn)) != s.valid { t.Errorf("Expected %v for %q using regexp matching", s.valid, s.mn) } } } ================================================ FILE: vendor/github.com/prometheus/common/model/model.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package model contains common data structures that are shared across // Prometheus components and libraries. package model ================================================ FILE: vendor/github.com/prometheus/common/model/signature.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "sort" ) // SeparatorByte is a byte that cannot occur in valid UTF-8 sequences and is // used to separate label names, label values, and other strings from each other // when calculating their combined hash value (aka signature aka fingerprint). const SeparatorByte byte = 255 var ( // cache the signature of an empty label set. emptyLabelSignature = hashNew() ) // LabelsToSignature returns a quasi-unique signature (i.e., fingerprint) for a // given label set. (Collisions are possible but unlikely if the number of label // sets the function is applied to is small.) func LabelsToSignature(labels map[string]string) uint64 { if len(labels) == 0 { return emptyLabelSignature } labelNames := make([]string, 0, len(labels)) for labelName := range labels { labelNames = append(labelNames, labelName) } sort.Strings(labelNames) sum := hashNew() for _, labelName := range labelNames { sum = hashAdd(sum, labelName) sum = hashAddByte(sum, SeparatorByte) sum = hashAdd(sum, labels[labelName]) sum = hashAddByte(sum, SeparatorByte) } return sum } // labelSetToFingerprint works exactly as LabelsToSignature but takes a LabelSet as // parameter (rather than a label map) and returns a Fingerprint. func labelSetToFingerprint(ls LabelSet) Fingerprint { if len(ls) == 0 { return Fingerprint(emptyLabelSignature) } labelNames := make(LabelNames, 0, len(ls)) for labelName := range ls { labelNames = append(labelNames, labelName) } sort.Sort(labelNames) sum := hashNew() for _, labelName := range labelNames { sum = hashAdd(sum, string(labelName)) sum = hashAddByte(sum, SeparatorByte) sum = hashAdd(sum, string(ls[labelName])) sum = hashAddByte(sum, SeparatorByte) } return Fingerprint(sum) } // labelSetToFastFingerprint works similar to labelSetToFingerprint but uses a // faster and less allocation-heavy hash function, which is more susceptible to // create hash collisions. Therefore, collision detection should be applied. func labelSetToFastFingerprint(ls LabelSet) Fingerprint { if len(ls) == 0 { return Fingerprint(emptyLabelSignature) } var result uint64 for labelName, labelValue := range ls { sum := hashNew() sum = hashAdd(sum, string(labelName)) sum = hashAddByte(sum, SeparatorByte) sum = hashAdd(sum, string(labelValue)) result ^= sum } return Fingerprint(result) } // SignatureForLabels works like LabelsToSignature but takes a Metric as // parameter (rather than a label map) and only includes the labels with the // specified LabelNames into the signature calculation. The labels passed in // will be sorted by this function. func SignatureForLabels(m Metric, labels ...LabelName) uint64 { if len(labels) == 0 { return emptyLabelSignature } sort.Sort(LabelNames(labels)) sum := hashNew() for _, label := range labels { sum = hashAdd(sum, string(label)) sum = hashAddByte(sum, SeparatorByte) sum = hashAdd(sum, string(m[label])) sum = hashAddByte(sum, SeparatorByte) } return sum } // SignatureWithoutLabels works like LabelsToSignature but takes a Metric as // parameter (rather than a label map) and excludes the labels with any of the // specified LabelNames from the signature calculation. func SignatureWithoutLabels(m Metric, labels map[LabelName]struct{}) uint64 { if len(m) == 0 { return emptyLabelSignature } labelNames := make(LabelNames, 0, len(m)) for labelName := range m { if _, exclude := labels[labelName]; !exclude { labelNames = append(labelNames, labelName) } } if len(labelNames) == 0 { return emptyLabelSignature } sort.Sort(labelNames) sum := hashNew() for _, labelName := range labelNames { sum = hashAdd(sum, string(labelName)) sum = hashAddByte(sum, SeparatorByte) sum = hashAdd(sum, string(m[labelName])) sum = hashAddByte(sum, SeparatorByte) } return sum } ================================================ FILE: vendor/github.com/prometheus/common/model/signature_test.go ================================================ // Copyright 2014 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "runtime" "sync" "testing" ) func TestLabelsToSignature(t *testing.T) { var scenarios = []struct { in map[string]string out uint64 }{ { in: map[string]string{}, out: 14695981039346656037, }, { in: map[string]string{"name": "garland, briggs", "fear": "love is not enough"}, out: 5799056148416392346, }, } for i, scenario := range scenarios { actual := LabelsToSignature(scenario.in) if actual != scenario.out { t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) } } } func TestMetricToFingerprint(t *testing.T) { var scenarios = []struct { in LabelSet out Fingerprint }{ { in: LabelSet{}, out: 14695981039346656037, }, { in: LabelSet{"name": "garland, briggs", "fear": "love is not enough"}, out: 5799056148416392346, }, } for i, scenario := range scenarios { actual := labelSetToFingerprint(scenario.in) if actual != scenario.out { t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) } } } func TestMetricToFastFingerprint(t *testing.T) { var scenarios = []struct { in LabelSet out Fingerprint }{ { in: LabelSet{}, out: 14695981039346656037, }, { in: LabelSet{"name": "garland, briggs", "fear": "love is not enough"}, out: 12952432476264840823, }, } for i, scenario := range scenarios { actual := labelSetToFastFingerprint(scenario.in) if actual != scenario.out { t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) } } } func TestSignatureForLabels(t *testing.T) { var scenarios = []struct { in Metric labels LabelNames out uint64 }{ { in: Metric{}, labels: nil, out: 14695981039346656037, }, { in: Metric{}, labels: LabelNames{"empty"}, out: 7187873163539638612, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: LabelNames{"empty"}, out: 7187873163539638612, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: LabelNames{"fear", "name"}, out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, labels: LabelNames{"fear", "name"}, out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: LabelNames{}, out: 14695981039346656037, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: nil, out: 14695981039346656037, }, } for i, scenario := range scenarios { actual := SignatureForLabels(scenario.in, scenario.labels...) if actual != scenario.out { t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) } } } func TestSignatureWithoutLabels(t *testing.T) { var scenarios = []struct { in Metric labels map[LabelName]struct{} out uint64 }{ { in: Metric{}, labels: nil, out: 14695981039346656037, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: map[LabelName]struct{}{"fear": struct{}{}, "name": struct{}{}}, out: 14695981039346656037, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, labels: map[LabelName]struct{}{"foo": struct{}{}}, out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: map[LabelName]struct{}{}, out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: nil, out: 5799056148416392346, }, } for i, scenario := range scenarios { actual := SignatureWithoutLabels(scenario.in, scenario.labels) if actual != scenario.out { t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) } } } func benchmarkLabelToSignature(b *testing.B, l map[string]string, e uint64) { for i := 0; i < b.N; i++ { if a := LabelsToSignature(l); a != e { b.Fatalf("expected signature of %d for %s, got %d", e, l, a) } } } func BenchmarkLabelToSignatureScalar(b *testing.B) { benchmarkLabelToSignature(b, nil, 14695981039346656037) } func BenchmarkLabelToSignatureSingle(b *testing.B) { benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value"}, 5146282821936882169) } func BenchmarkLabelToSignatureDouble(b *testing.B) { benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value"}, 3195800080984914717) } func BenchmarkLabelToSignatureTriple(b *testing.B) { benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 13843036195897128121) } func benchmarkMetricToFingerprint(b *testing.B, ls LabelSet, e Fingerprint) { for i := 0; i < b.N; i++ { if a := labelSetToFingerprint(ls); a != e { b.Fatalf("expected signature of %d for %s, got %d", e, ls, a) } } } func BenchmarkMetricToFingerprintScalar(b *testing.B) { benchmarkMetricToFingerprint(b, nil, 14695981039346656037) } func BenchmarkMetricToFingerprintSingle(b *testing.B) { benchmarkMetricToFingerprint(b, LabelSet{"first-label": "first-label-value"}, 5146282821936882169) } func BenchmarkMetricToFingerprintDouble(b *testing.B) { benchmarkMetricToFingerprint(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value"}, 3195800080984914717) } func BenchmarkMetricToFingerprintTriple(b *testing.B) { benchmarkMetricToFingerprint(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 13843036195897128121) } func benchmarkMetricToFastFingerprint(b *testing.B, ls LabelSet, e Fingerprint) { for i := 0; i < b.N; i++ { if a := labelSetToFastFingerprint(ls); a != e { b.Fatalf("expected signature of %d for %s, got %d", e, ls, a) } } } func BenchmarkMetricToFastFingerprintScalar(b *testing.B) { benchmarkMetricToFastFingerprint(b, nil, 14695981039346656037) } func BenchmarkMetricToFastFingerprintSingle(b *testing.B) { benchmarkMetricToFastFingerprint(b, LabelSet{"first-label": "first-label-value"}, 5147259542624943964) } func BenchmarkMetricToFastFingerprintDouble(b *testing.B) { benchmarkMetricToFastFingerprint(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value"}, 18269973311206963528) } func BenchmarkMetricToFastFingerprintTriple(b *testing.B) { benchmarkMetricToFastFingerprint(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676) } func BenchmarkEmptyLabelSignature(b *testing.B) { input := []map[string]string{nil, {}} var ms runtime.MemStats runtime.ReadMemStats(&ms) alloc := ms.Alloc for _, labels := range input { LabelsToSignature(labels) } runtime.ReadMemStats(&ms) if got := ms.Alloc; alloc != got { b.Fatal("expected LabelsToSignature with empty labels not to perform allocations") } } func benchmarkMetricToFastFingerprintConc(b *testing.B, ls LabelSet, e Fingerprint, concLevel int) { var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) for i := 0; i < concLevel; i++ { go func() { start.Wait() for j := b.N / concLevel; j >= 0; j-- { if a := labelSetToFastFingerprint(ls); a != e { b.Fatalf("expected signature of %d for %s, got %d", e, ls, a) } } end.Done() }() } b.ResetTimer() start.Done() end.Wait() } func BenchmarkMetricToFastFingerprintTripleConc1(b *testing.B) { benchmarkMetricToFastFingerprintConc(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 1) } func BenchmarkMetricToFastFingerprintTripleConc2(b *testing.B) { benchmarkMetricToFastFingerprintConc(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 2) } func BenchmarkMetricToFastFingerprintTripleConc4(b *testing.B) { benchmarkMetricToFastFingerprintConc(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 4) } func BenchmarkMetricToFastFingerprintTripleConc8(b *testing.B) { benchmarkMetricToFastFingerprintConc(b, LabelSet{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 8) } ================================================ FILE: vendor/github.com/prometheus/common/model/silence.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "fmt" "regexp" "time" ) // Matcher describes a matches the value of a given label. type Matcher struct { Name LabelName `json:"name"` Value string `json:"value"` IsRegex bool `json:"isRegex"` } func (m *Matcher) UnmarshalJSON(b []byte) error { type plain Matcher if err := json.Unmarshal(b, (*plain)(m)); err != nil { return err } if len(m.Name) == 0 { return fmt.Errorf("label name in matcher must not be empty") } if m.IsRegex { if _, err := regexp.Compile(m.Value); err != nil { return err } } return nil } // Validate returns true iff all fields of the matcher have valid values. func (m *Matcher) Validate() error { if !m.Name.IsValid() { return fmt.Errorf("invalid name %q", m.Name) } if m.IsRegex { if _, err := regexp.Compile(m.Value); err != nil { return fmt.Errorf("invalid regular expression %q", m.Value) } } else if !LabelValue(m.Value).IsValid() || len(m.Value) == 0 { return fmt.Errorf("invalid value %q", m.Value) } return nil } // Silence defines the representation of a silence definiton // in the Prometheus eco-system. type Silence struct { ID uint64 `json:"id,omitempty"` Matchers []*Matcher `json:"matchers"` StartsAt time.Time `json:"startsAt"` EndsAt time.Time `json:"endsAt"` CreatedAt time.Time `json:"createdAt,omitempty"` CreatedBy string `json:"createdBy"` Comment string `json:"comment,omitempty"` } // Validate returns true iff all fields of the silence have valid values. func (s *Silence) Validate() error { if len(s.Matchers) == 0 { return fmt.Errorf("at least one matcher required") } for _, m := range s.Matchers { if err := m.Validate(); err != nil { return fmt.Errorf("invalid matcher: %s", err) } } if s.StartsAt.IsZero() { return fmt.Errorf("start time missing") } if s.EndsAt.IsZero() { return fmt.Errorf("end time missing") } if s.EndsAt.Before(s.StartsAt) { return fmt.Errorf("start time must be before end time") } if s.CreatedBy == "" { return fmt.Errorf("creator information missing") } if s.Comment == "" { return fmt.Errorf("comment missing") } if s.CreatedAt.IsZero() { return fmt.Errorf("creation timestamp missing") } return nil } ================================================ FILE: vendor/github.com/prometheus/common/model/silence_test.go ================================================ // Copyright 2015 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "strings" "testing" "time" ) func TestMatcherValidate(t *testing.T) { var cases = []struct { matcher *Matcher err string }{ { matcher: &Matcher{ Name: "name", Value: "value", }, }, { matcher: &Matcher{ Name: "name", Value: "value", IsRegex: true, }, }, { matcher: &Matcher{ Name: "name!", Value: "value", }, err: "invalid name", }, { matcher: &Matcher{ Name: "", Value: "value", }, err: "invalid name", }, { matcher: &Matcher{ Name: "name", Value: "value\xff", }, err: "invalid value", }, { matcher: &Matcher{ Name: "name", Value: "", }, err: "invalid value", }, } for i, c := range cases { err := c.matcher.Validate() if err == nil { if c.err == "" { continue } t.Errorf("%d. Expected error %q but got none", i, c.err) continue } if c.err == "" && err != nil { t.Errorf("%d. Expected no error but got %q", i, err) continue } if !strings.Contains(err.Error(), c.err) { t.Errorf("%d. Expected error to contain %q but got %q", i, c.err, err) } } } func TestSilenceValidate(t *testing.T) { ts := time.Now() var cases = []struct { sil *Silence err string }{ { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, StartsAt: ts, EndsAt: ts, CreatedAt: ts, CreatedBy: "name", Comment: "comment", }, }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, {Name: "name", Value: "value"}, {Name: "name", Value: "value"}, {Name: "name", Value: "value", IsRegex: true}, }, StartsAt: ts, EndsAt: ts, CreatedAt: ts, CreatedBy: "name", Comment: "comment", }, }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, StartsAt: ts, EndsAt: ts.Add(-1 * time.Minute), CreatedAt: ts, CreatedBy: "name", Comment: "comment", }, err: "start time must be before end time", }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, StartsAt: ts, CreatedAt: ts, CreatedBy: "name", Comment: "comment", }, err: "end time missing", }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, EndsAt: ts, CreatedAt: ts, CreatedBy: "name", Comment: "comment", }, err: "start time missing", }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "!name", Value: "value"}, }, StartsAt: ts, EndsAt: ts, CreatedAt: ts, CreatedBy: "name", Comment: "comment", }, err: "invalid matcher", }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, StartsAt: ts, EndsAt: ts, CreatedAt: ts, CreatedBy: "name", }, err: "comment missing", }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, StartsAt: ts, EndsAt: ts, CreatedBy: "name", Comment: "comment", }, err: "creation timestamp missing", }, { sil: &Silence{ Matchers: []*Matcher{ {Name: "name", Value: "value"}, }, StartsAt: ts, EndsAt: ts, CreatedAt: ts, Comment: "comment", }, err: "creator information missing", }, } for i, c := range cases { err := c.sil.Validate() if err == nil { if c.err == "" { continue } t.Errorf("%d. Expected error %q but got none", i, c.err) continue } if c.err == "" && err != nil { t.Errorf("%d. Expected no error but got %q", i, err) continue } if !strings.Contains(err.Error(), c.err) { t.Errorf("%d. Expected error to contain %q but got %q", i, c.err, err) } } } ================================================ FILE: vendor/github.com/prometheus/common/model/time.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "math" "regexp" "strconv" "strings" "time" ) const ( // MinimumTick is the minimum supported time resolution. This has to be // at least time.Second in order for the code below to work. minimumTick = time.Millisecond // second is the Time duration equivalent to one second. second = int64(time.Second / minimumTick) // The number of nanoseconds per minimum tick. nanosPerTick = int64(minimumTick / time.Nanosecond) // Earliest is the earliest Time representable. Handy for // initializing a high watermark. Earliest = Time(math.MinInt64) // Latest is the latest Time representable. Handy for initializing // a low watermark. Latest = Time(math.MaxInt64) ) // Time is the number of milliseconds since the epoch // (1970-01-01 00:00 UTC) excluding leap seconds. type Time int64 // Interval describes and interval between two timestamps. type Interval struct { Start, End Time } // Now returns the current time as a Time. func Now() Time { return TimeFromUnixNano(time.Now().UnixNano()) } // TimeFromUnix returns the Time equivalent to the Unix Time t // provided in seconds. func TimeFromUnix(t int64) Time { return Time(t * second) } // TimeFromUnixNano returns the Time equivalent to the Unix Time // t provided in nanoseconds. func TimeFromUnixNano(t int64) Time { return Time(t / nanosPerTick) } // Equal reports whether two Times represent the same instant. func (t Time) Equal(o Time) bool { return t == o } // Before reports whether the Time t is before o. func (t Time) Before(o Time) bool { return t < o } // After reports whether the Time t is after o. func (t Time) After(o Time) bool { return t > o } // Add returns the Time t + d. func (t Time) Add(d time.Duration) Time { return t + Time(d/minimumTick) } // Sub returns the Duration t - o. func (t Time) Sub(o Time) time.Duration { return time.Duration(t-o) * minimumTick } // Time returns the time.Time representation of t. func (t Time) Time() time.Time { return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick) } // Unix returns t as a Unix time, the number of seconds elapsed // since January 1, 1970 UTC. func (t Time) Unix() int64 { return int64(t) / second } // UnixNano returns t as a Unix time, the number of nanoseconds elapsed // since January 1, 1970 UTC. func (t Time) UnixNano() int64 { return int64(t) * nanosPerTick } // The number of digits after the dot. var dotPrecision = int(math.Log10(float64(second))) // String returns a string representation of the Time. func (t Time) String() string { return strconv.FormatFloat(float64(t)/float64(second), 'f', -1, 64) } // MarshalJSON implements the json.Marshaler interface. func (t Time) MarshalJSON() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalJSON implements the json.Unmarshaler interface. func (t *Time) UnmarshalJSON(b []byte) error { p := strings.Split(string(b), ".") switch len(p) { case 1: v, err := strconv.ParseInt(string(p[0]), 10, 64) if err != nil { return err } *t = Time(v * second) case 2: v, err := strconv.ParseInt(string(p[0]), 10, 64) if err != nil { return err } v *= second prec := dotPrecision - len(p[1]) if prec < 0 { p[1] = p[1][:dotPrecision] } else if prec > 0 { p[1] = p[1] + strings.Repeat("0", prec) } va, err := strconv.ParseInt(p[1], 10, 32) if err != nil { return err } *t = Time(v + va) default: return fmt.Errorf("invalid time %q", string(b)) } return nil } // Duration wraps time.Duration. It is used to parse the custom duration format // from YAML. // This type should not propagate beyond the scope of input/output processing. type Duration time.Duration // Set implements pflag/flag.Value func (d *Duration) Set(s string) error { var err error *d, err = ParseDuration(s) return err } // Type implements pflag.Value func (d *Duration) Type() string { return "duration" } var durationRE = regexp.MustCompile("^([0-9]+)(y|w|d|h|m|s|ms)$") // ParseDuration parses a string into a time.Duration, assuming that a year // always has 365d, a week always has 7d, and a day always has 24h. func ParseDuration(durationStr string) (Duration, error) { matches := durationRE.FindStringSubmatch(durationStr) if len(matches) != 3 { return 0, fmt.Errorf("not a valid duration string: %q", durationStr) } var ( n, _ = strconv.Atoi(matches[1]) dur = time.Duration(n) * time.Millisecond ) switch unit := matches[2]; unit { case "y": dur *= 1000 * 60 * 60 * 24 * 365 case "w": dur *= 1000 * 60 * 60 * 24 * 7 case "d": dur *= 1000 * 60 * 60 * 24 case "h": dur *= 1000 * 60 * 60 case "m": dur *= 1000 * 60 case "s": dur *= 1000 case "ms": // Value already correct default: return 0, fmt.Errorf("invalid time unit in duration string: %q", unit) } return Duration(dur), nil } func (d Duration) String() string { var ( ms = int64(time.Duration(d) / time.Millisecond) unit = "ms" ) factors := map[string]int64{ "y": 1000 * 60 * 60 * 24 * 365, "w": 1000 * 60 * 60 * 24 * 7, "d": 1000 * 60 * 60 * 24, "h": 1000 * 60 * 60, "m": 1000 * 60, "s": 1000, "ms": 1, } switch int64(0) { case ms % factors["y"]: unit = "y" case ms % factors["w"]: unit = "w" case ms % factors["d"]: unit = "d" case ms % factors["h"]: unit = "h" case ms % factors["m"]: unit = "m" case ms % factors["s"]: unit = "s" } return fmt.Sprintf("%v%v", ms/factors[unit], unit) } // MarshalYAML implements the yaml.Marshaler interface. func (d Duration) MarshalYAML() (interface{}, error) { return d.String(), nil } // UnmarshalYAML implements the yaml.Unmarshaler interface. func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { var s string if err := unmarshal(&s); err != nil { return err } dur, err := ParseDuration(s) if err != nil { return err } *d = dur return nil } ================================================ FILE: vendor/github.com/prometheus/common/model/time_test.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "testing" "time" ) func TestComparators(t *testing.T) { t1a := TimeFromUnix(0) t1b := TimeFromUnix(0) t2 := TimeFromUnix(2*second - 1) if !t1a.Equal(t1b) { t.Fatalf("Expected %s to be equal to %s", t1a, t1b) } if t1a.Equal(t2) { t.Fatalf("Expected %s to not be equal to %s", t1a, t2) } if !t1a.Before(t2) { t.Fatalf("Expected %s to be before %s", t1a, t2) } if t1a.Before(t1b) { t.Fatalf("Expected %s to not be before %s", t1a, t1b) } if !t2.After(t1a) { t.Fatalf("Expected %s to be after %s", t2, t1a) } if t1b.After(t1a) { t.Fatalf("Expected %s to not be after %s", t1b, t1a) } } func TestTimeConversions(t *testing.T) { unixSecs := int64(1136239445) unixNsecs := int64(123456789) unixNano := unixSecs*1e9 + unixNsecs t1 := time.Unix(unixSecs, unixNsecs-unixNsecs%nanosPerTick) t2 := time.Unix(unixSecs, unixNsecs) ts := TimeFromUnixNano(unixNano) if !ts.Time().Equal(t1) { t.Fatalf("Expected %s, got %s", t1, ts.Time()) } // Test available precision. ts = TimeFromUnixNano(t2.UnixNano()) if !ts.Time().Equal(t1) { t.Fatalf("Expected %s, got %s", t1, ts.Time()) } if ts.UnixNano() != unixNano-unixNano%nanosPerTick { t.Fatalf("Expected %d, got %d", unixNano, ts.UnixNano()) } } func TestDuration(t *testing.T) { duration := time.Second + time.Minute + time.Hour goTime := time.Unix(1136239445, 0) ts := TimeFromUnix(goTime.Unix()) if !goTime.Add(duration).Equal(ts.Add(duration).Time()) { t.Fatalf("Expected %s to be equal to %s", goTime.Add(duration), ts.Add(duration)) } earlier := ts.Add(-duration) delta := ts.Sub(earlier) if delta != duration { t.Fatalf("Expected %s to be equal to %s", delta, duration) } } func TestParseDuration(t *testing.T) { var cases = []struct { in string out time.Duration }{ { in: "324ms", out: 324 * time.Millisecond, }, { in: "3s", out: 3 * time.Second, }, { in: "5m", out: 5 * time.Minute, }, { in: "1h", out: time.Hour, }, { in: "4d", out: 4 * 24 * time.Hour, }, { in: "3w", out: 3 * 7 * 24 * time.Hour, }, { in: "10y", out: 10 * 365 * 24 * time.Hour, }, } for _, c := range cases { d, err := ParseDuration(c.in) if err != nil { t.Errorf("Unexpected error on input %q", c.in) } if time.Duration(d) != c.out { t.Errorf("Expected %v but got %v", c.out, d) } if d.String() != c.in { t.Errorf("Expected duration string %q but got %q", c.in, d.String()) } } } ================================================ FILE: vendor/github.com/prometheus/common/model/value.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "fmt" "math" "sort" "strconv" "strings" ) var ( // ZeroSamplePair is the pseudo zero-value of SamplePair used to signal a // non-existing sample pair. It is a SamplePair with timestamp Earliest and // value 0.0. Note that the natural zero value of SamplePair has a timestamp // of 0, which is possible to appear in a real SamplePair and thus not // suitable to signal a non-existing SamplePair. ZeroSamplePair = SamplePair{Timestamp: Earliest} // ZeroSample is the pseudo zero-value of Sample used to signal a // non-existing sample. It is a Sample with timestamp Earliest, value 0.0, // and metric nil. Note that the natural zero value of Sample has a timestamp // of 0, which is possible to appear in a real Sample and thus not suitable // to signal a non-existing Sample. ZeroSample = Sample{Timestamp: Earliest} ) // A SampleValue is a representation of a value for a given sample at a given // time. type SampleValue float64 // MarshalJSON implements json.Marshaler. func (v SampleValue) MarshalJSON() ([]byte, error) { return json.Marshal(v.String()) } // UnmarshalJSON implements json.Unmarshaler. func (v *SampleValue) UnmarshalJSON(b []byte) error { if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { return fmt.Errorf("sample value must be a quoted string") } f, err := strconv.ParseFloat(string(b[1:len(b)-1]), 64) if err != nil { return err } *v = SampleValue(f) return nil } // Equal returns true if the value of v and o is equal or if both are NaN. Note // that v==o is false if both are NaN. If you want the conventional float // behavior, use == to compare two SampleValues. func (v SampleValue) Equal(o SampleValue) bool { if v == o { return true } return math.IsNaN(float64(v)) && math.IsNaN(float64(o)) } func (v SampleValue) String() string { return strconv.FormatFloat(float64(v), 'f', -1, 64) } // SamplePair pairs a SampleValue with a Timestamp. type SamplePair struct { Timestamp Time Value SampleValue } // MarshalJSON implements json.Marshaler. func (s SamplePair) MarshalJSON() ([]byte, error) { t, err := json.Marshal(s.Timestamp) if err != nil { return nil, err } v, err := json.Marshal(s.Value) if err != nil { return nil, err } return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil } // UnmarshalJSON implements json.Unmarshaler. func (s *SamplePair) UnmarshalJSON(b []byte) error { v := [...]json.Unmarshaler{&s.Timestamp, &s.Value} return json.Unmarshal(b, &v) } // Equal returns true if this SamplePair and o have equal Values and equal // Timestamps. The sematics of Value equality is defined by SampleValue.Equal. func (s *SamplePair) Equal(o *SamplePair) bool { return s == o || (s.Value.Equal(o.Value) && s.Timestamp.Equal(o.Timestamp)) } func (s SamplePair) String() string { return fmt.Sprintf("%s @[%s]", s.Value, s.Timestamp) } // Sample is a sample pair associated with a metric. type Sample struct { Metric Metric `json:"metric"` Value SampleValue `json:"value"` Timestamp Time `json:"timestamp"` } // Equal compares first the metrics, then the timestamp, then the value. The // sematics of value equality is defined by SampleValue.Equal. func (s *Sample) Equal(o *Sample) bool { if s == o { return true } if !s.Metric.Equal(o.Metric) { return false } if !s.Timestamp.Equal(o.Timestamp) { return false } return s.Value.Equal(o.Value) } func (s Sample) String() string { return fmt.Sprintf("%s => %s", s.Metric, SamplePair{ Timestamp: s.Timestamp, Value: s.Value, }) } // MarshalJSON implements json.Marshaler. func (s Sample) MarshalJSON() ([]byte, error) { v := struct { Metric Metric `json:"metric"` Value SamplePair `json:"value"` }{ Metric: s.Metric, Value: SamplePair{ Timestamp: s.Timestamp, Value: s.Value, }, } return json.Marshal(&v) } // UnmarshalJSON implements json.Unmarshaler. func (s *Sample) UnmarshalJSON(b []byte) error { v := struct { Metric Metric `json:"metric"` Value SamplePair `json:"value"` }{ Metric: s.Metric, Value: SamplePair{ Timestamp: s.Timestamp, Value: s.Value, }, } if err := json.Unmarshal(b, &v); err != nil { return err } s.Metric = v.Metric s.Timestamp = v.Value.Timestamp s.Value = v.Value.Value return nil } // Samples is a sortable Sample slice. It implements sort.Interface. type Samples []*Sample func (s Samples) Len() int { return len(s) } // Less compares first the metrics, then the timestamp. func (s Samples) Less(i, j int) bool { switch { case s[i].Metric.Before(s[j].Metric): return true case s[j].Metric.Before(s[i].Metric): return false case s[i].Timestamp.Before(s[j].Timestamp): return true default: return false } } func (s Samples) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // Equal compares two sets of samples and returns true if they are equal. func (s Samples) Equal(o Samples) bool { if len(s) != len(o) { return false } for i, sample := range s { if !sample.Equal(o[i]) { return false } } return true } // SampleStream is a stream of Values belonging to an attached COWMetric. type SampleStream struct { Metric Metric `json:"metric"` Values []SamplePair `json:"values"` } func (ss SampleStream) String() string { vals := make([]string, len(ss.Values)) for i, v := range ss.Values { vals[i] = v.String() } return fmt.Sprintf("%s =>\n%s", ss.Metric, strings.Join(vals, "\n")) } // Value is a generic interface for values resulting from a query evaluation. type Value interface { Type() ValueType String() string } func (Matrix) Type() ValueType { return ValMatrix } func (Vector) Type() ValueType { return ValVector } func (*Scalar) Type() ValueType { return ValScalar } func (*String) Type() ValueType { return ValString } type ValueType int const ( ValNone ValueType = iota ValScalar ValVector ValMatrix ValString ) // MarshalJSON implements json.Marshaler. func (et ValueType) MarshalJSON() ([]byte, error) { return json.Marshal(et.String()) } func (et *ValueType) UnmarshalJSON(b []byte) error { var s string if err := json.Unmarshal(b, &s); err != nil { return err } switch s { case "": *et = ValNone case "scalar": *et = ValScalar case "vector": *et = ValVector case "matrix": *et = ValMatrix case "string": *et = ValString default: return fmt.Errorf("unknown value type %q", s) } return nil } func (e ValueType) String() string { switch e { case ValNone: return "" case ValScalar: return "scalar" case ValVector: return "vector" case ValMatrix: return "matrix" case ValString: return "string" } panic("ValueType.String: unhandled value type") } // Scalar is a scalar value evaluated at the set timestamp. type Scalar struct { Value SampleValue `json:"value"` Timestamp Time `json:"timestamp"` } func (s Scalar) String() string { return fmt.Sprintf("scalar: %v @[%v]", s.Value, s.Timestamp) } // MarshalJSON implements json.Marshaler. func (s Scalar) MarshalJSON() ([]byte, error) { v := strconv.FormatFloat(float64(s.Value), 'f', -1, 64) return json.Marshal([...]interface{}{s.Timestamp, string(v)}) } // UnmarshalJSON implements json.Unmarshaler. func (s *Scalar) UnmarshalJSON(b []byte) error { var f string v := [...]interface{}{&s.Timestamp, &f} if err := json.Unmarshal(b, &v); err != nil { return err } value, err := strconv.ParseFloat(f, 64) if err != nil { return fmt.Errorf("error parsing sample value: %s", err) } s.Value = SampleValue(value) return nil } // String is a string value evaluated at the set timestamp. type String struct { Value string `json:"value"` Timestamp Time `json:"timestamp"` } func (s *String) String() string { return s.Value } // MarshalJSON implements json.Marshaler. func (s String) MarshalJSON() ([]byte, error) { return json.Marshal([]interface{}{s.Timestamp, s.Value}) } // UnmarshalJSON implements json.Unmarshaler. func (s *String) UnmarshalJSON(b []byte) error { v := [...]interface{}{&s.Timestamp, &s.Value} return json.Unmarshal(b, &v) } // Vector is basically only an alias for Samples, but the // contract is that in a Vector, all Samples have the same timestamp. type Vector []*Sample func (vec Vector) String() string { entries := make([]string, len(vec)) for i, s := range vec { entries[i] = s.String() } return strings.Join(entries, "\n") } func (vec Vector) Len() int { return len(vec) } func (vec Vector) Swap(i, j int) { vec[i], vec[j] = vec[j], vec[i] } // Less compares first the metrics, then the timestamp. func (vec Vector) Less(i, j int) bool { switch { case vec[i].Metric.Before(vec[j].Metric): return true case vec[j].Metric.Before(vec[i].Metric): return false case vec[i].Timestamp.Before(vec[j].Timestamp): return true default: return false } } // Equal compares two sets of samples and returns true if they are equal. func (vec Vector) Equal(o Vector) bool { if len(vec) != len(o) { return false } for i, sample := range vec { if !sample.Equal(o[i]) { return false } } return true } // Matrix is a list of time series. type Matrix []*SampleStream func (m Matrix) Len() int { return len(m) } func (m Matrix) Less(i, j int) bool { return m[i].Metric.Before(m[j].Metric) } func (m Matrix) Swap(i, j int) { m[i], m[j] = m[j], m[i] } func (mat Matrix) String() string { matCp := make(Matrix, len(mat)) copy(matCp, mat) sort.Sort(matCp) strs := make([]string, len(matCp)) for i, ss := range matCp { strs[i] = ss.String() } return strings.Join(strs, "\n") } ================================================ FILE: vendor/github.com/prometheus/common/model/value_test.go ================================================ // Copyright 2013 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "math" "reflect" "sort" "testing" ) func TestEqualValues(t *testing.T) { tests := map[string]struct { in1, in2 SampleValue want bool }{ "equal floats": { in1: 3.14, in2: 3.14, want: true, }, "unequal floats": { in1: 3.14, in2: 3.1415, want: false, }, "positive inifinities": { in1: SampleValue(math.Inf(+1)), in2: SampleValue(math.Inf(+1)), want: true, }, "negative inifinities": { in1: SampleValue(math.Inf(-1)), in2: SampleValue(math.Inf(-1)), want: true, }, "different inifinities": { in1: SampleValue(math.Inf(+1)), in2: SampleValue(math.Inf(-1)), want: false, }, "number and infinity": { in1: 42, in2: SampleValue(math.Inf(+1)), want: false, }, "number and NaN": { in1: 42, in2: SampleValue(math.NaN()), want: false, }, "NaNs": { in1: SampleValue(math.NaN()), in2: SampleValue(math.NaN()), want: true, // !!! }, } for name, test := range tests { got := test.in1.Equal(test.in2) if got != test.want { t.Errorf("Comparing %s, %f and %f: got %t, want %t", name, test.in1, test.in2, got, test.want) } } } func TestEqualSamples(t *testing.T) { testSample := &Sample{} tests := map[string]struct { in1, in2 *Sample want bool }{ "equal pointers": { in1: testSample, in2: testSample, want: true, }, "different metrics": { in1: &Sample{Metric: Metric{"foo": "bar"}}, in2: &Sample{Metric: Metric{"foo": "biz"}}, want: false, }, "different timestamp": { in1: &Sample{Timestamp: 0}, in2: &Sample{Timestamp: 1}, want: false, }, "different value": { in1: &Sample{Value: 0}, in2: &Sample{Value: 1}, want: false, }, "equal samples": { in1: &Sample{ Metric: Metric{"foo": "bar"}, Timestamp: 0, Value: 1, }, in2: &Sample{ Metric: Metric{"foo": "bar"}, Timestamp: 0, Value: 1, }, want: true, }, } for name, test := range tests { got := test.in1.Equal(test.in2) if got != test.want { t.Errorf("Comparing %s, %v and %v: got %t, want %t", name, test.in1, test.in2, got, test.want) } } } func TestSamplePairJSON(t *testing.T) { input := []struct { plain string value SamplePair }{ { plain: `[1234.567,"123.1"]`, value: SamplePair{ Value: 123.1, Timestamp: 1234567, }, }, } for _, test := range input { b, err := json.Marshal(test.value) if err != nil { t.Error(err) continue } if string(b) != test.plain { t.Errorf("encoding error: expected %q, got %q", test.plain, b) continue } var sp SamplePair err = json.Unmarshal(b, &sp) if err != nil { t.Error(err) continue } if sp != test.value { t.Errorf("decoding error: expected %v, got %v", test.value, sp) } } } func TestSampleJSON(t *testing.T) { input := []struct { plain string value Sample }{ { plain: `{"metric":{"__name__":"test_metric"},"value":[1234.567,"123.1"]}`, value: Sample{ Metric: Metric{ MetricNameLabel: "test_metric", }, Value: 123.1, Timestamp: 1234567, }, }, } for _, test := range input { b, err := json.Marshal(test.value) if err != nil { t.Error(err) continue } if string(b) != test.plain { t.Errorf("encoding error: expected %q, got %q", test.plain, b) continue } var sv Sample err = json.Unmarshal(b, &sv) if err != nil { t.Error(err) continue } if !reflect.DeepEqual(sv, test.value) { t.Errorf("decoding error: expected %v, got %v", test.value, sv) } } } func TestVectorJSON(t *testing.T) { input := []struct { plain string value Vector }{ { plain: `[]`, value: Vector{}, }, { plain: `[{"metric":{"__name__":"test_metric"},"value":[1234.567,"123.1"]}]`, value: Vector{&Sample{ Metric: Metric{ MetricNameLabel: "test_metric", }, Value: 123.1, Timestamp: 1234567, }}, }, { plain: `[{"metric":{"__name__":"test_metric"},"value":[1234.567,"123.1"]},{"metric":{"foo":"bar"},"value":[1.234,"+Inf"]}]`, value: Vector{ &Sample{ Metric: Metric{ MetricNameLabel: "test_metric", }, Value: 123.1, Timestamp: 1234567, }, &Sample{ Metric: Metric{ "foo": "bar", }, Value: SampleValue(math.Inf(1)), Timestamp: 1234, }, }, }, } for _, test := range input { b, err := json.Marshal(test.value) if err != nil { t.Error(err) continue } if string(b) != test.plain { t.Errorf("encoding error: expected %q, got %q", test.plain, b) continue } var vec Vector err = json.Unmarshal(b, &vec) if err != nil { t.Error(err) continue } if !reflect.DeepEqual(vec, test.value) { t.Errorf("decoding error: expected %v, got %v", test.value, vec) } } } func TestScalarJSON(t *testing.T) { input := []struct { plain string value Scalar }{ { plain: `[123.456,"456"]`, value: Scalar{ Timestamp: 123456, Value: 456, }, }, { plain: `[123123.456,"+Inf"]`, value: Scalar{ Timestamp: 123123456, Value: SampleValue(math.Inf(1)), }, }, { plain: `[123123.456,"-Inf"]`, value: Scalar{ Timestamp: 123123456, Value: SampleValue(math.Inf(-1)), }, }, } for _, test := range input { b, err := json.Marshal(test.value) if err != nil { t.Error(err) continue } if string(b) != test.plain { t.Errorf("encoding error: expected %q, got %q", test.plain, b) continue } var sv Scalar err = json.Unmarshal(b, &sv) if err != nil { t.Error(err) continue } if sv != test.value { t.Errorf("decoding error: expected %v, got %v", test.value, sv) } } } func TestStringJSON(t *testing.T) { input := []struct { plain string value String }{ { plain: `[123.456,"test"]`, value: String{ Timestamp: 123456, Value: "test", }, }, { plain: `[123123.456,"台北"]`, value: String{ Timestamp: 123123456, Value: "台北", }, }, } for _, test := range input { b, err := json.Marshal(test.value) if err != nil { t.Error(err) continue } if string(b) != test.plain { t.Errorf("encoding error: expected %q, got %q", test.plain, b) continue } var sv String err = json.Unmarshal(b, &sv) if err != nil { t.Error(err) continue } if sv != test.value { t.Errorf("decoding error: expected %v, got %v", test.value, sv) } } } func TestVectorSort(t *testing.T) { input := Vector{ &Sample{ Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 1, }, &Sample{ Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 2, }, &Sample{ Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 1, }, &Sample{ Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 2, }, &Sample{ Metric: Metric{ MetricNameLabel: "B", }, Timestamp: 1, }, &Sample{ Metric: Metric{ MetricNameLabel: "B", }, Timestamp: 2, }, } expected := Vector{ &Sample{ Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 1, }, &Sample{ Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 2, }, &Sample{ Metric: Metric{ MetricNameLabel: "B", }, Timestamp: 1, }, &Sample{ Metric: Metric{ MetricNameLabel: "B", }, Timestamp: 2, }, &Sample{ Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 1, }, &Sample{ Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 2, }, } sort.Sort(input) for i, actual := range input { actualFp := actual.Metric.Fingerprint() expectedFp := expected[i].Metric.Fingerprint() if actualFp != expectedFp { t.Fatalf("%d. Incorrect fingerprint. Got %s; want %s", i, actualFp.String(), expectedFp.String()) } if actual.Timestamp != expected[i].Timestamp { t.Fatalf("%d. Incorrect timestamp. Got %s; want %s", i, actual.Timestamp, expected[i].Timestamp) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/.travis.yml ================================================ sudo: false language: go go: - 1.7.6 - 1.8.3 ================================================ FILE: vendor/github.com/prometheus/procfs/CONTRIBUTING.md ================================================ # Contributing Prometheus uses GitHub to manage reviews of pull requests. * If you have a trivial fix or improvement, go ahead and create a pull request, addressing (with `@...`) the maintainer of this repository (see [MAINTAINERS.md](MAINTAINERS.md)) in the description of the pull request. * If you plan to do something more involved, first discuss your ideas on our [mailing list](https://groups.google.com/forum/?fromgroups#!forum/prometheus-developers). This will avoid unnecessary work and surely give you and us a good deal of inspiration. * Relevant coding style guidelines are the [Go Code Review Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) and the _Formatting and style_ section of Peter Bourgon's [Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/#formatting-and-style). ================================================ FILE: vendor/github.com/prometheus/procfs/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: vendor/github.com/prometheus/procfs/MAINTAINERS.md ================================================ * Tobias Schmidt ================================================ FILE: vendor/github.com/prometheus/procfs/Makefile ================================================ ci: fmt lint test fmt: ! gofmt -l *.go | read nothing go vet lint: go get github.com/golang/lint/golint golint *.go test: sysfs/fixtures/.unpacked go test -v ./... sysfs/fixtures/.unpacked: sysfs/fixtures.ttar ./ttar -C sysfs -x -f sysfs/fixtures.ttar touch $@ .PHONY: fmt lint test ci ================================================ FILE: vendor/github.com/prometheus/procfs/NOTICE ================================================ procfs provides functions to retrieve system, kernel and process metrics from the pseudo-filesystem proc. Copyright 2014-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). ================================================ FILE: vendor/github.com/prometheus/procfs/README.md ================================================ # procfs This procfs package provides functions to retrieve system, kernel and process metrics from the pseudo-filesystem proc. *WARNING*: This package is a work in progress. Its API may still break in backwards-incompatible ways without warnings. Use it at your own risk. [![GoDoc](https://godoc.org/github.com/prometheus/procfs?status.png)](https://godoc.org/github.com/prometheus/procfs) [![Build Status](https://travis-ci.org/prometheus/procfs.svg?branch=master)](https://travis-ci.org/prometheus/procfs) [![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/procfs)](https://goreportcard.com/report/github.com/prometheus/procfs) ================================================ FILE: vendor/github.com/prometheus/procfs/buddyinfo.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package procfs import ( "bufio" "fmt" "io" "os" "strconv" "strings" ) // A BuddyInfo is the details parsed from /proc/buddyinfo. // The data is comprised of an array of free fragments of each size. // The sizes are 2^n*PAGE_SIZE, where n is the array index. type BuddyInfo struct { Node string Zone string Sizes []float64 } // NewBuddyInfo reads the buddyinfo statistics. func NewBuddyInfo() ([]BuddyInfo, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return nil, err } return fs.NewBuddyInfo() } // NewBuddyInfo reads the buddyinfo statistics from the specified `proc` filesystem. func (fs FS) NewBuddyInfo() ([]BuddyInfo, error) { file, err := os.Open(fs.Path("buddyinfo")) if err != nil { return nil, err } defer file.Close() return parseBuddyInfo(file) } func parseBuddyInfo(r io.Reader) ([]BuddyInfo, error) { var ( buddyInfo = []BuddyInfo{} scanner = bufio.NewScanner(r) bucketCount = -1 ) for scanner.Scan() { var err error line := scanner.Text() parts := strings.Fields(string(line)) if len(parts) < 4 { return nil, fmt.Errorf("invalid number of fields when parsing buddyinfo") } node := strings.TrimRight(parts[1], ",") zone := strings.TrimRight(parts[3], ",") arraySize := len(parts[4:]) if bucketCount == -1 { bucketCount = arraySize } else { if bucketCount != arraySize { return nil, fmt.Errorf("mismatch in number of buddyinfo buckets, previous count %d, new count %d", bucketCount, arraySize) } } sizes := make([]float64, arraySize) for i := 0; i < arraySize; i++ { sizes[i], err = strconv.ParseFloat(parts[i+4], 64) if err != nil { return nil, fmt.Errorf("invalid value in buddyinfo: %s", err) } } buddyInfo = append(buddyInfo, BuddyInfo{node, zone, sizes}) } return buddyInfo, scanner.Err() } ================================================ FILE: vendor/github.com/prometheus/procfs/buddyinfo_test.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package procfs import ( "strings" "testing" ) func TestBuddyInfo(t *testing.T) { buddyInfo, err := FS("fixtures/buddyinfo/valid").NewBuddyInfo() if err != nil { t.Fatal(err) } if want, got := "DMA", buddyInfo[0].Zone; want != got { t.Errorf("want Node 0, Zone %s, got %s", want, got) } if want, got := "Normal", buddyInfo[2].Zone; want != got { t.Errorf("want Node 0, Zone %s, got %s", want, got) } if want, got := 4381.0, buddyInfo[2].Sizes[0]; want != got { t.Errorf("want Node 0, Zone Normal %f, got %f", want, got) } if want, got := 572.0, buddyInfo[1].Sizes[1]; want != got { t.Errorf("want Node 0, Zone DMA32 %f, got %f", want, got) } } func TestBuddyInfoShort(t *testing.T) { _, err := FS("fixtures/buddyinfo/short").NewBuddyInfo() if err == nil { t.Errorf("expected error, but none occurred") } if want, got := "invalid number of fields when parsing buddyinfo", err.Error(); want != got { t.Errorf("wrong error returned, wanted %q, got %q", want, got) } } func TestBuddyInfoSizeMismatch(t *testing.T) { _, err := FS("fixtures/buddyinfo/sizemismatch").NewBuddyInfo() if err == nil { t.Errorf("expected error, but none occurred") } if want, got := "mismatch in number of buddyinfo buckets", err.Error(); !strings.HasPrefix(got, want) { t.Errorf("wrong error returned, wanted prefix %q, got %q", want, got) } } ================================================ FILE: vendor/github.com/prometheus/procfs/doc.go ================================================ // Copyright 2014 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package procfs provides functions to retrieve system, kernel and process // metrics from the pseudo-filesystem proc. // // Example: // // package main // // import ( // "fmt" // "log" // // "github.com/prometheus/procfs" // ) // // func main() { // p, err := procfs.Self() // if err != nil { // log.Fatalf("could not get process: %s", err) // } // // stat, err := p.NewStat() // if err != nil { // log.Fatalf("could not get process stat: %s", err) // } // // fmt.Printf("command: %s\n", stat.Comm) // fmt.Printf("cpu time: %fs\n", stat.CPUTime()) // fmt.Printf("vsize: %dB\n", stat.VirtualMemory()) // fmt.Printf("rss: %dB\n", stat.ResidentMemory()) // } // package procfs ================================================ FILE: vendor/github.com/prometheus/procfs/fs.go ================================================ package procfs import ( "fmt" "os" "path" "github.com/prometheus/procfs/xfs" ) // FS represents the pseudo-filesystem proc, which provides an interface to // kernel data structures. type FS string // DefaultMountPoint is the common mount point of the proc filesystem. const DefaultMountPoint = "/proc" // NewFS returns a new FS mounted under the given mountPoint. It will error // if the mount point can't be read. func NewFS(mountPoint string) (FS, error) { info, err := os.Stat(mountPoint) if err != nil { return "", fmt.Errorf("could not read %s: %s", mountPoint, err) } if !info.IsDir() { return "", fmt.Errorf("mount point %s is not a directory", mountPoint) } return FS(mountPoint), nil } // Path returns the path of the given subsystem relative to the procfs root. func (fs FS) Path(p ...string) string { return path.Join(append([]string{string(fs)}, p...)...) } // XFSStats retrieves XFS filesystem runtime statistics. func (fs FS) XFSStats() (*xfs.Stats, error) { f, err := os.Open(fs.Path("fs/xfs/stat")) if err != nil { return nil, err } defer f.Close() return xfs.ParseStats(f) } ================================================ FILE: vendor/github.com/prometheus/procfs/fs_test.go ================================================ package procfs import "testing" func TestNewFS(t *testing.T) { if _, err := NewFS("foobar"); err == nil { t.Error("want NewFS to fail for non-existing mount point") } if _, err := NewFS("procfs.go"); err == nil { t.Error("want NewFS to fail if mount point is not a directory") } } func TestFSXFSStats(t *testing.T) { stats, err := FS("fixtures").XFSStats() if err != nil { t.Fatalf("failed to parse XFS stats: %v", err) } // Very lightweight test just to sanity check the path used // to open XFS stats. Heavier tests in package xfs. if want, got := uint32(92447), stats.ExtentAllocation.ExtentsAllocated; want != got { t.Errorf("unexpected extents allocated:\nwant: %d\nhave: %d", want, got) } } ================================================ FILE: vendor/github.com/prometheus/procfs/ipvs.go ================================================ package procfs import ( "bufio" "encoding/hex" "errors" "fmt" "io" "io/ioutil" "net" "os" "strconv" "strings" ) // IPVSStats holds IPVS statistics, as exposed by the kernel in `/proc/net/ip_vs_stats`. type IPVSStats struct { // Total count of connections. Connections uint64 // Total incoming packages processed. IncomingPackets uint64 // Total outgoing packages processed. OutgoingPackets uint64 // Total incoming traffic. IncomingBytes uint64 // Total outgoing traffic. OutgoingBytes uint64 } // IPVSBackendStatus holds current metrics of one virtual / real address pair. type IPVSBackendStatus struct { // The local (virtual) IP address. LocalAddress net.IP // The local (virtual) port. LocalPort uint16 // The local firewall mark LocalMark string // The transport protocol (TCP, UDP). Proto string // The remote (real) IP address. RemoteAddress net.IP // The remote (real) port. RemotePort uint16 // The current number of active connections for this virtual/real address pair. ActiveConn uint64 // The current number of inactive connections for this virtual/real address pair. InactConn uint64 // The current weight of this virtual/real address pair. Weight uint64 } // NewIPVSStats reads the IPVS statistics. func NewIPVSStats() (IPVSStats, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return IPVSStats{}, err } return fs.NewIPVSStats() } // NewIPVSStats reads the IPVS statistics from the specified `proc` filesystem. func (fs FS) NewIPVSStats() (IPVSStats, error) { file, err := os.Open(fs.Path("net/ip_vs_stats")) if err != nil { return IPVSStats{}, err } defer file.Close() return parseIPVSStats(file) } // parseIPVSStats performs the actual parsing of `ip_vs_stats`. func parseIPVSStats(file io.Reader) (IPVSStats, error) { var ( statContent []byte statLines []string statFields []string stats IPVSStats ) statContent, err := ioutil.ReadAll(file) if err != nil { return IPVSStats{}, err } statLines = strings.SplitN(string(statContent), "\n", 4) if len(statLines) != 4 { return IPVSStats{}, errors.New("ip_vs_stats corrupt: too short") } statFields = strings.Fields(statLines[2]) if len(statFields) != 5 { return IPVSStats{}, errors.New("ip_vs_stats corrupt: unexpected number of fields") } stats.Connections, err = strconv.ParseUint(statFields[0], 16, 64) if err != nil { return IPVSStats{}, err } stats.IncomingPackets, err = strconv.ParseUint(statFields[1], 16, 64) if err != nil { return IPVSStats{}, err } stats.OutgoingPackets, err = strconv.ParseUint(statFields[2], 16, 64) if err != nil { return IPVSStats{}, err } stats.IncomingBytes, err = strconv.ParseUint(statFields[3], 16, 64) if err != nil { return IPVSStats{}, err } stats.OutgoingBytes, err = strconv.ParseUint(statFields[4], 16, 64) if err != nil { return IPVSStats{}, err } return stats, nil } // NewIPVSBackendStatus reads and returns the status of all (virtual,real) server pairs. func NewIPVSBackendStatus() ([]IPVSBackendStatus, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return []IPVSBackendStatus{}, err } return fs.NewIPVSBackendStatus() } // NewIPVSBackendStatus reads and returns the status of all (virtual,real) server pairs from the specified `proc` filesystem. func (fs FS) NewIPVSBackendStatus() ([]IPVSBackendStatus, error) { file, err := os.Open(fs.Path("net/ip_vs")) if err != nil { return nil, err } defer file.Close() return parseIPVSBackendStatus(file) } func parseIPVSBackendStatus(file io.Reader) ([]IPVSBackendStatus, error) { var ( status []IPVSBackendStatus scanner = bufio.NewScanner(file) proto string localMark string localAddress net.IP localPort uint16 err error ) for scanner.Scan() { fields := strings.Fields(string(scanner.Text())) if len(fields) == 0 { continue } switch { case fields[0] == "IP" || fields[0] == "Prot" || fields[1] == "RemoteAddress:Port": continue case fields[0] == "TCP" || fields[0] == "UDP": if len(fields) < 2 { continue } proto = fields[0] localMark = "" localAddress, localPort, err = parseIPPort(fields[1]) if err != nil { return nil, err } case fields[0] == "FWM": if len(fields) < 2 { continue } proto = fields[0] localMark = fields[1] localAddress = nil localPort = 0 case fields[0] == "->": if len(fields) < 6 { continue } remoteAddress, remotePort, err := parseIPPort(fields[1]) if err != nil { return nil, err } weight, err := strconv.ParseUint(fields[3], 10, 64) if err != nil { return nil, err } activeConn, err := strconv.ParseUint(fields[4], 10, 64) if err != nil { return nil, err } inactConn, err := strconv.ParseUint(fields[5], 10, 64) if err != nil { return nil, err } status = append(status, IPVSBackendStatus{ LocalAddress: localAddress, LocalPort: localPort, LocalMark: localMark, RemoteAddress: remoteAddress, RemotePort: remotePort, Proto: proto, Weight: weight, ActiveConn: activeConn, InactConn: inactConn, }) } } return status, nil } func parseIPPort(s string) (net.IP, uint16, error) { var ( ip net.IP err error ) switch len(s) { case 13: ip, err = hex.DecodeString(s[0:8]) if err != nil { return nil, 0, err } case 46: ip = net.ParseIP(s[1:40]) if ip == nil { return nil, 0, fmt.Errorf("invalid IPv6 address: %s", s[1:40]) } default: return nil, 0, fmt.Errorf("unexpected IP:Port: %s", s) } portString := s[len(s)-4:] if len(portString) != 4 { return nil, 0, fmt.Errorf("unexpected port string format: %s", portString) } port, err := strconv.ParseUint(portString, 16, 16) if err != nil { return nil, 0, err } return ip, uint16(port), nil } ================================================ FILE: vendor/github.com/prometheus/procfs/ipvs_test.go ================================================ package procfs import ( "net" "testing" ) var ( expectedIPVSStats = IPVSStats{ Connections: 23765872, IncomingPackets: 3811989221, OutgoingPackets: 0, IncomingBytes: 89991519156915, OutgoingBytes: 0, } expectedIPVSBackendStatuses = []IPVSBackendStatus{ { LocalAddress: net.ParseIP("192.168.0.22"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.82.22"), RemotePort: 3306, Proto: "TCP", Weight: 100, ActiveConn: 248, InactConn: 2, }, { LocalAddress: net.ParseIP("192.168.0.22"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.83.24"), RemotePort: 3306, Proto: "TCP", Weight: 100, ActiveConn: 248, InactConn: 2, }, { LocalAddress: net.ParseIP("192.168.0.22"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.83.21"), RemotePort: 3306, Proto: "TCP", Weight: 100, ActiveConn: 248, InactConn: 1, }, { LocalAddress: net.ParseIP("192.168.0.57"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.84.22"), RemotePort: 3306, Proto: "TCP", Weight: 0, ActiveConn: 0, InactConn: 0, }, { LocalAddress: net.ParseIP("192.168.0.57"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.82.21"), RemotePort: 3306, Proto: "TCP", Weight: 100, ActiveConn: 1499, InactConn: 0, }, { LocalAddress: net.ParseIP("192.168.0.57"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.50.21"), RemotePort: 3306, Proto: "TCP", Weight: 100, ActiveConn: 1498, InactConn: 0, }, { LocalAddress: net.ParseIP("192.168.0.55"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.50.26"), RemotePort: 3306, Proto: "TCP", Weight: 0, ActiveConn: 0, InactConn: 0, }, { LocalAddress: net.ParseIP("192.168.0.55"), LocalPort: 3306, RemoteAddress: net.ParseIP("192.168.49.32"), RemotePort: 3306, Proto: "TCP", Weight: 100, ActiveConn: 0, InactConn: 0, }, { LocalAddress: net.ParseIP("2620::1"), LocalPort: 80, RemoteAddress: net.ParseIP("2620::2"), RemotePort: 80, Proto: "TCP", Weight: 1, ActiveConn: 0, InactConn: 0, }, { LocalAddress: net.ParseIP("2620::1"), LocalPort: 80, RemoteAddress: net.ParseIP("2620::3"), RemotePort: 80, Proto: "TCP", Weight: 1, ActiveConn: 0, InactConn: 0, }, { LocalAddress: net.ParseIP("2620::1"), LocalPort: 80, RemoteAddress: net.ParseIP("2620::4"), RemotePort: 80, Proto: "TCP", Weight: 1, ActiveConn: 1, InactConn: 1, }, { LocalMark: "10001000", RemoteAddress: net.ParseIP("192.168.50.26"), RemotePort: 3306, Proto: "FWM", Weight: 0, ActiveConn: 0, InactConn: 1, }, { LocalMark: "10001000", RemoteAddress: net.ParseIP("192.168.50.21"), RemotePort: 3306, Proto: "FWM", Weight: 0, ActiveConn: 0, InactConn: 2, }, } ) func TestIPVSStats(t *testing.T) { stats, err := FS("fixtures").NewIPVSStats() if err != nil { t.Fatal(err) } if stats != expectedIPVSStats { t.Errorf("want %+v, have %+v", expectedIPVSStats, stats) } } func TestParseIPPort(t *testing.T) { ip := net.ParseIP("192.168.0.22") port := uint16(3306) gotIP, gotPort, err := parseIPPort("C0A80016:0CEA") if err != nil { t.Fatal(err) } if !(gotIP.Equal(ip) && port == gotPort) { t.Errorf("want %s:%d, have %s:%d", ip, port, gotIP, gotPort) } } func TestParseIPPortInvalid(t *testing.T) { testcases := []string{ "", "C0A80016", "C0A800:1234", "FOOBARBA:1234", "C0A80016:0CEA:1234", } for _, s := range testcases { ip, port, err := parseIPPort(s) if ip != nil || port != uint16(0) || err == nil { t.Errorf("Expected error for input %s, have ip = %s, port = %v, err = %v", s, ip, port, err) } } } func TestParseIPPortIPv6(t *testing.T) { ip := net.ParseIP("dead:beef::1") port := uint16(8080) gotIP, gotPort, err := parseIPPort("[DEAD:BEEF:0000:0000:0000:0000:0000:0001]:1F90") if err != nil { t.Fatal(err) } if !(gotIP.Equal(ip) && port == gotPort) { t.Errorf("want %s:%d, have %s:%d", ip, port, gotIP, gotPort) } } func TestIPVSBackendStatus(t *testing.T) { backendStats, err := FS("fixtures").NewIPVSBackendStatus() if err != nil { t.Fatal(err) } if want, have := len(expectedIPVSBackendStatuses), len(backendStats); want != have { t.Fatalf("want %d backend statuses, have %d", want, have) } for idx, expect := range expectedIPVSBackendStatuses { if !backendStats[idx].LocalAddress.Equal(expect.LocalAddress) { t.Errorf("want LocalAddress %s, have %s", expect.LocalAddress, backendStats[idx].LocalAddress) } if backendStats[idx].LocalPort != expect.LocalPort { t.Errorf("want LocalPort %d, have %d", expect.LocalPort, backendStats[idx].LocalPort) } if !backendStats[idx].RemoteAddress.Equal(expect.RemoteAddress) { t.Errorf("want RemoteAddress %s, have %s", expect.RemoteAddress, backendStats[idx].RemoteAddress) } if backendStats[idx].RemotePort != expect.RemotePort { t.Errorf("want RemotePort %d, have %d", expect.RemotePort, backendStats[idx].RemotePort) } if backendStats[idx].Proto != expect.Proto { t.Errorf("want Proto %s, have %s", expect.Proto, backendStats[idx].Proto) } if backendStats[idx].Weight != expect.Weight { t.Errorf("want Weight %d, have %d", expect.Weight, backendStats[idx].Weight) } if backendStats[idx].ActiveConn != expect.ActiveConn { t.Errorf("want ActiveConn %d, have %d", expect.ActiveConn, backendStats[idx].ActiveConn) } if backendStats[idx].InactConn != expect.InactConn { t.Errorf("want InactConn %d, have %d", expect.InactConn, backendStats[idx].InactConn) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/mdstat.go ================================================ package procfs import ( "fmt" "io/ioutil" "regexp" "strconv" "strings" ) var ( statuslineRE = regexp.MustCompile(`(\d+) blocks .*\[(\d+)/(\d+)\] \[[U_]+\]`) buildlineRE = regexp.MustCompile(`\((\d+)/\d+\)`) ) // MDStat holds info parsed from /proc/mdstat. type MDStat struct { // Name of the device. Name string // activity-state of the device. ActivityState string // Number of active disks. DisksActive int64 // Total number of disks the device consists of. DisksTotal int64 // Number of blocks the device holds. BlocksTotal int64 // Number of blocks on the device that are in sync. BlocksSynced int64 } // ParseMDStat parses an mdstat-file and returns a struct with the relevant infos. func (fs FS) ParseMDStat() (mdstates []MDStat, err error) { mdStatusFilePath := fs.Path("mdstat") content, err := ioutil.ReadFile(mdStatusFilePath) if err != nil { return []MDStat{}, fmt.Errorf("error parsing %s: %s", mdStatusFilePath, err) } mdStates := []MDStat{} lines := strings.Split(string(content), "\n") for i, l := range lines { if l == "" { continue } if l[0] == ' ' { continue } if strings.HasPrefix(l, "Personalities") || strings.HasPrefix(l, "unused") { continue } mainLine := strings.Split(l, " ") if len(mainLine) < 3 { return mdStates, fmt.Errorf("error parsing mdline: %s", l) } mdName := mainLine[0] activityState := mainLine[2] if len(lines) <= i+3 { return mdStates, fmt.Errorf( "error parsing %s: too few lines for md device %s", mdStatusFilePath, mdName, ) } active, total, size, err := evalStatusline(lines[i+1]) if err != nil { return mdStates, fmt.Errorf("error parsing %s: %s", mdStatusFilePath, err) } // j is the line number of the syncing-line. j := i + 2 if strings.Contains(lines[i+2], "bitmap") { // skip bitmap line j = i + 3 } // If device is syncing at the moment, get the number of currently // synced bytes, otherwise that number equals the size of the device. syncedBlocks := size if strings.Contains(lines[j], "recovery") || strings.Contains(lines[j], "resync") { syncedBlocks, err = evalBuildline(lines[j]) if err != nil { return mdStates, fmt.Errorf("error parsing %s: %s", mdStatusFilePath, err) } } mdStates = append(mdStates, MDStat{ Name: mdName, ActivityState: activityState, DisksActive: active, DisksTotal: total, BlocksTotal: size, BlocksSynced: syncedBlocks, }) } return mdStates, nil } func evalStatusline(statusline string) (active, total, size int64, err error) { matches := statuslineRE.FindStringSubmatch(statusline) if len(matches) != 4 { return 0, 0, 0, fmt.Errorf("unexpected statusline: %s", statusline) } size, err = strconv.ParseInt(matches[1], 10, 64) if err != nil { return 0, 0, 0, fmt.Errorf("unexpected statusline %s: %s", statusline, err) } total, err = strconv.ParseInt(matches[2], 10, 64) if err != nil { return 0, 0, 0, fmt.Errorf("unexpected statusline %s: %s", statusline, err) } active, err = strconv.ParseInt(matches[3], 10, 64) if err != nil { return 0, 0, 0, fmt.Errorf("unexpected statusline %s: %s", statusline, err) } return active, total, size, nil } func evalBuildline(buildline string) (syncedBlocks int64, err error) { matches := buildlineRE.FindStringSubmatch(buildline) if len(matches) != 2 { return 0, fmt.Errorf("unexpected buildline: %s", buildline) } syncedBlocks, err = strconv.ParseInt(matches[1], 10, 64) if err != nil { return 0, fmt.Errorf("%s in buildline: %s", err, buildline) } return syncedBlocks, nil } ================================================ FILE: vendor/github.com/prometheus/procfs/mdstat_test.go ================================================ package procfs import ( "testing" ) func TestMDStat(t *testing.T) { mdStates, err := FS("fixtures").ParseMDStat() if err != nil { t.Fatalf("parsing of reference-file failed entirely: %s", err) } refs := map[string]MDStat{ "md3": {"md3", "active", 8, 8, 5853468288, 5853468288}, "md127": {"md127", "active", 2, 2, 312319552, 312319552}, "md0": {"md0", "active", 2, 2, 248896, 248896}, "md4": {"md4", "inactive", 2, 2, 4883648, 4883648}, "md6": {"md6", "active", 1, 2, 195310144, 16775552}, "md8": {"md8", "active", 2, 2, 195310144, 16775552}, "md7": {"md7", "active", 3, 4, 7813735424, 7813735424}, } if want, have := len(refs), len(mdStates); want != have { t.Errorf("want %d parsed md-devices, have %d", want, have) } for _, md := range mdStates { if want, have := refs[md.Name], md; want != have { t.Errorf("%s: want %v, have %v", md.Name, want, have) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/mountstats.go ================================================ package procfs // While implementing parsing of /proc/[pid]/mountstats, this blog was used // heavily as a reference: // https://utcc.utoronto.ca/~cks/space/blog/linux/NFSMountstatsIndex // // Special thanks to Chris Siebenmann for all of his posts explaining the // various statistics available for NFS. import ( "bufio" "fmt" "io" "strconv" "strings" "time" ) // Constants shared between multiple functions. const ( deviceEntryLen = 8 fieldBytesLen = 8 fieldEventsLen = 27 statVersion10 = "1.0" statVersion11 = "1.1" fieldTransport10Len = 10 fieldTransport11Len = 13 ) // A Mount is a device mount parsed from /proc/[pid]/mountstats. type Mount struct { // Name of the device. Device string // The mount point of the device. Mount string // The filesystem type used by the device. Type string // If available additional statistics related to this Mount. // Use a type assertion to determine if additional statistics are available. Stats MountStats } // A MountStats is a type which contains detailed statistics for a specific // type of Mount. type MountStats interface { mountStats() } // A MountStatsNFS is a MountStats implementation for NFSv3 and v4 mounts. type MountStatsNFS struct { // The version of statistics provided. StatVersion string // The age of the NFS mount. Age time.Duration // Statistics related to byte counters for various operations. Bytes NFSBytesStats // Statistics related to various NFS event occurrences. Events NFSEventsStats // Statistics broken down by filesystem operation. Operations []NFSOperationStats // Statistics about the NFS RPC transport. Transport NFSTransportStats } // mountStats implements MountStats. func (m MountStatsNFS) mountStats() {} // A NFSBytesStats contains statistics about the number of bytes read and written // by an NFS client to and from an NFS server. type NFSBytesStats struct { // Number of bytes read using the read() syscall. Read uint64 // Number of bytes written using the write() syscall. Write uint64 // Number of bytes read using the read() syscall in O_DIRECT mode. DirectRead uint64 // Number of bytes written using the write() syscall in O_DIRECT mode. DirectWrite uint64 // Number of bytes read from the NFS server, in total. ReadTotal uint64 // Number of bytes written to the NFS server, in total. WriteTotal uint64 // Number of pages read directly via mmap()'d files. ReadPages uint64 // Number of pages written directly via mmap()'d files. WritePages uint64 } // A NFSEventsStats contains statistics about NFS event occurrences. type NFSEventsStats struct { // Number of times cached inode attributes are re-validated from the server. InodeRevalidate uint64 // Number of times cached dentry nodes are re-validated from the server. DnodeRevalidate uint64 // Number of times an inode cache is cleared. DataInvalidate uint64 // Number of times cached inode attributes are invalidated. AttributeInvalidate uint64 // Number of times files or directories have been open()'d. VFSOpen uint64 // Number of times a directory lookup has occurred. VFSLookup uint64 // Number of times permissions have been checked. VFSAccess uint64 // Number of updates (and potential writes) to pages. VFSUpdatePage uint64 // Number of pages read directly via mmap()'d files. VFSReadPage uint64 // Number of times a group of pages have been read. VFSReadPages uint64 // Number of pages written directly via mmap()'d files. VFSWritePage uint64 // Number of times a group of pages have been written. VFSWritePages uint64 // Number of times directory entries have been read with getdents(). VFSGetdents uint64 // Number of times attributes have been set on inodes. VFSSetattr uint64 // Number of pending writes that have been forcefully flushed to the server. VFSFlush uint64 // Number of times fsync() has been called on directories and files. VFSFsync uint64 // Number of times locking has been attempted on a file. VFSLock uint64 // Number of times files have been closed and released. VFSFileRelease uint64 // Unknown. Possibly unused. CongestionWait uint64 // Number of times files have been truncated. Truncation uint64 // Number of times a file has been grown due to writes beyond its existing end. WriteExtension uint64 // Number of times a file was removed while still open by another process. SillyRename uint64 // Number of times the NFS server gave less data than expected while reading. ShortRead uint64 // Number of times the NFS server wrote less data than expected while writing. ShortWrite uint64 // Number of times the NFS server indicated EJUKEBOX; retrieving data from // offline storage. JukeboxDelay uint64 // Number of NFS v4.1+ pNFS reads. PNFSRead uint64 // Number of NFS v4.1+ pNFS writes. PNFSWrite uint64 } // A NFSOperationStats contains statistics for a single operation. type NFSOperationStats struct { // The name of the operation. Operation string // Number of requests performed for this operation. Requests uint64 // Number of times an actual RPC request has been transmitted for this operation. Transmissions uint64 // Number of times a request has had a major timeout. MajorTimeouts uint64 // Number of bytes sent for this operation, including RPC headers and payload. BytesSent uint64 // Number of bytes received for this operation, including RPC headers and payload. BytesReceived uint64 // Duration all requests spent queued for transmission before they were sent. CumulativeQueueTime time.Duration // Duration it took to get a reply back after the request was transmitted. CumulativeTotalResponseTime time.Duration // Duration from when a request was enqueued to when it was completely handled. CumulativeTotalRequestTime time.Duration } // A NFSTransportStats contains statistics for the NFS mount RPC requests and // responses. type NFSTransportStats struct { // The local port used for the NFS mount. Port uint64 // Number of times the client has had to establish a connection from scratch // to the NFS server. Bind uint64 // Number of times the client has made a TCP connection to the NFS server. Connect uint64 // Duration (in jiffies, a kernel internal unit of time) the NFS mount has // spent waiting for connections to the server to be established. ConnectIdleTime uint64 // Duration since the NFS mount last saw any RPC traffic. IdleTime time.Duration // Number of RPC requests for this mount sent to the NFS server. Sends uint64 // Number of RPC responses for this mount received from the NFS server. Receives uint64 // Number of times the NFS server sent a response with a transaction ID // unknown to this client. BadTransactionIDs uint64 // A running counter, incremented on each request as the current difference // ebetween sends and receives. CumulativeActiveRequests uint64 // A running counter, incremented on each request by the current backlog // queue size. CumulativeBacklog uint64 // Stats below only available with stat version 1.1. // Maximum number of simultaneously active RPC requests ever used. MaximumRPCSlotsUsed uint64 // A running counter, incremented on each request as the current size of the // sending queue. CumulativeSendingQueue uint64 // A running counter, incremented on each request as the current size of the // pending queue. CumulativePendingQueue uint64 } // parseMountStats parses a /proc/[pid]/mountstats file and returns a slice // of Mount structures containing detailed information about each mount. // If available, statistics for each mount are parsed as well. func parseMountStats(r io.Reader) ([]*Mount, error) { const ( device = "device" statVersionPrefix = "statvers=" nfs3Type = "nfs" nfs4Type = "nfs4" ) var mounts []*Mount s := bufio.NewScanner(r) for s.Scan() { // Only look for device entries in this function ss := strings.Fields(string(s.Bytes())) if len(ss) == 0 || ss[0] != device { continue } m, err := parseMount(ss) if err != nil { return nil, err } // Does this mount also possess statistics information? if len(ss) > deviceEntryLen { // Only NFSv3 and v4 are supported for parsing statistics if m.Type != nfs3Type && m.Type != nfs4Type { return nil, fmt.Errorf("cannot parse MountStats for fstype %q", m.Type) } statVersion := strings.TrimPrefix(ss[8], statVersionPrefix) stats, err := parseMountStatsNFS(s, statVersion) if err != nil { return nil, err } m.Stats = stats } mounts = append(mounts, m) } return mounts, s.Err() } // parseMount parses an entry in /proc/[pid]/mountstats in the format: // device [device] mounted on [mount] with fstype [type] func parseMount(ss []string) (*Mount, error) { if len(ss) < deviceEntryLen { return nil, fmt.Errorf("invalid device entry: %v", ss) } // Check for specific words appearing at specific indices to ensure // the format is consistent with what we expect format := []struct { i int s string }{ {i: 0, s: "device"}, {i: 2, s: "mounted"}, {i: 3, s: "on"}, {i: 5, s: "with"}, {i: 6, s: "fstype"}, } for _, f := range format { if ss[f.i] != f.s { return nil, fmt.Errorf("invalid device entry: %v", ss) } } return &Mount{ Device: ss[1], Mount: ss[4], Type: ss[7], }, nil } // parseMountStatsNFS parses a MountStatsNFS by scanning additional information // related to NFS statistics. func parseMountStatsNFS(s *bufio.Scanner, statVersion string) (*MountStatsNFS, error) { // Field indicators for parsing specific types of data const ( fieldAge = "age:" fieldBytes = "bytes:" fieldEvents = "events:" fieldPerOpStats = "per-op" fieldTransport = "xprt:" ) stats := &MountStatsNFS{ StatVersion: statVersion, } for s.Scan() { ss := strings.Fields(string(s.Bytes())) if len(ss) == 0 { break } if len(ss) < 2 { return nil, fmt.Errorf("not enough information for NFS stats: %v", ss) } switch ss[0] { case fieldAge: // Age integer is in seconds d, err := time.ParseDuration(ss[1] + "s") if err != nil { return nil, err } stats.Age = d case fieldBytes: bstats, err := parseNFSBytesStats(ss[1:]) if err != nil { return nil, err } stats.Bytes = *bstats case fieldEvents: estats, err := parseNFSEventsStats(ss[1:]) if err != nil { return nil, err } stats.Events = *estats case fieldTransport: if len(ss) < 3 { return nil, fmt.Errorf("not enough information for NFS transport stats: %v", ss) } tstats, err := parseNFSTransportStats(ss[2:], statVersion) if err != nil { return nil, err } stats.Transport = *tstats } // When encountering "per-operation statistics", we must break this // loop and parse them separately to ensure we can terminate parsing // before reaching another device entry; hence why this 'if' statement // is not just another switch case if ss[0] == fieldPerOpStats { break } } if err := s.Err(); err != nil { return nil, err } // NFS per-operation stats appear last before the next device entry perOpStats, err := parseNFSOperationStats(s) if err != nil { return nil, err } stats.Operations = perOpStats return stats, nil } // parseNFSBytesStats parses a NFSBytesStats line using an input set of // integer fields. func parseNFSBytesStats(ss []string) (*NFSBytesStats, error) { if len(ss) != fieldBytesLen { return nil, fmt.Errorf("invalid NFS bytes stats: %v", ss) } ns := make([]uint64, 0, fieldBytesLen) for _, s := range ss { n, err := strconv.ParseUint(s, 10, 64) if err != nil { return nil, err } ns = append(ns, n) } return &NFSBytesStats{ Read: ns[0], Write: ns[1], DirectRead: ns[2], DirectWrite: ns[3], ReadTotal: ns[4], WriteTotal: ns[5], ReadPages: ns[6], WritePages: ns[7], }, nil } // parseNFSEventsStats parses a NFSEventsStats line using an input set of // integer fields. func parseNFSEventsStats(ss []string) (*NFSEventsStats, error) { if len(ss) != fieldEventsLen { return nil, fmt.Errorf("invalid NFS events stats: %v", ss) } ns := make([]uint64, 0, fieldEventsLen) for _, s := range ss { n, err := strconv.ParseUint(s, 10, 64) if err != nil { return nil, err } ns = append(ns, n) } return &NFSEventsStats{ InodeRevalidate: ns[0], DnodeRevalidate: ns[1], DataInvalidate: ns[2], AttributeInvalidate: ns[3], VFSOpen: ns[4], VFSLookup: ns[5], VFSAccess: ns[6], VFSUpdatePage: ns[7], VFSReadPage: ns[8], VFSReadPages: ns[9], VFSWritePage: ns[10], VFSWritePages: ns[11], VFSGetdents: ns[12], VFSSetattr: ns[13], VFSFlush: ns[14], VFSFsync: ns[15], VFSLock: ns[16], VFSFileRelease: ns[17], CongestionWait: ns[18], Truncation: ns[19], WriteExtension: ns[20], SillyRename: ns[21], ShortRead: ns[22], ShortWrite: ns[23], JukeboxDelay: ns[24], PNFSRead: ns[25], PNFSWrite: ns[26], }, nil } // parseNFSOperationStats parses a slice of NFSOperationStats by scanning // additional information about per-operation statistics until an empty // line is reached. func parseNFSOperationStats(s *bufio.Scanner) ([]NFSOperationStats, error) { const ( // Number of expected fields in each per-operation statistics set numFields = 9 ) var ops []NFSOperationStats for s.Scan() { ss := strings.Fields(string(s.Bytes())) if len(ss) == 0 { // Must break when reading a blank line after per-operation stats to // enable top-level function to parse the next device entry break } if len(ss) != numFields { return nil, fmt.Errorf("invalid NFS per-operations stats: %v", ss) } // Skip string operation name for integers ns := make([]uint64, 0, numFields-1) for _, st := range ss[1:] { n, err := strconv.ParseUint(st, 10, 64) if err != nil { return nil, err } ns = append(ns, n) } ops = append(ops, NFSOperationStats{ Operation: strings.TrimSuffix(ss[0], ":"), Requests: ns[0], Transmissions: ns[1], MajorTimeouts: ns[2], BytesSent: ns[3], BytesReceived: ns[4], CumulativeQueueTime: time.Duration(ns[5]) * time.Millisecond, CumulativeTotalResponseTime: time.Duration(ns[6]) * time.Millisecond, CumulativeTotalRequestTime: time.Duration(ns[7]) * time.Millisecond, }) } return ops, s.Err() } // parseNFSTransportStats parses a NFSTransportStats line using an input set of // integer fields matched to a specific stats version. func parseNFSTransportStats(ss []string, statVersion string) (*NFSTransportStats, error) { switch statVersion { case statVersion10: if len(ss) != fieldTransport10Len { return nil, fmt.Errorf("invalid NFS transport stats 1.0 statement: %v", ss) } case statVersion11: if len(ss) != fieldTransport11Len { return nil, fmt.Errorf("invalid NFS transport stats 1.1 statement: %v", ss) } default: return nil, fmt.Errorf("unrecognized NFS transport stats version: %q", statVersion) } // Allocate enough for v1.1 stats since zero value for v1.1 stats will be okay // in a v1.0 response. // // Note: slice length must be set to length of v1.1 stats to avoid a panic when // only v1.0 stats are present. // See: https://github.com/prometheus/node_exporter/issues/571. ns := make([]uint64, fieldTransport11Len) for i, s := range ss { n, err := strconv.ParseUint(s, 10, 64) if err != nil { return nil, err } ns[i] = n } return &NFSTransportStats{ Port: ns[0], Bind: ns[1], Connect: ns[2], ConnectIdleTime: ns[3], IdleTime: time.Duration(ns[4]) * time.Second, Sends: ns[5], Receives: ns[6], BadTransactionIDs: ns[7], CumulativeActiveRequests: ns[8], CumulativeBacklog: ns[9], MaximumRPCSlotsUsed: ns[10], CumulativeSendingQueue: ns[11], CumulativePendingQueue: ns[12], }, nil } ================================================ FILE: vendor/github.com/prometheus/procfs/mountstats_test.go ================================================ package procfs import ( "fmt" "reflect" "strings" "testing" "time" ) func TestMountStats(t *testing.T) { tests := []struct { name string s string mounts []*Mount invalid bool }{ { name: "no devices", s: `hello`, }, { name: "device has too few fields", s: `device foo`, invalid: true, }, { name: "device incorrect format", s: `device rootfs BAD on / with fstype rootfs`, invalid: true, }, { name: "device incorrect format", s: `device rootfs mounted BAD / with fstype rootfs`, invalid: true, }, { name: "device incorrect format", s: `device rootfs mounted on / BAD fstype rootfs`, invalid: true, }, { name: "device incorrect format", s: `device rootfs mounted on / with BAD rootfs`, invalid: true, }, { name: "device rootfs cannot have stats", s: `device rootfs mounted on / with fstype rootfs stats`, invalid: true, }, { name: "NFSv4 device with too little info", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.1\nhello", invalid: true, }, { name: "NFSv4 device with bad bytes", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.1\nbytes: 0", invalid: true, }, { name: "NFSv4 device with bad events", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.1\nevents: 0", invalid: true, }, { name: "NFSv4 device with bad per-op stats", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.1\nper-op statistics\nFOO 0", invalid: true, }, { name: "NFSv4 device with bad transport stats", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.1\nxprt: tcp", invalid: true, }, { name: "NFSv4 device with bad transport version", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=foo\nxprt: tcp 0", invalid: true, }, { name: "NFSv4 device with bad transport stats version 1.0", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.0\nxprt: tcp 0 0 0 0 0 0 0 0 0 0 0 0 0", invalid: true, }, { name: "NFSv4 device with bad transport stats version 1.1", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs4 statvers=1.1\nxprt: tcp 0 0 0 0 0 0 0 0 0 0", invalid: true, }, { name: "NFSv3 device with transport stats version 1.0 OK", s: "device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs statvers=1.0\nxprt: tcp 1 2 3 4 5 6 7 8 9 10", mounts: []*Mount{{ Device: "192.168.1.1:/srv", Mount: "/mnt/nfs", Type: "nfs", Stats: &MountStatsNFS{ StatVersion: "1.0", Transport: NFSTransportStats{ Port: 1, Bind: 2, Connect: 3, ConnectIdleTime: 4, IdleTime: 5 * time.Second, Sends: 6, Receives: 7, BadTransactionIDs: 8, CumulativeActiveRequests: 9, CumulativeBacklog: 10, }, }, }}, }, { name: "device rootfs OK", s: `device rootfs mounted on / with fstype rootfs`, mounts: []*Mount{{ Device: "rootfs", Mount: "/", Type: "rootfs", }}, }, { name: "NFSv3 device with minimal stats OK", s: `device 192.168.1.1:/srv mounted on /mnt/nfs with fstype nfs statvers=1.1`, mounts: []*Mount{{ Device: "192.168.1.1:/srv", Mount: "/mnt/nfs", Type: "nfs", Stats: &MountStatsNFS{ StatVersion: "1.1", }, }}, }, { name: "fixtures OK", mounts: []*Mount{ { Device: "rootfs", Mount: "/", Type: "rootfs", }, { Device: "sysfs", Mount: "/sys", Type: "sysfs", }, { Device: "proc", Mount: "/proc", Type: "proc", }, { Device: "/dev/sda1", Mount: "/", Type: "ext4", }, { Device: "192.168.1.1:/srv/test", Mount: "/mnt/nfs/test", Type: "nfs4", Stats: &MountStatsNFS{ StatVersion: "1.1", Age: 13968 * time.Second, Bytes: NFSBytesStats{ Read: 1207640230, ReadTotal: 1210214218, ReadPages: 295483, }, Events: NFSEventsStats{ InodeRevalidate: 52, DnodeRevalidate: 226, VFSOpen: 1, VFSLookup: 13, VFSAccess: 398, VFSReadPages: 331, VFSWritePages: 47, VFSFlush: 77, VFSFileRelease: 77, }, Operations: []NFSOperationStats{ { Operation: "NULL", }, { Operation: "READ", Requests: 1298, Transmissions: 1298, BytesSent: 207680, BytesReceived: 1210292152, CumulativeQueueTime: 6 * time.Millisecond, CumulativeTotalResponseTime: 79386 * time.Millisecond, CumulativeTotalRequestTime: 79407 * time.Millisecond, }, { Operation: "WRITE", }, }, Transport: NFSTransportStats{ Port: 832, Connect: 1, IdleTime: 11 * time.Second, Sends: 6428, Receives: 6428, CumulativeActiveRequests: 12154, MaximumRPCSlotsUsed: 24, CumulativeSendingQueue: 26, CumulativePendingQueue: 5726, }, }, }, }, }, } for i, tt := range tests { t.Logf("[%02d] test %q", i, tt.name) var mounts []*Mount var err error if tt.s != "" { mounts, err = parseMountStats(strings.NewReader(tt.s)) } else { proc, e := FS("fixtures").NewProc(26231) if e != nil { t.Fatalf("failed to create proc: %v", err) } mounts, err = proc.MountStats() } if tt.invalid && err == nil { t.Error("expected an error, but none occurred") } if !tt.invalid && err != nil { t.Errorf("unexpected error: %v", err) } if want, have := tt.mounts, mounts; !reflect.DeepEqual(want, have) { t.Errorf("mounts:\nwant:\n%v\nhave:\n%v", mountsStr(want), mountsStr(have)) } } } func mountsStr(mounts []*Mount) string { var out string for i, m := range mounts { out += fmt.Sprintf("[%d] %q on %q (%q)", i, m.Device, m.Mount, m.Type) stats, ok := m.Stats.(*MountStatsNFS) if !ok { out += "\n" continue } out += fmt.Sprintf("\n\t- v%s, age: %s", stats.StatVersion, stats.Age) out += fmt.Sprintf("\n\t- bytes: %v", stats.Bytes) out += fmt.Sprintf("\n\t- events: %v", stats.Events) out += fmt.Sprintf("\n\t- transport: %v", stats.Transport) out += fmt.Sprintf("\n\t- per-operation stats:") for _, o := range stats.Operations { out += fmt.Sprintf("\n\t\t- %v", o) } out += "\n" } return out } ================================================ FILE: vendor/github.com/prometheus/procfs/proc.go ================================================ package procfs import ( "fmt" "io/ioutil" "os" "strconv" "strings" ) // Proc provides information about a running process. type Proc struct { // The process ID. PID int fs FS } // Procs represents a list of Proc structs. type Procs []Proc func (p Procs) Len() int { return len(p) } func (p Procs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Procs) Less(i, j int) bool { return p[i].PID < p[j].PID } // Self returns a process for the current process read via /proc/self. func Self() (Proc, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return Proc{}, err } return fs.Self() } // NewProc returns a process for the given pid under /proc. func NewProc(pid int) (Proc, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return Proc{}, err } return fs.NewProc(pid) } // AllProcs returns a list of all currently available processes under /proc. func AllProcs() (Procs, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return Procs{}, err } return fs.AllProcs() } // Self returns a process for the current process. func (fs FS) Self() (Proc, error) { p, err := os.Readlink(fs.Path("self")) if err != nil { return Proc{}, err } pid, err := strconv.Atoi(strings.Replace(p, string(fs), "", -1)) if err != nil { return Proc{}, err } return fs.NewProc(pid) } // NewProc returns a process for the given pid. func (fs FS) NewProc(pid int) (Proc, error) { if _, err := os.Stat(fs.Path(strconv.Itoa(pid))); err != nil { return Proc{}, err } return Proc{PID: pid, fs: fs}, nil } // AllProcs returns a list of all currently available processes. func (fs FS) AllProcs() (Procs, error) { d, err := os.Open(fs.Path()) if err != nil { return Procs{}, err } defer d.Close() names, err := d.Readdirnames(-1) if err != nil { return Procs{}, fmt.Errorf("could not read %s: %s", d.Name(), err) } p := Procs{} for _, n := range names { pid, err := strconv.ParseInt(n, 10, 64) if err != nil { continue } p = append(p, Proc{PID: int(pid), fs: fs}) } return p, nil } // CmdLine returns the command line of a process. func (p Proc) CmdLine() ([]string, error) { f, err := os.Open(p.path("cmdline")) if err != nil { return nil, err } defer f.Close() data, err := ioutil.ReadAll(f) if err != nil { return nil, err } if len(data) < 1 { return []string{}, nil } return strings.Split(string(data[:len(data)-1]), string(byte(0))), nil } // Comm returns the command name of a process. func (p Proc) Comm() (string, error) { f, err := os.Open(p.path("comm")) if err != nil { return "", err } defer f.Close() data, err := ioutil.ReadAll(f) if err != nil { return "", err } return strings.TrimSpace(string(data)), nil } // Executable returns the absolute path of the executable command of a process. func (p Proc) Executable() (string, error) { exe, err := os.Readlink(p.path("exe")) if os.IsNotExist(err) { return "", nil } return exe, err } // FileDescriptors returns the currently open file descriptors of a process. func (p Proc) FileDescriptors() ([]uintptr, error) { names, err := p.fileDescriptors() if err != nil { return nil, err } fds := make([]uintptr, len(names)) for i, n := range names { fd, err := strconv.ParseInt(n, 10, 32) if err != nil { return nil, fmt.Errorf("could not parse fd %s: %s", n, err) } fds[i] = uintptr(fd) } return fds, nil } // FileDescriptorTargets returns the targets of all file descriptors of a process. // If a file descriptor is not a symlink to a file (like a socket), that value will be the empty string. func (p Proc) FileDescriptorTargets() ([]string, error) { names, err := p.fileDescriptors() if err != nil { return nil, err } targets := make([]string, len(names)) for i, name := range names { target, err := os.Readlink(p.path("fd", name)) if err == nil { targets[i] = target } } return targets, nil } // FileDescriptorsLen returns the number of currently open file descriptors of // a process. func (p Proc) FileDescriptorsLen() (int, error) { fds, err := p.fileDescriptors() if err != nil { return 0, err } return len(fds), nil } // MountStats retrieves statistics and configuration for mount points in a // process's namespace. func (p Proc) MountStats() ([]*Mount, error) { f, err := os.Open(p.path("mountstats")) if err != nil { return nil, err } defer f.Close() return parseMountStats(f) } func (p Proc) fileDescriptors() ([]string, error) { d, err := os.Open(p.path("fd")) if err != nil { return nil, err } defer d.Close() names, err := d.Readdirnames(-1) if err != nil { return nil, fmt.Errorf("could not read %s: %s", d.Name(), err) } return names, nil } func (p Proc) path(pa ...string) string { return p.fs.Path(append([]string{strconv.Itoa(p.PID)}, pa...)...) } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_io.go ================================================ package procfs import ( "fmt" "io/ioutil" "os" ) // ProcIO models the content of /proc//io. type ProcIO struct { // Chars read. RChar uint64 // Chars written. WChar uint64 // Read syscalls. SyscR uint64 // Write syscalls. SyscW uint64 // Bytes read. ReadBytes uint64 // Bytes written. WriteBytes uint64 // Bytes written, but taking into account truncation. See // Documentation/filesystems/proc.txt in the kernel sources for // detailed explanation. CancelledWriteBytes int64 } // NewIO creates a new ProcIO instance from a given Proc instance. func (p Proc) NewIO() (ProcIO, error) { pio := ProcIO{} f, err := os.Open(p.path("io")) if err != nil { return pio, err } defer f.Close() data, err := ioutil.ReadAll(f) if err != nil { return pio, err } ioFormat := "rchar: %d\nwchar: %d\nsyscr: %d\nsyscw: %d\n" + "read_bytes: %d\nwrite_bytes: %d\n" + "cancelled_write_bytes: %d\n" _, err = fmt.Sscanf(string(data), ioFormat, &pio.RChar, &pio.WChar, &pio.SyscR, &pio.SyscW, &pio.ReadBytes, &pio.WriteBytes, &pio.CancelledWriteBytes) if err != nil { return pio, err } return pio, nil } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_io_test.go ================================================ package procfs import "testing" func TestProcIO(t *testing.T) { p, err := FS("fixtures").NewProc(26231) if err != nil { t.Fatal(err) } s, err := p.NewIO() if err != nil { t.Fatal(err) } for _, test := range []struct { name string want int64 have int64 }{ {name: "RChar", want: 750339, have: int64(s.RChar)}, {name: "WChar", want: 818609, have: int64(s.WChar)}, {name: "SyscR", want: 7405, have: int64(s.SyscR)}, {name: "SyscW", want: 5245, have: int64(s.SyscW)}, {name: "ReadBytes", want: 1024, have: int64(s.ReadBytes)}, {name: "WriteBytes", want: 2048, have: int64(s.WriteBytes)}, {name: "CancelledWriteBytes", want: -1024, have: s.CancelledWriteBytes}, } { if test.want != test.have { t.Errorf("want %s %d, have %d", test.name, test.want, test.have) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_limits.go ================================================ package procfs import ( "bufio" "fmt" "os" "regexp" "strconv" ) // ProcLimits represents the soft limits for each of the process's resource // limits. For more information see getrlimit(2): // http://man7.org/linux/man-pages/man2/getrlimit.2.html. type ProcLimits struct { // CPU time limit in seconds. CPUTime int // Maximum size of files that the process may create. FileSize int // Maximum size of the process's data segment (initialized data, // uninitialized data, and heap). DataSize int // Maximum size of the process stack in bytes. StackSize int // Maximum size of a core file. CoreFileSize int // Limit of the process's resident set in pages. ResidentSet int // Maximum number of processes that can be created for the real user ID of // the calling process. Processes int // Value one greater than the maximum file descriptor number that can be // opened by this process. OpenFiles int // Maximum number of bytes of memory that may be locked into RAM. LockedMemory int // Maximum size of the process's virtual memory address space in bytes. AddressSpace int // Limit on the combined number of flock(2) locks and fcntl(2) leases that // this process may establish. FileLocks int // Limit of signals that may be queued for the real user ID of the calling // process. PendingSignals int // Limit on the number of bytes that can be allocated for POSIX message // queues for the real user ID of the calling process. MsqqueueSize int // Limit of the nice priority set using setpriority(2) or nice(2). NicePriority int // Limit of the real-time priority set using sched_setscheduler(2) or // sched_setparam(2). RealtimePriority int // Limit (in microseconds) on the amount of CPU time that a process // scheduled under a real-time scheduling policy may consume without making // a blocking system call. RealtimeTimeout int } const ( limitsFields = 3 limitsUnlimited = "unlimited" ) var ( limitsDelimiter = regexp.MustCompile(" +") ) // NewLimits returns the current soft limits of the process. func (p Proc) NewLimits() (ProcLimits, error) { f, err := os.Open(p.path("limits")) if err != nil { return ProcLimits{}, err } defer f.Close() var ( l = ProcLimits{} s = bufio.NewScanner(f) ) for s.Scan() { fields := limitsDelimiter.Split(s.Text(), limitsFields) if len(fields) != limitsFields { return ProcLimits{}, fmt.Errorf( "couldn't parse %s line %s", f.Name(), s.Text()) } switch fields[0] { case "Max cpu time": l.CPUTime, err = parseInt(fields[1]) case "Max file size": l.FileSize, err = parseInt(fields[1]) case "Max data size": l.DataSize, err = parseInt(fields[1]) case "Max stack size": l.StackSize, err = parseInt(fields[1]) case "Max core file size": l.CoreFileSize, err = parseInt(fields[1]) case "Max resident set": l.ResidentSet, err = parseInt(fields[1]) case "Max processes": l.Processes, err = parseInt(fields[1]) case "Max open files": l.OpenFiles, err = parseInt(fields[1]) case "Max locked memory": l.LockedMemory, err = parseInt(fields[1]) case "Max address space": l.AddressSpace, err = parseInt(fields[1]) case "Max file locks": l.FileLocks, err = parseInt(fields[1]) case "Max pending signals": l.PendingSignals, err = parseInt(fields[1]) case "Max msgqueue size": l.MsqqueueSize, err = parseInt(fields[1]) case "Max nice priority": l.NicePriority, err = parseInt(fields[1]) case "Max realtime priority": l.RealtimePriority, err = parseInt(fields[1]) case "Max realtime timeout": l.RealtimeTimeout, err = parseInt(fields[1]) } if err != nil { return ProcLimits{}, err } } return l, s.Err() } func parseInt(s string) (int, error) { if s == limitsUnlimited { return -1, nil } i, err := strconv.ParseInt(s, 10, 32) if err != nil { return 0, fmt.Errorf("couldn't parse value %s: %s", s, err) } return int(i), nil } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_limits_test.go ================================================ package procfs import "testing" func TestNewLimits(t *testing.T) { p, err := FS("fixtures").NewProc(26231) if err != nil { t.Fatal(err) } l, err := p.NewLimits() if err != nil { t.Fatal(err) } for _, test := range []struct { name string want int have int }{ {name: "cpu time", want: -1, have: l.CPUTime}, {name: "open files", want: 2048, have: l.OpenFiles}, {name: "msgqueue size", want: 819200, have: l.MsqqueueSize}, {name: "nice priority", want: 0, have: l.NicePriority}, {name: "address space", want: -1, have: l.AddressSpace}, } { if test.want != test.have { t.Errorf("want %s %d, have %d", test.name, test.want, test.have) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_stat.go ================================================ package procfs import ( "bytes" "fmt" "io/ioutil" "os" ) // Originally, this USER_HZ value was dynamically retrieved via a sysconf call // which required cgo. However, that caused a lot of problems regarding // cross-compilation. Alternatives such as running a binary to determine the // value, or trying to derive it in some other way were all problematic. After // much research it was determined that USER_HZ is actually hardcoded to 100 on // all Go-supported platforms as of the time of this writing. This is why we // decided to hardcode it here as well. It is not impossible that there could // be systems with exceptions, but they should be very exotic edge cases, and // in that case, the worst outcome will be two misreported metrics. // // See also the following discussions: // // - https://github.com/prometheus/node_exporter/issues/52 // - https://github.com/prometheus/procfs/pull/2 // - http://stackoverflow.com/questions/17410841/how-does-user-hz-solve-the-jiffy-scaling-issue const userHZ = 100 // ProcStat provides status information about the process, // read from /proc/[pid]/stat. type ProcStat struct { // The process ID. PID int // The filename of the executable. Comm string // The process state. State string // The PID of the parent of this process. PPID int // The process group ID of the process. PGRP int // The session ID of the process. Session int // The controlling terminal of the process. TTY int // The ID of the foreground process group of the controlling terminal of // the process. TPGID int // The kernel flags word of the process. Flags uint // The number of minor faults the process has made which have not required // loading a memory page from disk. MinFlt uint // The number of minor faults that the process's waited-for children have // made. CMinFlt uint // The number of major faults the process has made which have required // loading a memory page from disk. MajFlt uint // The number of major faults that the process's waited-for children have // made. CMajFlt uint // Amount of time that this process has been scheduled in user mode, // measured in clock ticks. UTime uint // Amount of time that this process has been scheduled in kernel mode, // measured in clock ticks. STime uint // Amount of time that this process's waited-for children have been // scheduled in user mode, measured in clock ticks. CUTime uint // Amount of time that this process's waited-for children have been // scheduled in kernel mode, measured in clock ticks. CSTime uint // For processes running a real-time scheduling policy, this is the negated // scheduling priority, minus one. Priority int // The nice value, a value in the range 19 (low priority) to -20 (high // priority). Nice int // Number of threads in this process. NumThreads int // The time the process started after system boot, the value is expressed // in clock ticks. Starttime uint64 // Virtual memory size in bytes. VSize int // Resident set size in pages. RSS int fs FS } // NewStat returns the current status information of the process. func (p Proc) NewStat() (ProcStat, error) { f, err := os.Open(p.path("stat")) if err != nil { return ProcStat{}, err } defer f.Close() data, err := ioutil.ReadAll(f) if err != nil { return ProcStat{}, err } var ( ignore int s = ProcStat{PID: p.PID, fs: p.fs} l = bytes.Index(data, []byte("(")) r = bytes.LastIndex(data, []byte(")")) ) if l < 0 || r < 0 { return ProcStat{}, fmt.Errorf( "unexpected format, couldn't extract comm: %s", data, ) } s.Comm = string(data[l+1 : r]) _, err = fmt.Fscan( bytes.NewBuffer(data[r+2:]), &s.State, &s.PPID, &s.PGRP, &s.Session, &s.TTY, &s.TPGID, &s.Flags, &s.MinFlt, &s.CMinFlt, &s.MajFlt, &s.CMajFlt, &s.UTime, &s.STime, &s.CUTime, &s.CSTime, &s.Priority, &s.Nice, &s.NumThreads, &ignore, &s.Starttime, &s.VSize, &s.RSS, ) if err != nil { return ProcStat{}, err } return s, nil } // VirtualMemory returns the virtual memory size in bytes. func (s ProcStat) VirtualMemory() int { return s.VSize } // ResidentMemory returns the resident memory size in bytes. func (s ProcStat) ResidentMemory() int { return s.RSS * os.Getpagesize() } // StartTime returns the unix timestamp of the process in seconds. func (s ProcStat) StartTime() (float64, error) { stat, err := s.fs.NewStat() if err != nil { return 0, err } return float64(stat.BootTime) + (float64(s.Starttime) / userHZ), nil } // CPUTime returns the total CPU user and system time in seconds. func (s ProcStat) CPUTime() float64 { return float64(s.UTime+s.STime) / userHZ } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_stat_test.go ================================================ package procfs import ( "os" "testing" ) func TestProcStat(t *testing.T) { p, err := FS("fixtures").NewProc(26231) if err != nil { t.Fatal(err) } s, err := p.NewStat() if err != nil { t.Fatal(err) } for _, test := range []struct { name string want int have int }{ {name: "pid", want: 26231, have: s.PID}, {name: "user time", want: 1677, have: int(s.UTime)}, {name: "system time", want: 44, have: int(s.STime)}, {name: "start time", want: 82375, have: int(s.Starttime)}, {name: "virtual memory size", want: 56274944, have: s.VSize}, {name: "resident set size", want: 1981, have: s.RSS}, } { if test.want != test.have { t.Errorf("want %s %d, have %d", test.name, test.want, test.have) } } } func TestProcStatComm(t *testing.T) { s1, err := testProcStat(26231) if err != nil { t.Fatal(err) } if want, have := "vim", s1.Comm; want != have { t.Errorf("want comm %s, have %s", want, have) } s2, err := testProcStat(584) if err != nil { t.Fatal(err) } if want, have := "(a b ) ( c d) ", s2.Comm; want != have { t.Errorf("want comm %s, have %s", want, have) } } func TestProcStatVirtualMemory(t *testing.T) { s, err := testProcStat(26231) if err != nil { t.Fatal(err) } if want, have := 56274944, s.VirtualMemory(); want != have { t.Errorf("want virtual memory %d, have %d", want, have) } } func TestProcStatResidentMemory(t *testing.T) { s, err := testProcStat(26231) if err != nil { t.Fatal(err) } if want, have := 1981*os.Getpagesize(), s.ResidentMemory(); want != have { t.Errorf("want resident memory %d, have %d", want, have) } } func TestProcStatStartTime(t *testing.T) { s, err := testProcStat(26231) if err != nil { t.Fatal(err) } time, err := s.StartTime() if err != nil { t.Fatal(err) } if want, have := 1418184099.75, time; want != have { t.Errorf("want start time %f, have %f", want, have) } } func TestProcStatCPUTime(t *testing.T) { s, err := testProcStat(26231) if err != nil { t.Fatal(err) } if want, have := 17.21, s.CPUTime(); want != have { t.Errorf("want cpu time %f, have %f", want, have) } } func testProcStat(pid int) (ProcStat, error) { p, err := FS("fixtures").NewProc(pid) if err != nil { return ProcStat{}, err } return p.NewStat() } ================================================ FILE: vendor/github.com/prometheus/procfs/proc_test.go ================================================ package procfs import ( "reflect" "sort" "testing" ) func TestSelf(t *testing.T) { fs := FS("fixtures") p1, err := fs.NewProc(26231) if err != nil { t.Fatal(err) } p2, err := fs.Self() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(p1, p2) { t.Errorf("want process %v, have %v", p1, p2) } } func TestAllProcs(t *testing.T) { procs, err := FS("fixtures").AllProcs() if err != nil { t.Fatal(err) } sort.Sort(procs) for i, p := range []*Proc{{PID: 584}, {PID: 26231}} { if want, have := p.PID, procs[i].PID; want != have { t.Errorf("want processes %d, have %d", want, have) } } } func TestCmdLine(t *testing.T) { for _, tt := range []struct { process int want []string }{ {process: 26231, want: []string{"vim", "test.go", "+10"}}, {process: 26232, want: []string{}}, } { p1, err := FS("fixtures").NewProc(tt.process) if err != nil { t.Fatal(err) } c1, err := p1.CmdLine() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(tt.want, c1) { t.Errorf("want cmdline %v, have %v", tt.want, c1) } } } func TestComm(t *testing.T) { for _, tt := range []struct { process int want string }{ {process: 26231, want: "vim"}, {process: 26232, want: "ata_sff"}, } { p1, err := FS("fixtures").NewProc(tt.process) if err != nil { t.Fatal(err) } c1, err := p1.Comm() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(tt.want, c1) { t.Errorf("want comm %v, have %v", tt.want, c1) } } } func TestExecutable(t *testing.T) { for _, tt := range []struct { process int want string }{ {process: 26231, want: "/usr/bin/vim"}, {process: 26232, want: ""}, } { p, err := FS("fixtures").NewProc(tt.process) if err != nil { t.Fatal(err) } exe, err := p.Executable() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(tt.want, exe) { t.Errorf("want absolute path to cmdline %v, have %v", tt.want, exe) } } } func TestFileDescriptors(t *testing.T) { p1, err := FS("fixtures").NewProc(26231) if err != nil { t.Fatal(err) } fds, err := p1.FileDescriptors() if err != nil { t.Fatal(err) } sort.Sort(byUintptr(fds)) if want := []uintptr{0, 1, 2, 3, 10}; !reflect.DeepEqual(want, fds) { t.Errorf("want fds %v, have %v", want, fds) } } func TestFileDescriptorTargets(t *testing.T) { p1, err := FS("fixtures").NewProc(26231) if err != nil { t.Fatal(err) } fds, err := p1.FileDescriptorTargets() if err != nil { t.Fatal(err) } sort.Strings(fds) var want = []string{ "../../symlinktargets/abc", "../../symlinktargets/def", "../../symlinktargets/ghi", "../../symlinktargets/uvw", "../../symlinktargets/xyz", } if !reflect.DeepEqual(want, fds) { t.Errorf("want fds %v, have %v", want, fds) } } func TestFileDescriptorsLen(t *testing.T) { p1, err := FS("fixtures").NewProc(26231) if err != nil { t.Fatal(err) } l, err := p1.FileDescriptorsLen() if err != nil { t.Fatal(err) } if want, have := 5, l; want != have { t.Errorf("want fds %d, have %d", want, have) } } type byUintptr []uintptr func (a byUintptr) Len() int { return len(a) } func (a byUintptr) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byUintptr) Less(i, j int) bool { return a[i] < a[j] } ================================================ FILE: vendor/github.com/prometheus/procfs/stat.go ================================================ package procfs import ( "bufio" "fmt" "io" "os" "strconv" "strings" ) // CPUStat shows how much time the cpu spend in various stages. type CPUStat struct { User float64 Nice float64 System float64 Idle float64 Iowait float64 IRQ float64 SoftIRQ float64 Steal float64 Guest float64 GuestNice float64 } // SoftIRQStat represent the softirq statistics as exported in the procfs stat file. // A nice introduction can be found at https://0xax.gitbooks.io/linux-insides/content/interrupts/interrupts-9.html // It is possible to get per-cpu stats by reading /proc/softirqs type SoftIRQStat struct { Hi uint64 Timer uint64 NetTx uint64 NetRx uint64 Block uint64 BlockIoPoll uint64 Tasklet uint64 Sched uint64 Hrtimer uint64 Rcu uint64 } // Stat represents kernel/system statistics. type Stat struct { // Boot time in seconds since the Epoch. BootTime uint64 // Summed up cpu statistics. CPUTotal CPUStat // Per-CPU statistics. CPU []CPUStat // Number of times interrupts were handled, which contains numbered and unnumbered IRQs. IRQTotal uint64 // Number of times a numbered IRQ was triggered. IRQ []uint64 // Number of times a context switch happened. ContextSwitches uint64 // Number of times a process was created. ProcessCreated uint64 // Number of processes currently running. ProcessesRunning uint64 // Number of processes currently blocked (waiting for IO). ProcessesBlocked uint64 // Number of times a softirq was scheduled. SoftIRQTotal uint64 // Detailed softirq statistics. SoftIRQ SoftIRQStat } // NewStat returns kernel/system statistics read from /proc/stat. func NewStat() (Stat, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return Stat{}, err } return fs.NewStat() } // Parse a cpu statistics line and returns the CPUStat struct plus the cpu id (or -1 for the overall sum). func parseCPUStat(line string) (CPUStat, int64, error) { cpuStat := CPUStat{} var cpu string count, err := fmt.Sscanf(line, "%s %f %f %f %f %f %f %f %f %f %f", &cpu, &cpuStat.User, &cpuStat.Nice, &cpuStat.System, &cpuStat.Idle, &cpuStat.Iowait, &cpuStat.IRQ, &cpuStat.SoftIRQ, &cpuStat.Steal, &cpuStat.Guest, &cpuStat.GuestNice) if err != nil && err != io.EOF { return CPUStat{}, -1, fmt.Errorf("couldn't parse %s (cpu): %s", line, err) } if count == 0 { return CPUStat{}, -1, fmt.Errorf("couldn't parse %s (cpu): 0 elements parsed", line) } cpuStat.User /= userHZ cpuStat.Nice /= userHZ cpuStat.System /= userHZ cpuStat.Idle /= userHZ cpuStat.Iowait /= userHZ cpuStat.IRQ /= userHZ cpuStat.SoftIRQ /= userHZ cpuStat.Steal /= userHZ cpuStat.Guest /= userHZ cpuStat.GuestNice /= userHZ if cpu == "cpu" { return cpuStat, -1, nil } cpuID, err := strconv.ParseInt(cpu[3:], 10, 64) if err != nil { return CPUStat{}, -1, fmt.Errorf("couldn't parse %s (cpu/cpuid): %s", line, err) } return cpuStat, cpuID, nil } // Parse a softirq line. func parseSoftIRQStat(line string) (SoftIRQStat, uint64, error) { softIRQStat := SoftIRQStat{} var total uint64 var prefix string _, err := fmt.Sscanf(line, "%s %d %d %d %d %d %d %d %d %d %d %d", &prefix, &total, &softIRQStat.Hi, &softIRQStat.Timer, &softIRQStat.NetTx, &softIRQStat.NetRx, &softIRQStat.Block, &softIRQStat.BlockIoPoll, &softIRQStat.Tasklet, &softIRQStat.Sched, &softIRQStat.Hrtimer, &softIRQStat.Rcu) if err != nil { return SoftIRQStat{}, 0, fmt.Errorf("couldn't parse %s (softirq): %s", line, err) } return softIRQStat, total, nil } // NewStat returns an information about current kernel/system statistics. func (fs FS) NewStat() (Stat, error) { // See https://www.kernel.org/doc/Documentation/filesystems/proc.txt f, err := os.Open(fs.Path("stat")) if err != nil { return Stat{}, err } defer f.Close() stat := Stat{} scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() parts := strings.Fields(scanner.Text()) // require at least if len(parts) < 2 { continue } switch { case parts[0] == "btime": if stat.BootTime, err = strconv.ParseUint(parts[1], 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (btime): %s", parts[1], err) } case parts[0] == "intr": if stat.IRQTotal, err = strconv.ParseUint(parts[1], 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (intr): %s", parts[1], err) } numberedIRQs := parts[2:] stat.IRQ = make([]uint64, len(numberedIRQs)) for i, count := range numberedIRQs { if stat.IRQ[i], err = strconv.ParseUint(count, 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (intr%d): %s", count, i, err) } } case parts[0] == "ctxt": if stat.ContextSwitches, err = strconv.ParseUint(parts[1], 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (ctxt): %s", parts[1], err) } case parts[0] == "processes": if stat.ProcessCreated, err = strconv.ParseUint(parts[1], 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (processes): %s", parts[1], err) } case parts[0] == "procs_running": if stat.ProcessesRunning, err = strconv.ParseUint(parts[1], 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (procs_running): %s", parts[1], err) } case parts[0] == "procs_blocked": if stat.ProcessesBlocked, err = strconv.ParseUint(parts[1], 10, 64); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s (procs_blocked): %s", parts[1], err) } case parts[0] == "softirq": softIRQStats, total, err := parseSoftIRQStat(line) if err != nil { return Stat{}, err } stat.SoftIRQTotal = total stat.SoftIRQ = softIRQStats case strings.HasPrefix(parts[0], "cpu"): cpuStat, cpuID, err := parseCPUStat(line) if err != nil { return Stat{}, err } if cpuID == -1 { stat.CPUTotal = cpuStat } else { for int64(len(stat.CPU)) <= cpuID { stat.CPU = append(stat.CPU, CPUStat{}) } stat.CPU[cpuID] = cpuStat } } } if err := scanner.Err(); err != nil { return Stat{}, fmt.Errorf("couldn't parse %s: %s", f.Name(), err) } return stat, nil } ================================================ FILE: vendor/github.com/prometheus/procfs/stat_test.go ================================================ package procfs import "testing" func TestStat(t *testing.T) { s, err := FS("fixtures").NewStat() if err != nil { t.Fatal(err) } // cpu if want, have := float64(301854)/userHZ, s.CPUTotal.User; want != have { t.Errorf("want cpu/user %v, have %v", want, have) } if want, have := float64(31)/userHZ, s.CPU[7].SoftIRQ; want != have { t.Errorf("want cpu7/softirq %v, have %v", want, have) } // intr if want, have := uint64(8885917), s.IRQTotal; want != have { t.Errorf("want irq/total %d, have %d", want, have) } if want, have := uint64(1), s.IRQ[8]; want != have { t.Errorf("want irq8 %d, have %d", want, have) } // ctxt if want, have := uint64(38014093), s.ContextSwitches; want != have { t.Errorf("want context switches (ctxt) %d, have %d", want, have) } // btime if want, have := uint64(1418183276), s.BootTime; want != have { t.Errorf("want boot time (btime) %d, have %d", want, have) } // processes if want, have := uint64(26442), s.ProcessCreated; want != have { t.Errorf("want process created (processes) %d, have %d", want, have) } // procs_running if want, have := uint64(2), s.ProcessesRunning; want != have { t.Errorf("want processes running (procs_running) %d, have %d", want, have) } // procs_blocked if want, have := uint64(1), s.ProcessesBlocked; want != have { t.Errorf("want processes blocked (procs_blocked) %d, have %d", want, have) } // softirq if want, have := uint64(5057579), s.SoftIRQTotal; want != have { t.Errorf("want softirq total %d, have %d", want, have) } if want, have := uint64(508444), s.SoftIRQ.Rcu; want != have { t.Errorf("want softirq RCU %d, have %d", want, have) } } ================================================ FILE: vendor/github.com/prometheus/procfs/ttar ================================================ #!/usr/bin/env bash # Purpose: plain text tar format # Limitations: - only suitable for text files, directories, and symlinks # - stores only filename, content, and mode # - not designed for untrusted input # Note: must work with bash version 3.2 (macOS) set -o errexit -o nounset # Sanitize environment (for instance, standard sorting of glob matches) export LC_ALL=C path="" CMD="" function usage { bname=$(basename "$0") cat << USAGE Usage: $bname [-C

] -c -f (create archive) $bname -t -f (list archive contents) $bname [-C ] -x -f (extract archive) Options: -C (change directory) Example: Change to sysfs directory, create ttar file from fixtures directory $bname -C sysfs -c -f sysfs/fixtures.ttar fixtures/ USAGE exit "$1" } function vecho { if [ "${VERBOSE:-}" == "yes" ]; then echo >&7 "$@" fi } function set_cmd { if [ -n "$CMD" ]; then echo "ERROR: more than one command given" echo usage 2 fi CMD=$1 } while getopts :cf:htxvC: opt; do case $opt in c) set_cmd "create" ;; f) ARCHIVE=$OPTARG ;; h) usage 0 ;; t) set_cmd "list" ;; x) set_cmd "extract" ;; v) VERBOSE=yes exec 7>&1 ;; C) CDIR=$OPTARG ;; *) echo >&2 "ERROR: invalid option -$OPTARG" echo usage 1 ;; esac done # Remove processed options from arguments shift $(( OPTIND - 1 )); if [ "${CMD:-}" == "" ]; then echo >&2 "ERROR: no command given" echo usage 1 elif [ "${ARCHIVE:-}" == "" ]; then echo >&2 "ERROR: no archive name given" echo usage 1 fi function list { local path="" local size=0 local line_no=0 local ttar_file=$1 if [ -n "${2:-}" ]; then echo >&2 "ERROR: too many arguments." echo usage 1 fi if [ ! -e "$ttar_file" ]; then echo >&2 "ERROR: file not found ($ttar_file)" echo usage 1 fi while read -r line; do line_no=$(( line_no + 1 )) if [ $size -gt 0 ]; then size=$(( size - 1 )) continue fi if [[ $line =~ ^Path:\ (.*)$ ]]; then path=${BASH_REMATCH[1]} elif [[ $line =~ ^Lines:\ (.*)$ ]]; then size=${BASH_REMATCH[1]} echo "$path" elif [[ $line =~ ^Directory:\ (.*)$ ]]; then path=${BASH_REMATCH[1]} echo "$path/" elif [[ $line =~ ^SymlinkTo:\ (.*)$ ]]; then echo "$path -> ${BASH_REMATCH[1]}" fi done < "$ttar_file" } function extract { local path="" local size=0 local line_no=0 local ttar_file=$1 if [ -n "${2:-}" ]; then echo >&2 "ERROR: too many arguments." echo usage 1 fi if [ ! -e "$ttar_file" ]; then echo >&2 "ERROR: file not found ($ttar_file)" echo usage 1 fi while IFS= read -r line; do line_no=$(( line_no + 1 )) if [ "$size" -gt 0 ]; then echo "$line" >> "$path" size=$(( size - 1 )) continue fi if [[ $line =~ ^Path:\ (.*)$ ]]; then path=${BASH_REMATCH[1]} if [ -e "$path" ] || [ -L "$path" ]; then rm "$path" fi elif [[ $line =~ ^Lines:\ (.*)$ ]]; then size=${BASH_REMATCH[1]} # Create file even if it is zero-length. touch "$path" vecho " $path" elif [[ $line =~ ^Mode:\ (.*)$ ]]; then mode=${BASH_REMATCH[1]} chmod "$mode" "$path" vecho "$mode" elif [[ $line =~ ^Directory:\ (.*)$ ]]; then path=${BASH_REMATCH[1]} mkdir -p "$path" vecho " $path/" elif [[ $line =~ ^SymlinkTo:\ (.*)$ ]]; then ln -s "${BASH_REMATCH[1]}" "$path" vecho " $path -> ${BASH_REMATCH[1]}" elif [[ $line =~ ^# ]]; then # Ignore comments between files continue else echo >&2 "ERROR: Unknown keyword on line $line_no: $line" exit 1 fi done < "$ttar_file" } function div { echo "# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" \ "- - - - - -" } function get_mode { local mfile=$1 if [ -z "${STAT_OPTION:-}" ]; then if stat -c '%a' "$mfile" >/dev/null 2>&1; then STAT_OPTION='-c' STAT_FORMAT='%a' else STAT_OPTION='-f' STAT_FORMAT='%A' fi fi stat "${STAT_OPTION}" "${STAT_FORMAT}" "$mfile" } function _create { shopt -s nullglob local mode while (( "$#" )); do file=$1 if [ -L "$file" ]; then echo "Path: $file" symlinkTo=$(readlink "$file") echo "SymlinkTo: $symlinkTo" vecho " $file -> $symlinkTo" div elif [ -d "$file" ]; then # Strip trailing slash (if there is one) file=${file%/} echo "Directory: $file" mode=$(get_mode "$file") echo "Mode: $mode" vecho "$mode $file/" div # Find all files and dirs, including hidden/dot files for x in "$file/"{*,.[^.]*}; do _create "$x" done elif [ -f "$file" ]; then echo "Path: $file" lines=$(wc -l "$file"|awk '{print $1}') echo "Lines: $lines" cat "$file" mode=$(get_mode "$file") echo "Mode: $mode" vecho "$mode $file" div else echo >&2 "ERROR: file not found ($file in $(pwd))" exit 2 fi shift done } function create { ttar_file=$1 shift if [ -z "${1:-}" ]; then echo >&2 "ERROR: missing arguments." echo usage 1 fi if [ -e "$ttar_file" ]; then rm "$ttar_file" fi exec > "$ttar_file" _create "$@" } if [ -n "${CDIR:-}" ]; then if [[ "$ARCHIVE" != /* ]]; then # Relative path: preserve the archive's location before changing # directory ARCHIVE="$(pwd)/$ARCHIVE" fi cd "$CDIR" fi "$CMD" "$ARCHIVE" "$@" ================================================ FILE: vendor/github.com/prometheus/procfs/xfrm.go ================================================ // Copyright 2017 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package procfs import ( "bufio" "fmt" "os" "strconv" "strings" ) // XfrmStat models the contents of /proc/net/xfrm_stat. type XfrmStat struct { // All errors which are not matched by other XfrmInError int // No buffer is left XfrmInBufferError int // Header Error XfrmInHdrError int // No state found // i.e. either inbound SPI, address, or IPSEC protocol at SA is wrong XfrmInNoStates int // Transformation protocol specific error // e.g. SA Key is wrong XfrmInStateProtoError int // Transformation mode specific error XfrmInStateModeError int // Sequence error // e.g. sequence number is out of window XfrmInStateSeqError int // State is expired XfrmInStateExpired int // State has mismatch option // e.g. UDP encapsulation type is mismatched XfrmInStateMismatch int // State is invalid XfrmInStateInvalid int // No matching template for states // e.g. Inbound SAs are correct but SP rule is wrong XfrmInTmplMismatch int // No policy is found for states // e.g. Inbound SAs are correct but no SP is found XfrmInNoPols int // Policy discards XfrmInPolBlock int // Policy error XfrmInPolError int // All errors which are not matched by others XfrmOutError int // Bundle generation error XfrmOutBundleGenError int // Bundle check error XfrmOutBundleCheckError int // No state was found XfrmOutNoStates int // Transformation protocol specific error XfrmOutStateProtoError int // Transportation mode specific error XfrmOutStateModeError int // Sequence error // i.e sequence number overflow XfrmOutStateSeqError int // State is expired XfrmOutStateExpired int // Policy discads XfrmOutPolBlock int // Policy is dead XfrmOutPolDead int // Policy Error XfrmOutPolError int XfrmFwdHdrError int XfrmOutStateInvalid int XfrmAcquireError int } // NewXfrmStat reads the xfrm_stat statistics. func NewXfrmStat() (XfrmStat, error) { fs, err := NewFS(DefaultMountPoint) if err != nil { return XfrmStat{}, err } return fs.NewXfrmStat() } // NewXfrmStat reads the xfrm_stat statistics from the 'proc' filesystem. func (fs FS) NewXfrmStat() (XfrmStat, error) { file, err := os.Open(fs.Path("net/xfrm_stat")) if err != nil { return XfrmStat{}, err } defer file.Close() var ( x = XfrmStat{} s = bufio.NewScanner(file) ) for s.Scan() { fields := strings.Fields(s.Text()) if len(fields) != 2 { return XfrmStat{}, fmt.Errorf( "couldnt parse %s line %s", file.Name(), s.Text()) } name := fields[0] value, err := strconv.Atoi(fields[1]) if err != nil { return XfrmStat{}, err } switch name { case "XfrmInError": x.XfrmInError = value case "XfrmInBufferError": x.XfrmInBufferError = value case "XfrmInHdrError": x.XfrmInHdrError = value case "XfrmInNoStates": x.XfrmInNoStates = value case "XfrmInStateProtoError": x.XfrmInStateProtoError = value case "XfrmInStateModeError": x.XfrmInStateModeError = value case "XfrmInStateSeqError": x.XfrmInStateSeqError = value case "XfrmInStateExpired": x.XfrmInStateExpired = value case "XfrmInStateInvalid": x.XfrmInStateInvalid = value case "XfrmInTmplMismatch": x.XfrmInTmplMismatch = value case "XfrmInNoPols": x.XfrmInNoPols = value case "XfrmInPolBlock": x.XfrmInPolBlock = value case "XfrmInPolError": x.XfrmInPolError = value case "XfrmOutError": x.XfrmOutError = value case "XfrmInStateMismatch": x.XfrmInStateMismatch = value case "XfrmOutBundleGenError": x.XfrmOutBundleGenError = value case "XfrmOutBundleCheckError": x.XfrmOutBundleCheckError = value case "XfrmOutNoStates": x.XfrmOutNoStates = value case "XfrmOutStateProtoError": x.XfrmOutStateProtoError = value case "XfrmOutStateModeError": x.XfrmOutStateModeError = value case "XfrmOutStateSeqError": x.XfrmOutStateSeqError = value case "XfrmOutStateExpired": x.XfrmOutStateExpired = value case "XfrmOutPolBlock": x.XfrmOutPolBlock = value case "XfrmOutPolDead": x.XfrmOutPolDead = value case "XfrmOutPolError": x.XfrmOutPolError = value case "XfrmFwdHdrError": x.XfrmFwdHdrError = value case "XfrmOutStateInvalid": x.XfrmOutStateInvalid = value case "XfrmAcquireError": x.XfrmAcquireError = value } } return x, s.Err() } ================================================ FILE: vendor/github.com/prometheus/procfs/xfrm_test.go ================================================ // Copyright 2017 Prometheus Team // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package procfs import ( "testing" ) func TestXfrmStats(t *testing.T) { xfrmStats, err := FS("fixtures").NewXfrmStat() if err != nil { t.Fatal(err) } for _, test := range []struct { name string want int got int }{ {name: "XfrmInError", want: 1, got: xfrmStats.XfrmInError}, {name: "XfrmInBufferError", want: 2, got: xfrmStats.XfrmInBufferError}, {name: "XfrmInHdrError", want: 4, got: xfrmStats.XfrmInHdrError}, {name: "XfrmInNoStates", want: 3, got: xfrmStats.XfrmInNoStates}, {name: "XfrmInStateProtoError", want: 40, got: xfrmStats.XfrmInStateProtoError}, {name: "XfrmInStateModeError", want: 100, got: xfrmStats.XfrmInStateModeError}, {name: "XfrmInStateSeqError", want: 6000, got: xfrmStats.XfrmInStateSeqError}, {name: "XfrmInStateExpired", want: 4, got: xfrmStats.XfrmInStateExpired}, {name: "XfrmInStateMismatch", want: 23451, got: xfrmStats.XfrmInStateMismatch}, {name: "XfrmInStateInvalid", want: 55555, got: xfrmStats.XfrmInStateInvalid}, {name: "XfrmInTmplMismatch", want: 51, got: xfrmStats.XfrmInTmplMismatch}, {name: "XfrmInNoPols", want: 65432, got: xfrmStats.XfrmInNoPols}, {name: "XfrmInPolBlock", want: 100, got: xfrmStats.XfrmInPolBlock}, {name: "XfrmInPolError", want: 10000, got: xfrmStats.XfrmInPolError}, {name: "XfrmOutError", want: 1000000, got: xfrmStats.XfrmOutError}, {name: "XfrmOutBundleGenError", want: 43321, got: xfrmStats.XfrmOutBundleGenError}, {name: "XfrmOutBundleCheckError", want: 555, got: xfrmStats.XfrmOutBundleCheckError}, {name: "XfrmOutNoStates", want: 869, got: xfrmStats.XfrmOutNoStates}, {name: "XfrmOutStateProtoError", want: 4542, got: xfrmStats.XfrmOutStateProtoError}, {name: "XfrmOutStateModeError", want: 4, got: xfrmStats.XfrmOutStateModeError}, {name: "XfrmOutStateSeqError", want: 543, got: xfrmStats.XfrmOutStateSeqError}, {name: "XfrmOutStateExpired", want: 565, got: xfrmStats.XfrmOutStateExpired}, {name: "XfrmOutPolBlock", want: 43456, got: xfrmStats.XfrmOutPolBlock}, {name: "XfrmOutPolDead", want: 7656, got: xfrmStats.XfrmOutPolDead}, {name: "XfrmOutPolError", want: 1454, got: xfrmStats.XfrmOutPolError}, {name: "XfrmFwdHdrError", want: 6654, got: xfrmStats.XfrmFwdHdrError}, {name: "XfrmOutStateInvaliad", want: 28765, got: xfrmStats.XfrmOutStateInvalid}, {name: "XfrmAcquireError", want: 24532, got: xfrmStats.XfrmAcquireError}, {name: "XfrmInStateInvalid", want: 55555, got: xfrmStats.XfrmInStateInvalid}, {name: "XfrmOutError", want: 1000000, got: xfrmStats.XfrmOutError}, } { if test.want != test.got { t.Errorf("Want %s %d, have %d", test.name, test.want, test.got) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/xfs/parse.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package xfs import ( "bufio" "fmt" "io" "strconv" "strings" ) // ParseStats parses a Stats from an input io.Reader, using the format // found in /proc/fs/xfs/stat. func ParseStats(r io.Reader) (*Stats, error) { const ( // Fields parsed into stats structures. fieldExtentAlloc = "extent_alloc" fieldAbt = "abt" fieldBlkMap = "blk_map" fieldBmbt = "bmbt" fieldDir = "dir" fieldTrans = "trans" fieldIg = "ig" fieldLog = "log" fieldRw = "rw" fieldAttr = "attr" fieldIcluster = "icluster" fieldVnodes = "vnodes" fieldBuf = "buf" fieldXpc = "xpc" // Unimplemented at this time due to lack of documentation. fieldPushAil = "push_ail" fieldXstrat = "xstrat" fieldAbtb2 = "abtb2" fieldAbtc2 = "abtc2" fieldBmbt2 = "bmbt2" fieldIbt2 = "ibt2" fieldFibt2 = "fibt2" fieldQm = "qm" fieldDebug = "debug" ) var xfss Stats s := bufio.NewScanner(r) for s.Scan() { // Expect at least a string label and a single integer value, ex: // - abt 0 // - rw 1 2 ss := strings.Fields(string(s.Bytes())) if len(ss) < 2 { continue } label := ss[0] // Extended precision counters are uint64 values. if label == fieldXpc { us, err := parseUint64s(ss[1:]) if err != nil { return nil, err } xfss.ExtendedPrecision, err = extendedPrecisionStats(us) if err != nil { return nil, err } continue } // All other counters are uint32 values. us, err := parseUint32s(ss[1:]) if err != nil { return nil, err } switch label { case fieldExtentAlloc: xfss.ExtentAllocation, err = extentAllocationStats(us) case fieldAbt: xfss.AllocationBTree, err = btreeStats(us) case fieldBlkMap: xfss.BlockMapping, err = blockMappingStats(us) case fieldBmbt: xfss.BlockMapBTree, err = btreeStats(us) case fieldDir: xfss.DirectoryOperation, err = directoryOperationStats(us) case fieldTrans: xfss.Transaction, err = transactionStats(us) case fieldIg: xfss.InodeOperation, err = inodeOperationStats(us) case fieldLog: xfss.LogOperation, err = logOperationStats(us) case fieldRw: xfss.ReadWrite, err = readWriteStats(us) case fieldAttr: xfss.AttributeOperation, err = attributeOperationStats(us) case fieldIcluster: xfss.InodeClustering, err = inodeClusteringStats(us) case fieldVnodes: xfss.Vnode, err = vnodeStats(us) case fieldBuf: xfss.Buffer, err = bufferStats(us) } if err != nil { return nil, err } } return &xfss, s.Err() } // extentAllocationStats builds an ExtentAllocationStats from a slice of uint32s. func extentAllocationStats(us []uint32) (ExtentAllocationStats, error) { if l := len(us); l != 4 { return ExtentAllocationStats{}, fmt.Errorf("incorrect number of values for XFS extent allocation stats: %d", l) } return ExtentAllocationStats{ ExtentsAllocated: us[0], BlocksAllocated: us[1], ExtentsFreed: us[2], BlocksFreed: us[3], }, nil } // btreeStats builds a BTreeStats from a slice of uint32s. func btreeStats(us []uint32) (BTreeStats, error) { if l := len(us); l != 4 { return BTreeStats{}, fmt.Errorf("incorrect number of values for XFS btree stats: %d", l) } return BTreeStats{ Lookups: us[0], Compares: us[1], RecordsInserted: us[2], RecordsDeleted: us[3], }, nil } // BlockMappingStat builds a BlockMappingStats from a slice of uint32s. func blockMappingStats(us []uint32) (BlockMappingStats, error) { if l := len(us); l != 7 { return BlockMappingStats{}, fmt.Errorf("incorrect number of values for XFS block mapping stats: %d", l) } return BlockMappingStats{ Reads: us[0], Writes: us[1], Unmaps: us[2], ExtentListInsertions: us[3], ExtentListDeletions: us[4], ExtentListLookups: us[5], ExtentListCompares: us[6], }, nil } // DirectoryOperationStats builds a DirectoryOperationStats from a slice of uint32s. func directoryOperationStats(us []uint32) (DirectoryOperationStats, error) { if l := len(us); l != 4 { return DirectoryOperationStats{}, fmt.Errorf("incorrect number of values for XFS directory operation stats: %d", l) } return DirectoryOperationStats{ Lookups: us[0], Creates: us[1], Removes: us[2], Getdents: us[3], }, nil } // TransactionStats builds a TransactionStats from a slice of uint32s. func transactionStats(us []uint32) (TransactionStats, error) { if l := len(us); l != 3 { return TransactionStats{}, fmt.Errorf("incorrect number of values for XFS transaction stats: %d", l) } return TransactionStats{ Sync: us[0], Async: us[1], Empty: us[2], }, nil } // InodeOperationStats builds an InodeOperationStats from a slice of uint32s. func inodeOperationStats(us []uint32) (InodeOperationStats, error) { if l := len(us); l != 7 { return InodeOperationStats{}, fmt.Errorf("incorrect number of values for XFS inode operation stats: %d", l) } return InodeOperationStats{ Attempts: us[0], Found: us[1], Recycle: us[2], Missed: us[3], Duplicate: us[4], Reclaims: us[5], AttributeChange: us[6], }, nil } // LogOperationStats builds a LogOperationStats from a slice of uint32s. func logOperationStats(us []uint32) (LogOperationStats, error) { if l := len(us); l != 5 { return LogOperationStats{}, fmt.Errorf("incorrect number of values for XFS log operation stats: %d", l) } return LogOperationStats{ Writes: us[0], Blocks: us[1], NoInternalBuffers: us[2], Force: us[3], ForceSleep: us[4], }, nil } // ReadWriteStats builds a ReadWriteStats from a slice of uint32s. func readWriteStats(us []uint32) (ReadWriteStats, error) { if l := len(us); l != 2 { return ReadWriteStats{}, fmt.Errorf("incorrect number of values for XFS read write stats: %d", l) } return ReadWriteStats{ Read: us[0], Write: us[1], }, nil } // AttributeOperationStats builds an AttributeOperationStats from a slice of uint32s. func attributeOperationStats(us []uint32) (AttributeOperationStats, error) { if l := len(us); l != 4 { return AttributeOperationStats{}, fmt.Errorf("incorrect number of values for XFS attribute operation stats: %d", l) } return AttributeOperationStats{ Get: us[0], Set: us[1], Remove: us[2], List: us[3], }, nil } // InodeClusteringStats builds an InodeClusteringStats from a slice of uint32s. func inodeClusteringStats(us []uint32) (InodeClusteringStats, error) { if l := len(us); l != 3 { return InodeClusteringStats{}, fmt.Errorf("incorrect number of values for XFS inode clustering stats: %d", l) } return InodeClusteringStats{ Iflush: us[0], Flush: us[1], FlushInode: us[2], }, nil } // VnodeStats builds a VnodeStats from a slice of uint32s. func vnodeStats(us []uint32) (VnodeStats, error) { // The attribute "Free" appears to not be available on older XFS // stats versions. Therefore, 7 or 8 elements may appear in // this slice. l := len(us) if l != 7 && l != 8 { return VnodeStats{}, fmt.Errorf("incorrect number of values for XFS vnode stats: %d", l) } s := VnodeStats{ Active: us[0], Allocate: us[1], Get: us[2], Hold: us[3], Release: us[4], Reclaim: us[5], Remove: us[6], } // Skip adding free, unless it is present. The zero value will // be used in place of an actual count. if l == 7 { return s, nil } s.Free = us[7] return s, nil } // BufferStats builds a BufferStats from a slice of uint32s. func bufferStats(us []uint32) (BufferStats, error) { if l := len(us); l != 9 { return BufferStats{}, fmt.Errorf("incorrect number of values for XFS buffer stats: %d", l) } return BufferStats{ Get: us[0], Create: us[1], GetLocked: us[2], GetLockedWaited: us[3], BusyLocked: us[4], MissLocked: us[5], PageRetries: us[6], PageFound: us[7], GetRead: us[8], }, nil } // ExtendedPrecisionStats builds an ExtendedPrecisionStats from a slice of uint32s. func extendedPrecisionStats(us []uint64) (ExtendedPrecisionStats, error) { if l := len(us); l != 3 { return ExtendedPrecisionStats{}, fmt.Errorf("incorrect number of values for XFS extended precision stats: %d", l) } return ExtendedPrecisionStats{ FlushBytes: us[0], WriteBytes: us[1], ReadBytes: us[2], }, nil } // parseUint32s parses a slice of strings into a slice of uint32s. func parseUint32s(ss []string) ([]uint32, error) { us := make([]uint32, 0, len(ss)) for _, s := range ss { u, err := strconv.ParseUint(s, 10, 32) if err != nil { return nil, err } us = append(us, uint32(u)) } return us, nil } // parseUint64s parses a slice of strings into a slice of uint64s. func parseUint64s(ss []string) ([]uint64, error) { us := make([]uint64, 0, len(ss)) for _, s := range ss { u, err := strconv.ParseUint(s, 10, 64) if err != nil { return nil, err } us = append(us, u) } return us, nil } ================================================ FILE: vendor/github.com/prometheus/procfs/xfs/parse_test.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package xfs_test import ( "reflect" "strings" "testing" "github.com/prometheus/procfs" "github.com/prometheus/procfs/xfs" ) func TestParseStats(t *testing.T) { tests := []struct { name string s string fs bool stats *xfs.Stats invalid bool }{ { name: "empty file OK", }, { name: "short or empty lines and unknown labels ignored", s: "one\n\ntwo 1 2 3\n", stats: &xfs.Stats{}, }, { name: "bad uint32", s: "extent_alloc XXX", invalid: true, }, { name: "bad uint64", s: "xpc XXX", invalid: true, }, { name: "extent_alloc bad", s: "extent_alloc 1", invalid: true, }, { name: "extent_alloc OK", s: "extent_alloc 1 2 3 4", stats: &xfs.Stats{ ExtentAllocation: xfs.ExtentAllocationStats{ ExtentsAllocated: 1, BlocksAllocated: 2, ExtentsFreed: 3, BlocksFreed: 4, }, }, }, { name: "abt bad", s: "abt 1", invalid: true, }, { name: "abt OK", s: "abt 1 2 3 4", stats: &xfs.Stats{ AllocationBTree: xfs.BTreeStats{ Lookups: 1, Compares: 2, RecordsInserted: 3, RecordsDeleted: 4, }, }, }, { name: "blk_map bad", s: "blk_map 1", invalid: true, }, { name: "blk_map OK", s: "blk_map 1 2 3 4 5 6 7", stats: &xfs.Stats{ BlockMapping: xfs.BlockMappingStats{ Reads: 1, Writes: 2, Unmaps: 3, ExtentListInsertions: 4, ExtentListDeletions: 5, ExtentListLookups: 6, ExtentListCompares: 7, }, }, }, { name: "bmbt bad", s: "bmbt 1", invalid: true, }, { name: "bmbt OK", s: "bmbt 1 2 3 4", stats: &xfs.Stats{ BlockMapBTree: xfs.BTreeStats{ Lookups: 1, Compares: 2, RecordsInserted: 3, RecordsDeleted: 4, }, }, }, { name: "dir bad", s: "dir 1", invalid: true, }, { name: "dir OK", s: "dir 1 2 3 4", stats: &xfs.Stats{ DirectoryOperation: xfs.DirectoryOperationStats{ Lookups: 1, Creates: 2, Removes: 3, Getdents: 4, }, }, }, { name: "trans bad", s: "trans 1", invalid: true, }, { name: "trans OK", s: "trans 1 2 3", stats: &xfs.Stats{ Transaction: xfs.TransactionStats{ Sync: 1, Async: 2, Empty: 3, }, }, }, { name: "ig bad", s: "ig 1", invalid: true, }, { name: "ig OK", s: "ig 1 2 3 4 5 6 7", stats: &xfs.Stats{ InodeOperation: xfs.InodeOperationStats{ Attempts: 1, Found: 2, Recycle: 3, Missed: 4, Duplicate: 5, Reclaims: 6, AttributeChange: 7, }, }, }, { name: "log bad", s: "log 1", invalid: true, }, { name: "log OK", s: "log 1 2 3 4 5", stats: &xfs.Stats{ LogOperation: xfs.LogOperationStats{ Writes: 1, Blocks: 2, NoInternalBuffers: 3, Force: 4, ForceSleep: 5, }, }, }, { name: "rw bad", s: "rw 1", invalid: true, }, { name: "rw OK", s: "rw 1 2", stats: &xfs.Stats{ ReadWrite: xfs.ReadWriteStats{ Read: 1, Write: 2, }, }, }, { name: "attr bad", s: "attr 1", invalid: true, }, { name: "attr OK", s: "attr 1 2 3 4", stats: &xfs.Stats{ AttributeOperation: xfs.AttributeOperationStats{ Get: 1, Set: 2, Remove: 3, List: 4, }, }, }, { name: "icluster bad", s: "icluster 1", invalid: true, }, { name: "icluster OK", s: "icluster 1 2 3", stats: &xfs.Stats{ InodeClustering: xfs.InodeClusteringStats{ Iflush: 1, Flush: 2, FlushInode: 3, }, }, }, { name: "vnodes bad", s: "vnodes 1", invalid: true, }, { name: "vnodes (missing free) OK", s: "vnodes 1 2 3 4 5 6 7", stats: &xfs.Stats{ Vnode: xfs.VnodeStats{ Active: 1, Allocate: 2, Get: 3, Hold: 4, Release: 5, Reclaim: 6, Remove: 7, }, }, }, { name: "vnodes (with free) OK", s: "vnodes 1 2 3 4 5 6 7 8", stats: &xfs.Stats{ Vnode: xfs.VnodeStats{ Active: 1, Allocate: 2, Get: 3, Hold: 4, Release: 5, Reclaim: 6, Remove: 7, Free: 8, }, }, }, { name: "buf bad", s: "buf 1", invalid: true, }, { name: "buf OK", s: "buf 1 2 3 4 5 6 7 8 9", stats: &xfs.Stats{ Buffer: xfs.BufferStats{ Get: 1, Create: 2, GetLocked: 3, GetLockedWaited: 4, BusyLocked: 5, MissLocked: 6, PageRetries: 7, PageFound: 8, GetRead: 9, }, }, }, { name: "xpc bad", s: "xpc 1", invalid: true, }, { name: "xpc OK", s: "xpc 1 2 3", stats: &xfs.Stats{ ExtendedPrecision: xfs.ExtendedPrecisionStats{ FlushBytes: 1, WriteBytes: 2, ReadBytes: 3, }, }, }, { name: "fixtures OK", fs: true, stats: &xfs.Stats{ ExtentAllocation: xfs.ExtentAllocationStats{ ExtentsAllocated: 92447, BlocksAllocated: 97589, ExtentsFreed: 92448, BlocksFreed: 93751, }, AllocationBTree: xfs.BTreeStats{ Lookups: 0, Compares: 0, RecordsInserted: 0, RecordsDeleted: 0, }, BlockMapping: xfs.BlockMappingStats{ Reads: 1767055, Writes: 188820, Unmaps: 184891, ExtentListInsertions: 92447, ExtentListDeletions: 92448, ExtentListLookups: 2140766, ExtentListCompares: 0, }, BlockMapBTree: xfs.BTreeStats{ Lookups: 0, Compares: 0, RecordsInserted: 0, RecordsDeleted: 0, }, DirectoryOperation: xfs.DirectoryOperationStats{ Lookups: 185039, Creates: 92447, Removes: 92444, Getdents: 136422, }, Transaction: xfs.TransactionStats{ Sync: 706, Async: 944304, Empty: 0, }, InodeOperation: xfs.InodeOperationStats{ Attempts: 185045, Found: 58807, Recycle: 0, Missed: 126238, Duplicate: 0, Reclaims: 33637, AttributeChange: 22, }, LogOperation: xfs.LogOperationStats{ Writes: 2883, Blocks: 113448, NoInternalBuffers: 9, Force: 17360, ForceSleep: 739, }, ReadWrite: xfs.ReadWriteStats{ Read: 107739, Write: 94045, }, AttributeOperation: xfs.AttributeOperationStats{ Get: 4, Set: 0, Remove: 0, List: 0, }, InodeClustering: xfs.InodeClusteringStats{ Iflush: 8677, Flush: 7849, FlushInode: 135802, }, Vnode: xfs.VnodeStats{ Active: 92601, Allocate: 0, Get: 0, Hold: 0, Release: 92444, Reclaim: 92444, Remove: 92444, Free: 0, }, Buffer: xfs.BufferStats{ Get: 2666287, Create: 7122, GetLocked: 2659202, GetLockedWaited: 3599, BusyLocked: 2, MissLocked: 7085, PageRetries: 0, PageFound: 10297, GetRead: 7085, }, ExtendedPrecision: xfs.ExtendedPrecisionStats{ FlushBytes: 399724544, WriteBytes: 92823103, ReadBytes: 86219234, }, }, }, } for _, tt := range tests { var ( stats *xfs.Stats err error ) if tt.s != "" { stats, err = xfs.ParseStats(strings.NewReader(tt.s)) } if tt.fs { stats, err = procfs.FS("../fixtures").XFSStats() } if tt.invalid && err == nil { t.Error("expected an error, but none occurred") } if !tt.invalid && err != nil { t.Errorf("unexpected error: %v", err) } if want, have := tt.stats, stats; !reflect.DeepEqual(want, have) { t.Errorf("unexpected XFS stats:\nwant:\n%v\nhave:\n%v", want, have) } } } ================================================ FILE: vendor/github.com/prometheus/procfs/xfs/xfs.go ================================================ // Copyright 2017 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package xfs provides access to statistics exposed by the XFS filesystem. package xfs // Stats contains XFS filesystem runtime statistics, parsed from // /proc/fs/xfs/stat. // // The names and meanings of each statistic were taken from // http://xfs.org/index.php/Runtime_Stats and xfs_stats.h in the Linux // kernel source. Most counters are uint32s (same data types used in // xfs_stats.h), but some of the "extended precision stats" are uint64s. type Stats struct { // The name of the filesystem used to source these statistics. // If empty, this indicates aggregated statistics for all XFS // filesystems on the host. Name string ExtentAllocation ExtentAllocationStats AllocationBTree BTreeStats BlockMapping BlockMappingStats BlockMapBTree BTreeStats DirectoryOperation DirectoryOperationStats Transaction TransactionStats InodeOperation InodeOperationStats LogOperation LogOperationStats ReadWrite ReadWriteStats AttributeOperation AttributeOperationStats InodeClustering InodeClusteringStats Vnode VnodeStats Buffer BufferStats ExtendedPrecision ExtendedPrecisionStats } // ExtentAllocationStats contains statistics regarding XFS extent allocations. type ExtentAllocationStats struct { ExtentsAllocated uint32 BlocksAllocated uint32 ExtentsFreed uint32 BlocksFreed uint32 } // BTreeStats contains statistics regarding an XFS internal B-tree. type BTreeStats struct { Lookups uint32 Compares uint32 RecordsInserted uint32 RecordsDeleted uint32 } // BlockMappingStats contains statistics regarding XFS block maps. type BlockMappingStats struct { Reads uint32 Writes uint32 Unmaps uint32 ExtentListInsertions uint32 ExtentListDeletions uint32 ExtentListLookups uint32 ExtentListCompares uint32 } // DirectoryOperationStats contains statistics regarding XFS directory entries. type DirectoryOperationStats struct { Lookups uint32 Creates uint32 Removes uint32 Getdents uint32 } // TransactionStats contains statistics regarding XFS metadata transactions. type TransactionStats struct { Sync uint32 Async uint32 Empty uint32 } // InodeOperationStats contains statistics regarding XFS inode operations. type InodeOperationStats struct { Attempts uint32 Found uint32 Recycle uint32 Missed uint32 Duplicate uint32 Reclaims uint32 AttributeChange uint32 } // LogOperationStats contains statistics regarding the XFS log buffer. type LogOperationStats struct { Writes uint32 Blocks uint32 NoInternalBuffers uint32 Force uint32 ForceSleep uint32 } // ReadWriteStats contains statistics regarding the number of read and write // system calls for XFS filesystems. type ReadWriteStats struct { Read uint32 Write uint32 } // AttributeOperationStats contains statistics regarding manipulation of // XFS extended file attributes. type AttributeOperationStats struct { Get uint32 Set uint32 Remove uint32 List uint32 } // InodeClusteringStats contains statistics regarding XFS inode clustering // operations. type InodeClusteringStats struct { Iflush uint32 Flush uint32 FlushInode uint32 } // VnodeStats contains statistics regarding XFS vnode operations. type VnodeStats struct { Active uint32 Allocate uint32 Get uint32 Hold uint32 Release uint32 Reclaim uint32 Remove uint32 Free uint32 } // BufferStats contains statistics regarding XFS read/write I/O buffers. type BufferStats struct { Get uint32 Create uint32 GetLocked uint32 GetLockedWaited uint32 BusyLocked uint32 MissLocked uint32 PageRetries uint32 PageFound uint32 GetRead uint32 } // ExtendedPrecisionStats contains high precision counters used to track the // total number of bytes read, written, or flushed, during XFS operations. type ExtendedPrecisionStats struct { FlushBytes uint64 WriteBytes uint64 ReadBytes uint64 } ================================================ FILE: vendor/github.com/stretchr/testify/.gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe .DS_Store ================================================ FILE: vendor/github.com/stretchr/testify/.travis.yml ================================================ language: go sudo: false go: - 1.1 - 1.2 - 1.3 - 1.4 - 1.5 - 1.6 - 1.7 - tip script: - go test -v ./... ================================================ FILE: vendor/github.com/stretchr/testify/LICENCE.txt ================================================ Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell Please consider promoting this project if you find it useful. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/stretchr/testify/LICENSE ================================================ Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell Please consider promoting this project if you find it useful. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/stretchr/testify/README.md ================================================ Testify - Thou Shalt Write Tests ================================ [![Build Status](https://travis-ci.org/stretchr/testify.svg)](https://travis-ci.org/stretchr/testify) [![Go Report Card](https://goreportcard.com/badge/github.com/stretchr/testify)](https://goreportcard.com/report/github.com/stretchr/testify) [![GoDoc](https://godoc.org/github.com/stretchr/testify?status.svg)](https://godoc.org/github.com/stretchr/testify) Go code (golang) set of packages that provide many tools for testifying that your code will behave as you intend. Features include: * [Easy assertions](#assert-package) * [Mocking](#mock-package) * [HTTP response trapping](#http-package) * [Testing suite interfaces and functions](#suite-package) Get started: * Install testify with [one line of code](#installation), or [update it with another](#staying-up-to-date) * For an introduction to writing test code in Go, see http://golang.org/doc/code.html#Testing * Check out the API Documentation http://godoc.org/github.com/stretchr/testify * To make your testing life easier, check out our other project, [gorc](http://github.com/stretchr/gorc) * A little about [Test-Driven Development (TDD)](http://en.wikipedia.org/wiki/Test-driven_development) [`assert`](http://godoc.org/github.com/stretchr/testify/assert "API documentation") package ------------------------------------------------------------------------------------------- The `assert` package provides some helpful methods that allow you to write better test code in Go. * Prints friendly, easy to read failure descriptions * Allows for very readable code * Optionally annotate each assertion with a message See it in action: ```go package yours import ( "testing" "github.com/stretchr/testify/assert" ) func TestSomething(t *testing.T) { // assert equality assert.Equal(t, 123, 123, "they should be equal") // assert inequality assert.NotEqual(t, 123, 456, "they should not be equal") // assert for nil (good for errors) assert.Nil(t, object) // assert for not nil (good when you expect something) if assert.NotNil(t, object) { // now we know that object isn't nil, we are safe to make // further assertions without causing any errors assert.Equal(t, "Something", object.Value) } } ``` * Every assert func takes the `testing.T` object as the first argument. This is how it writes the errors out through the normal `go test` capabilities. * Every assert func returns a bool indicating whether the assertion was successful or not, this is useful for if you want to go on making further assertions under certain conditions. if you assert many times, use the below: ```go package yours import ( "testing" "github.com/stretchr/testify/assert" ) func TestSomething(t *testing.T) { assert := assert.New(t) // assert equality assert.Equal(123, 123, "they should be equal") // assert inequality assert.NotEqual(123, 456, "they should not be equal") // assert for nil (good for errors) assert.Nil(object) // assert for not nil (good when you expect something) if assert.NotNil(object) { // now we know that object isn't nil, we are safe to make // further assertions without causing any errors assert.Equal("Something", object.Value) } } ``` [`require`](http://godoc.org/github.com/stretchr/testify/require "API documentation") package --------------------------------------------------------------------------------------------- The `require` package provides same global functions as the `assert` package, but instead of returning a boolean result they terminate current test. See [t.FailNow](http://golang.org/pkg/testing/#T.FailNow) for details. [`http`](http://godoc.org/github.com/stretchr/testify/http "API documentation") package --------------------------------------------------------------------------------------- The `http` package contains test objects useful for testing code that relies on the `net/http` package. Check out the [(deprecated) API documentation for the `http` package](http://godoc.org/github.com/stretchr/testify/http). We recommend you use [httptest](http://golang.org/pkg/net/http/httptest) instead. [`mock`](http://godoc.org/github.com/stretchr/testify/mock "API documentation") package ---------------------------------------------------------------------------------------- The `mock` package provides a mechanism for easily writing mock objects that can be used in place of real objects when writing test code. An example test function that tests a piece of code that relies on an external object `testObj`, can setup expectations (testify) and assert that they indeed happened: ```go package yours import ( "testing" "github.com/stretchr/testify/mock" ) /* Test objects */ // MyMockedObject is a mocked object that implements an interface // that describes an object that the code I am testing relies on. type MyMockedObject struct{ mock.Mock } // DoSomething is a method on MyMockedObject that implements some interface // and just records the activity, and returns what the Mock object tells it to. // // In the real object, this method would do something useful, but since this // is a mocked object - we're just going to stub it out. // // NOTE: This method is not being tested here, code that uses this object is. func (m *MyMockedObject) DoSomething(number int) (bool, error) { args := m.Called(number) return args.Bool(0), args.Error(1) } /* Actual test functions */ // TestSomething is an example of how to use our test object to // make assertions about some target code we are testing. func TestSomething(t *testing.T) { // create an instance of our test object testObj := new(MyMockedObject) // setup expectations testObj.On("DoSomething", 123).Return(true, nil) // call the code we are testing targetFuncThatDoesSomethingWithObj(testObj) // assert that the expectations were met testObj.AssertExpectations(t) } ``` For more information on how to write mock code, check out the [API documentation for the `mock` package](http://godoc.org/github.com/stretchr/testify/mock). You can use the [mockery tool](http://github.com/vektra/mockery) to autogenerate the mock code against an interface as well, making using mocks much quicker. [`suite`](http://godoc.org/github.com/stretchr/testify/suite "API documentation") package ----------------------------------------------------------------------------------------- The `suite` package provides functionality that you might be used to from more common object oriented languages. With it, you can build a testing suite as a struct, build setup/teardown methods and testing methods on your struct, and run them with 'go test' as per normal. An example suite is shown below: ```go // Basic imports import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) // Define the suite, and absorb the built-in basic suite // functionality from testify - including a T() method which // returns the current testing context type ExampleTestSuite struct { suite.Suite VariableThatShouldStartAtFive int } // Make sure that VariableThatShouldStartAtFive is set to five // before each test func (suite *ExampleTestSuite) SetupTest() { suite.VariableThatShouldStartAtFive = 5 } // All methods that begin with "Test" are run as tests within a // suite. func (suite *ExampleTestSuite) TestExample() { assert.Equal(suite.T(), 5, suite.VariableThatShouldStartAtFive) } // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestExampleTestSuite(t *testing.T) { suite.Run(t, new(ExampleTestSuite)) } ``` For a more complete example, using all of the functionality provided by the suite package, look at our [example testing suite](https://github.com/stretchr/testify/blob/master/suite/suite_test.go) For more information on writing suites, check out the [API documentation for the `suite` package](http://godoc.org/github.com/stretchr/testify/suite). `Suite` object has assertion methods: ```go // Basic imports import ( "testing" "github.com/stretchr/testify/suite" ) // Define the suite, and absorb the built-in basic suite // functionality from testify - including assertion methods. type ExampleTestSuite struct { suite.Suite VariableThatShouldStartAtFive int } // Make sure that VariableThatShouldStartAtFive is set to five // before each test func (suite *ExampleTestSuite) SetupTest() { suite.VariableThatShouldStartAtFive = 5 } // All methods that begin with "Test" are run as tests within a // suite. func (suite *ExampleTestSuite) TestExample() { suite.Equal(suite.VariableThatShouldStartAtFive, 5) } // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestExampleTestSuite(t *testing.T) { suite.Run(t, new(ExampleTestSuite)) } ``` ------ Installation ============ To install Testify, use `go get`: * Latest version: go get github.com/stretchr/testify * Specific version: go get gopkg.in/stretchr/testify.v1 This will then make the following packages available to you: github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/http Import the `testify/assert` package into your code using this template: ```go package yours import ( "testing" "github.com/stretchr/testify/assert" ) func TestSomething(t *testing.T) { assert.True(t, true, "True is true!") } ``` ------ Staying up to date ================== To update Testify to the latest version, use `go get -u github.com/stretchr/testify`. ------ Version History =============== * 1.0 - New package versioning strategy adopted. ------ Contributing ============ Please feel free to submit issues, fork the repository and send pull requests! When submitting an issue, we ask that you please include a complete test function that demonstrates the issue. Extra credit for those using Testify to write the test code that demonstrates it. ------ Licence ======= Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell Please consider promoting this project if you find it useful. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vendor/github.com/stretchr/testify/assert/assertion_forward.go ================================================ /* * CODE GENERATED AUTOMATICALLY WITH github.com/stretchr/testify/_codegen * THIS FILE MUST NOT BE EDITED BY HAND */ package assert import ( http "net/http" url "net/url" time "time" ) // Condition uses a Comparison to assert a complex condition. func (a *Assertions) Condition(comp Comparison, msgAndArgs ...interface{}) bool { return Condition(a.t, comp, msgAndArgs...) } // Contains asserts that the specified string, list(array, slice...) or map contains the // specified substring or element. // // a.Contains("Hello World", "World", "But 'Hello World' does contain 'World'") // a.Contains(["Hello", "World"], "World", "But ["Hello", "World"] does contain 'World'") // a.Contains({"Hello": "World"}, "Hello", "But {'Hello': 'World'} does contain 'Hello'") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { return Contains(a.t, s, contains, msgAndArgs...) } // Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // a.Empty(obj) // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { return Empty(a.t, object, msgAndArgs...) } // Equal asserts that two objects are equal. // // a.Equal(123, 123, "123 and 123 should be equal") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { return Equal(a.t, expected, actual, msgAndArgs...) } // EqualError asserts that a function returned an error (i.e. not `nil`) // and that it is equal to the provided error. // // actualObj, err := SomeFunction() // if assert.Error(t, err, "An error was expected") { // assert.Equal(t, err, expectedError) // } // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) bool { return EqualError(a.t, theError, errString, msgAndArgs...) } // EqualValues asserts that two objects are equal or convertable to the same types // and equal. // // a.EqualValues(uint32(123), int32(123), "123 and 123 should be equal") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { return EqualValues(a.t, expected, actual, msgAndArgs...) } // Error asserts that a function returned an error (i.e. not `nil`). // // actualObj, err := SomeFunction() // if a.Error(err, "An error was expected") { // assert.Equal(t, err, expectedError) // } // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool { return Error(a.t, err, msgAndArgs...) } // Exactly asserts that two objects are equal is value and type. // // a.Exactly(int32(123), int64(123), "123 and 123 should NOT be equal") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { return Exactly(a.t, expected, actual, msgAndArgs...) } // Fail reports a failure through func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) bool { return Fail(a.t, failureMessage, msgAndArgs...) } // FailNow fails test func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) bool { return FailNow(a.t, failureMessage, msgAndArgs...) } // False asserts that the specified value is false. // // a.False(myBool, "myBool should be false") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) False(value bool, msgAndArgs ...interface{}) bool { return False(a.t, value, msgAndArgs...) } // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // // a.HTTPBodyContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { return HTTPBodyContains(a.t, handler, method, url, values, str) } // HTTPBodyNotContains asserts that a specified handler returns a // body that does not contain a string. // // a.HTTPBodyNotContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { return HTTPBodyNotContains(a.t, handler, method, url, values, str) } // HTTPError asserts that a specified handler returns an error status code. // // a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values) bool { return HTTPError(a.t, handler, method, url, values) } // HTTPRedirect asserts that a specified handler returns a redirect status code. // // a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values) bool { return HTTPRedirect(a.t, handler, method, url, values) } // HTTPSuccess asserts that a specified handler returns a success status code. // // a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values) bool { return HTTPSuccess(a.t, handler, method, url, values) } // Implements asserts that an object is implemented by the specified interface. // // a.Implements((*MyInterface)(nil), new(MyObject), "MyObject") func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { return Implements(a.t, interfaceObject, object, msgAndArgs...) } // InDelta asserts that the two numerals are within delta of each other. // // a.InDelta(math.Pi, (22 / 7.0), 0.01) // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { return InDelta(a.t, expected, actual, delta, msgAndArgs...) } // InDeltaSlice is the same as InDelta, except it compares two slices. func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { return InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) } // InEpsilon asserts that expected and actual have a relative error less than epsilon // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { return InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) } // InEpsilonSlice is the same as InEpsilon, except it compares two slices. func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { return InEpsilonSlice(a.t, expected, actual, delta, msgAndArgs...) } // IsType asserts that the specified objects are of the same type. func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { return IsType(a.t, expectedType, object, msgAndArgs...) } // JSONEq asserts that two JSON strings are equivalent. // // a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) bool { return JSONEq(a.t, expected, actual, msgAndArgs...) } // Len asserts that the specified object has specific length. // Len also fails if the object has a type that len() not accept. // // a.Len(mySlice, 3, "The size of slice is not 3") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) bool { return Len(a.t, object, length, msgAndArgs...) } // Nil asserts that the specified object is nil. // // a.Nil(err, "err should be nothing") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) bool { return Nil(a.t, object, msgAndArgs...) } // NoError asserts that a function returned no error (i.e. `nil`). // // actualObj, err := SomeFunction() // if a.NoError(err) { // assert.Equal(t, actualObj, expectedObj) // } // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) bool { return NoError(a.t, err, msgAndArgs...) } // NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the // specified substring or element. // // a.NotContains("Hello World", "Earth", "But 'Hello World' does NOT contain 'Earth'") // a.NotContains(["Hello", "World"], "Earth", "But ['Hello', 'World'] does NOT contain 'Earth'") // a.NotContains({"Hello": "World"}, "Earth", "But {'Hello': 'World'} does NOT contain 'Earth'") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { return NotContains(a.t, s, contains, msgAndArgs...) } // NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either // a slice or a channel with len == 0. // // if a.NotEmpty(obj) { // assert.Equal(t, "two", obj[1]) // } // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) bool { return NotEmpty(a.t, object, msgAndArgs...) } // NotEqual asserts that the specified values are NOT equal. // // a.NotEqual(obj1, obj2, "two objects shouldn't be equal") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { return NotEqual(a.t, expected, actual, msgAndArgs...) } // NotNil asserts that the specified object is not nil. // // a.NotNil(err, "err should be something") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) bool { return NotNil(a.t, object, msgAndArgs...) } // NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. // // a.NotPanics(func(){ // RemainCalm() // }, "Calling RemainCalm() should NOT panic") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotPanics(f PanicTestFunc, msgAndArgs ...interface{}) bool { return NotPanics(a.t, f, msgAndArgs...) } // NotRegexp asserts that a specified regexp does not match a string. // // a.NotRegexp(regexp.MustCompile("starts"), "it's starting") // a.NotRegexp("^start", "it's not starting") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { return NotRegexp(a.t, rx, str, msgAndArgs...) } // NotZero asserts that i is not the zero value for its type and returns the truth. func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) bool { return NotZero(a.t, i, msgAndArgs...) } // Panics asserts that the code inside the specified PanicTestFunc panics. // // a.Panics(func(){ // GoCrazy() // }, "Calling GoCrazy() should panic") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Panics(f PanicTestFunc, msgAndArgs ...interface{}) bool { return Panics(a.t, f, msgAndArgs...) } // Regexp asserts that a specified regexp matches a string. // // a.Regexp(regexp.MustCompile("start"), "it's starting") // a.Regexp("start...$", "it's not starting") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { return Regexp(a.t, rx, str, msgAndArgs...) } // True asserts that the specified value is true. // // a.True(myBool, "myBool should be true") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) True(value bool, msgAndArgs ...interface{}) bool { return True(a.t, value, msgAndArgs...) } // WithinDuration asserts that the two times are within duration delta of each other. // // a.WithinDuration(time.Now(), time.Now(), 10*time.Second, "The difference should not be more than 10s") // // Returns whether the assertion was successful (true) or not (false). func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { return WithinDuration(a.t, expected, actual, delta, msgAndArgs...) } // Zero asserts that i is the zero value for its type and returns the truth. func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) bool { return Zero(a.t, i, msgAndArgs...) } ================================================ FILE: vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl ================================================ {{.CommentWithoutT "a"}} func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool { return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) } ================================================ FILE: vendor/github.com/stretchr/testify/assert/assertions.go ================================================ package assert import ( "bufio" "bytes" "encoding/json" "fmt" "math" "reflect" "regexp" "runtime" "strings" "time" "unicode" "unicode/utf8" "github.com/davecgh/go-spew/spew" "github.com/pmezard/go-difflib/difflib" ) func init() { spew.Config.SortKeys = true } // TestingT is an interface wrapper around *testing.T type TestingT interface { Errorf(format string, args ...interface{}) } // Comparison a custom function that returns true on success and false on failure type Comparison func() (success bool) /* Helper functions */ // ObjectsAreEqual determines if two objects are considered equal. // // This function does no assertion of any kind. func ObjectsAreEqual(expected, actual interface{}) bool { if expected == nil || actual == nil { return expected == actual } return reflect.DeepEqual(expected, actual) } // ObjectsAreEqualValues gets whether two objects are equal, or if their // values are equal. func ObjectsAreEqualValues(expected, actual interface{}) bool { if ObjectsAreEqual(expected, actual) { return true } actualType := reflect.TypeOf(actual) if actualType == nil { return false } expectedValue := reflect.ValueOf(expected) if expectedValue.IsValid() && expectedValue.Type().ConvertibleTo(actualType) { // Attempt comparison after type conversion return reflect.DeepEqual(expectedValue.Convert(actualType).Interface(), actual) } return false } /* CallerInfo is necessary because the assert functions use the testing object internally, causing it to print the file:line of the assert method, rather than where the problem actually occurred in calling code.*/ // CallerInfo returns an array of strings containing the file and line number // of each stack frame leading from the current test to the assert call that // failed. func CallerInfo() []string { pc := uintptr(0) file := "" line := 0 ok := false name := "" callers := []string{} for i := 0; ; i++ { pc, file, line, ok = runtime.Caller(i) if !ok { // The breaks below failed to terminate the loop, and we ran off the // end of the call stack. break } // This is a huge edge case, but it will panic if this is the case, see #180 if file == "" { break } f := runtime.FuncForPC(pc) if f == nil { break } name = f.Name() // testing.tRunner is the standard library function that calls // tests. Subtests are called directly by tRunner, without going through // the Test/Benchmark/Example function that contains the t.Run calls, so // with subtests we should break when we hit tRunner, without adding it // to the list of callers. if name == "testing.tRunner" { break } parts := strings.Split(file, "/") dir := parts[len(parts)-2] file = parts[len(parts)-1] if (dir != "assert" && dir != "mock" && dir != "require") || file == "mock_test.go" { callers = append(callers, fmt.Sprintf("%s:%d", file, line)) } // Drop the package segments := strings.Split(name, ".") name = segments[len(segments)-1] if isTest(name, "Test") || isTest(name, "Benchmark") || isTest(name, "Example") { break } } return callers } // Stolen from the `go test` tool. // isTest tells whether name looks like a test (or benchmark, according to prefix). // It is a Test (say) if there is a character after Test that is not a lower-case letter. // We don't want TesticularCancer. func isTest(name, prefix string) bool { if !strings.HasPrefix(name, prefix) { return false } if len(name) == len(prefix) { // "Test" is ok return true } rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) return !unicode.IsLower(rune) } // getWhitespaceString returns a string that is long enough to overwrite the default // output from the go testing framework. func getWhitespaceString() string { _, file, line, ok := runtime.Caller(1) if !ok { return "" } parts := strings.Split(file, "/") file = parts[len(parts)-1] return strings.Repeat(" ", len(fmt.Sprintf("%s:%d: ", file, line))) } func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { if len(msgAndArgs) == 0 || msgAndArgs == nil { return "" } if len(msgAndArgs) == 1 { return msgAndArgs[0].(string) } if len(msgAndArgs) > 1 { return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) } return "" } // Indents all lines of the message by appending a number of tabs to each line, in an output format compatible with Go's // test printing (see inner comment for specifics) func indentMessageLines(message string, tabs int) string { outBuf := new(bytes.Buffer) for i, scanner := 0, bufio.NewScanner(strings.NewReader(message)); scanner.Scan(); i++ { if i != 0 { outBuf.WriteRune('\n') } for ii := 0; ii < tabs; ii++ { outBuf.WriteRune('\t') // Bizarrely, all lines except the first need one fewer tabs prepended, so deliberately advance the counter // by 1 prematurely. if ii == 0 && i > 0 { ii++ } } outBuf.WriteString(scanner.Text()) } return outBuf.String() } type failNower interface { FailNow() } // FailNow fails test func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { Fail(t, failureMessage, msgAndArgs...) // We cannot extend TestingT with FailNow() and // maintain backwards compatibility, so we fallback // to panicking when FailNow is not available in // TestingT. // See issue #263 if t, ok := t.(failNower); ok { t.FailNow() } else { panic("test failed and t is missing `FailNow()`") } return false } // Fail reports a failure through func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { message := messageFromMsgAndArgs(msgAndArgs...) errorTrace := strings.Join(CallerInfo(), "\n\r\t\t\t") if len(message) > 0 { t.Errorf("\r%s\r\tError Trace:\t%s\n"+ "\r\tError:%s\n"+ "\r\tMessages:\t%s\n\r", getWhitespaceString(), errorTrace, indentMessageLines(failureMessage, 2), message) } else { t.Errorf("\r%s\r\tError Trace:\t%s\n"+ "\r\tError:%s\n\r", getWhitespaceString(), errorTrace, indentMessageLines(failureMessage, 2)) } return false } // Implements asserts that an object is implemented by the specified interface. // // assert.Implements(t, (*MyInterface)(nil), new(MyObject), "MyObject") func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { interfaceType := reflect.TypeOf(interfaceObject).Elem() if !reflect.TypeOf(object).Implements(interfaceType) { return Fail(t, fmt.Sprintf("%T must implement %v", object, interfaceType), msgAndArgs...) } return true } // IsType asserts that the specified objects are of the same type. func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { if !ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) { return Fail(t, fmt.Sprintf("Object expected to be of type %v, but was %v", reflect.TypeOf(expectedType), reflect.TypeOf(object)), msgAndArgs...) } return true } // Equal asserts that two objects are equal. // // assert.Equal(t, 123, 123, "123 and 123 should be equal") // // Returns whether the assertion was successful (true) or not (false). func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { if !ObjectsAreEqual(expected, actual) { diff := diff(expected, actual) expected, actual = formatUnequalValues(expected, actual) return Fail(t, fmt.Sprintf("Not equal: %s (expected)\n"+ " != %s (actual)%s", expected, actual, diff), msgAndArgs...) } return true } // formatUnequalValues takes two values of arbitrary types and returns string // representations appropriate to be presented to the user. // // If the values are not of like type, the returned strings will be prefixed // with the type name, and the value will be enclosed in parenthesis similar // to a type conversion in the Go grammar. func formatUnequalValues(expected, actual interface{}) (e string, a string) { aType := reflect.TypeOf(expected) bType := reflect.TypeOf(actual) if aType != bType && isNumericType(aType) && isNumericType(bType) { return fmt.Sprintf("%v(%#v)", aType, expected), fmt.Sprintf("%v(%#v)", bType, actual) } return fmt.Sprintf("%#v", expected), fmt.Sprintf("%#v", actual) } func isNumericType(t reflect.Type) bool { switch t.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return true case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: return true case reflect.Float32, reflect.Float64: return true } return false } // EqualValues asserts that two objects are equal or convertable to the same types // and equal. // // assert.EqualValues(t, uint32(123), int32(123), "123 and 123 should be equal") // // Returns whether the assertion was successful (true) or not (false). func EqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { if !ObjectsAreEqualValues(expected, actual) { return Fail(t, fmt.Sprintf("Not equal: %#v (expected)\n"+ " != %#v (actual)", expected, actual), msgAndArgs...) } return true } // Exactly asserts that two objects are equal is value and type. // // assert.Exactly(t, int32(123), int64(123), "123 and 123 should NOT be equal") // // Returns whether the assertion was successful (true) or not (false). func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { aType := reflect.TypeOf(expected) bType := reflect.TypeOf(actual) if aType != bType { return Fail(t, fmt.Sprintf("Types expected to match exactly\n\r\t%v != %v", aType, bType), msgAndArgs...) } return Equal(t, expected, actual, msgAndArgs...) } // NotNil asserts that the specified object is not nil. // // assert.NotNil(t, err, "err should be something") // // Returns whether the assertion was successful (true) or not (false). func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { if !isNil(object) { return true } return Fail(t, "Expected value not to be nil.", msgAndArgs...) } // isNil checks if a specified object is nil or not, without Failing. func isNil(object interface{}) bool { if object == nil { return true } value := reflect.ValueOf(object) kind := value.Kind() if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { return true } return false } // Nil asserts that the specified object is nil. // // assert.Nil(t, err, "err should be nothing") // // Returns whether the assertion was successful (true) or not (false). func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { if isNil(object) { return true } return Fail(t, fmt.Sprintf("Expected nil, but got: %#v", object), msgAndArgs...) } var numericZeros = []interface{}{ int(0), int8(0), int16(0), int32(0), int64(0), uint(0), uint8(0), uint16(0), uint32(0), uint64(0), float32(0), float64(0), } // isEmpty gets whether the specified object is considered empty or not. func isEmpty(object interface{}) bool { if object == nil { return true } else if object == "" { return true } else if object == false { return true } for _, v := range numericZeros { if object == v { return true } } objValue := reflect.ValueOf(object) switch objValue.Kind() { case reflect.Map: fallthrough case reflect.Slice, reflect.Chan: { return (objValue.Len() == 0) } case reflect.Struct: switch object.(type) { case time.Time: return object.(time.Time).IsZero() } case reflect.Ptr: { if objValue.IsNil() { return true } switch object.(type) { case *time.Time: return object.(*time.Time).IsZero() default: return false } } } return false } // Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // assert.Empty(t, obj) // // Returns whether the assertion was successful (true) or not (false). func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { pass := isEmpty(object) if !pass { Fail(t, fmt.Sprintf("Should be empty, but was %v", object), msgAndArgs...) } return pass } // NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either // a slice or a channel with len == 0. // // if assert.NotEmpty(t, obj) { // assert.Equal(t, "two", obj[1]) // } // // Returns whether the assertion was successful (true) or not (false). func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { pass := !isEmpty(object) if !pass { Fail(t, fmt.Sprintf("Should NOT be empty, but was %v", object), msgAndArgs...) } return pass } // getLen try to get length of object. // return (false, 0) if impossible. func getLen(x interface{}) (ok bool, length int) { v := reflect.ValueOf(x) defer func() { if e := recover(); e != nil { ok = false } }() return true, v.Len() } // Len asserts that the specified object has specific length. // Len also fails if the object has a type that len() not accept. // // assert.Len(t, mySlice, 3, "The size of slice is not 3") // // Returns whether the assertion was successful (true) or not (false). func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) bool { ok, l := getLen(object) if !ok { return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", object), msgAndArgs...) } if l != length { return Fail(t, fmt.Sprintf("\"%s\" should have %d item(s), but has %d", object, length, l), msgAndArgs...) } return true } // True asserts that the specified value is true. // // assert.True(t, myBool, "myBool should be true") // // Returns whether the assertion was successful (true) or not (false). func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { if value != true { return Fail(t, "Should be true", msgAndArgs...) } return true } // False asserts that the specified value is false. // // assert.False(t, myBool, "myBool should be false") // // Returns whether the assertion was successful (true) or not (false). func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { if value != false { return Fail(t, "Should be false", msgAndArgs...) } return true } // NotEqual asserts that the specified values are NOT equal. // // assert.NotEqual(t, obj1, obj2, "two objects shouldn't be equal") // // Returns whether the assertion was successful (true) or not (false). func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { if ObjectsAreEqual(expected, actual) { return Fail(t, fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) } return true } // containsElement try loop over the list check if the list includes the element. // return (false, false) if impossible. // return (true, false) if element was not found. // return (true, true) if element was found. func includeElement(list interface{}, element interface{}) (ok, found bool) { listValue := reflect.ValueOf(list) elementValue := reflect.ValueOf(element) defer func() { if e := recover(); e != nil { ok = false found = false } }() if reflect.TypeOf(list).Kind() == reflect.String { return true, strings.Contains(listValue.String(), elementValue.String()) } if reflect.TypeOf(list).Kind() == reflect.Map { mapKeys := listValue.MapKeys() for i := 0; i < len(mapKeys); i++ { if ObjectsAreEqual(mapKeys[i].Interface(), element) { return true, true } } return true, false } for i := 0; i < listValue.Len(); i++ { if ObjectsAreEqual(listValue.Index(i).Interface(), element) { return true, true } } return true, false } // Contains asserts that the specified string, list(array, slice...) or map contains the // specified substring or element. // // assert.Contains(t, "Hello World", "World", "But 'Hello World' does contain 'World'") // assert.Contains(t, ["Hello", "World"], "World", "But ["Hello", "World"] does contain 'World'") // assert.Contains(t, {"Hello": "World"}, "Hello", "But {'Hello': 'World'} does contain 'Hello'") // // Returns whether the assertion was successful (true) or not (false). func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { ok, found := includeElement(s, contains) if !ok { return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", s), msgAndArgs...) } if !found { return Fail(t, fmt.Sprintf("\"%s\" does not contain \"%s\"", s, contains), msgAndArgs...) } return true } // NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the // specified substring or element. // // assert.NotContains(t, "Hello World", "Earth", "But 'Hello World' does NOT contain 'Earth'") // assert.NotContains(t, ["Hello", "World"], "Earth", "But ['Hello', 'World'] does NOT contain 'Earth'") // assert.NotContains(t, {"Hello": "World"}, "Earth", "But {'Hello': 'World'} does NOT contain 'Earth'") // // Returns whether the assertion was successful (true) or not (false). func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { ok, found := includeElement(s, contains) if !ok { return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", s), msgAndArgs...) } if found { return Fail(t, fmt.Sprintf("\"%s\" should not contain \"%s\"", s, contains), msgAndArgs...) } return true } // Condition uses a Comparison to assert a complex condition. func Condition(t TestingT, comp Comparison, msgAndArgs ...interface{}) bool { result := comp() if !result { Fail(t, "Condition failed!", msgAndArgs...) } return result } // PanicTestFunc defines a func that should be passed to the assert.Panics and assert.NotPanics // methods, and represents a simple func that takes no arguments, and returns nothing. type PanicTestFunc func() // didPanic returns true if the function passed to it panics. Otherwise, it returns false. func didPanic(f PanicTestFunc) (bool, interface{}) { didPanic := false var message interface{} func() { defer func() { if message = recover(); message != nil { didPanic = true } }() // call the target function f() }() return didPanic, message } // Panics asserts that the code inside the specified PanicTestFunc panics. // // assert.Panics(t, func(){ // GoCrazy() // }, "Calling GoCrazy() should panic") // // Returns whether the assertion was successful (true) or not (false). func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { if funcDidPanic, panicValue := didPanic(f); !funcDidPanic { return Fail(t, fmt.Sprintf("func %#v should panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...) } return true } // NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. // // assert.NotPanics(t, func(){ // RemainCalm() // }, "Calling RemainCalm() should NOT panic") // // Returns whether the assertion was successful (true) or not (false). func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { if funcDidPanic, panicValue := didPanic(f); funcDidPanic { return Fail(t, fmt.Sprintf("func %#v should not panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...) } return true } // WithinDuration asserts that the two times are within duration delta of each other. // // assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second, "The difference should not be more than 10s") // // Returns whether the assertion was successful (true) or not (false). func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { dt := expected.Sub(actual) if dt < -delta || dt > delta { return Fail(t, fmt.Sprintf("Max difference between %v and %v allowed is %v, but difference was %v", expected, actual, delta, dt), msgAndArgs...) } return true } func toFloat(x interface{}) (float64, bool) { var xf float64 xok := true switch xn := x.(type) { case uint8: xf = float64(xn) case uint16: xf = float64(xn) case uint32: xf = float64(xn) case uint64: xf = float64(xn) case int: xf = float64(xn) case int8: xf = float64(xn) case int16: xf = float64(xn) case int32: xf = float64(xn) case int64: xf = float64(xn) case float32: xf = float64(xn) case float64: xf = float64(xn) default: xok = false } return xf, xok } // InDelta asserts that the two numerals are within delta of each other. // // assert.InDelta(t, math.Pi, (22 / 7.0), 0.01) // // Returns whether the assertion was successful (true) or not (false). func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { af, aok := toFloat(expected) bf, bok := toFloat(actual) if !aok || !bok { return Fail(t, fmt.Sprintf("Parameters must be numerical"), msgAndArgs...) } if math.IsNaN(af) { return Fail(t, fmt.Sprintf("Actual must not be NaN"), msgAndArgs...) } if math.IsNaN(bf) { return Fail(t, fmt.Sprintf("Expected %v with delta %v, but was NaN", expected, delta), msgAndArgs...) } dt := af - bf if dt < -delta || dt > delta { return Fail(t, fmt.Sprintf("Max difference between %v and %v allowed is %v, but difference was %v", expected, actual, delta, dt), msgAndArgs...) } return true } // InDeltaSlice is the same as InDelta, except it compares two slices. func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { if expected == nil || actual == nil || reflect.TypeOf(actual).Kind() != reflect.Slice || reflect.TypeOf(expected).Kind() != reflect.Slice { return Fail(t, fmt.Sprintf("Parameters must be slice"), msgAndArgs...) } actualSlice := reflect.ValueOf(actual) expectedSlice := reflect.ValueOf(expected) for i := 0; i < actualSlice.Len(); i++ { result := InDelta(t, actualSlice.Index(i).Interface(), expectedSlice.Index(i).Interface(), delta) if !result { return result } } return true } func calcRelativeError(expected, actual interface{}) (float64, error) { af, aok := toFloat(expected) if !aok { return 0, fmt.Errorf("expected value %q cannot be converted to float", expected) } if af == 0 { return 0, fmt.Errorf("expected value must have a value other than zero to calculate the relative error") } bf, bok := toFloat(actual) if !bok { return 0, fmt.Errorf("expected value %q cannot be converted to float", actual) } return math.Abs(af-bf) / math.Abs(af), nil } // InEpsilon asserts that expected and actual have a relative error less than epsilon // // Returns whether the assertion was successful (true) or not (false). func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { actualEpsilon, err := calcRelativeError(expected, actual) if err != nil { return Fail(t, err.Error(), msgAndArgs...) } if actualEpsilon > epsilon { return Fail(t, fmt.Sprintf("Relative error is too high: %#v (expected)\n"+ " < %#v (actual)", actualEpsilon, epsilon), msgAndArgs...) } return true } // InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { if expected == nil || actual == nil || reflect.TypeOf(actual).Kind() != reflect.Slice || reflect.TypeOf(expected).Kind() != reflect.Slice { return Fail(t, fmt.Sprintf("Parameters must be slice"), msgAndArgs...) } actualSlice := reflect.ValueOf(actual) expectedSlice := reflect.ValueOf(expected) for i := 0; i < actualSlice.Len(); i++ { result := InEpsilon(t, actualSlice.Index(i).Interface(), expectedSlice.Index(i).Interface(), epsilon) if !result { return result } } return true } /* Errors */ // NoError asserts that a function returned no error (i.e. `nil`). // // actualObj, err := SomeFunction() // if assert.NoError(t, err) { // assert.Equal(t, actualObj, expectedObj) // } // // Returns whether the assertion was successful (true) or not (false). func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { if err != nil { return Fail(t, fmt.Sprintf("Received unexpected error %+v", err), msgAndArgs...) } return true } // Error asserts that a function returned an error (i.e. not `nil`). // // actualObj, err := SomeFunction() // if assert.Error(t, err, "An error was expected") { // assert.Equal(t, err, expectedError) // } // // Returns whether the assertion was successful (true) or not (false). func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { if err == nil { return Fail(t, "An error is expected but got nil.", msgAndArgs...) } return true } // EqualError asserts that a function returned an error (i.e. not `nil`) // and that it is equal to the provided error. // // actualObj, err := SomeFunction() // assert.EqualError(t, err, expectedErrorString, "An error was expected") // // Returns whether the assertion was successful (true) or not (false). func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) bool { message := messageFromMsgAndArgs(msgAndArgs...) if !NotNil(t, theError, "An error is expected but got nil. %s", message) { return false } s := "An error with value \"%s\" is expected but got \"%s\". %s" return Equal(t, errString, theError.Error(), s, errString, theError.Error(), message) } // matchRegexp return true if a specified regexp matches a string. func matchRegexp(rx interface{}, str interface{}) bool { var r *regexp.Regexp if rr, ok := rx.(*regexp.Regexp); ok { r = rr } else { r = regexp.MustCompile(fmt.Sprint(rx)) } return (r.FindStringIndex(fmt.Sprint(str)) != nil) } // Regexp asserts that a specified regexp matches a string. // // assert.Regexp(t, regexp.MustCompile("start"), "it's starting") // assert.Regexp(t, "start...$", "it's not starting") // // Returns whether the assertion was successful (true) or not (false). func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { match := matchRegexp(rx, str) if !match { Fail(t, fmt.Sprintf("Expect \"%v\" to match \"%v\"", str, rx), msgAndArgs...) } return match } // NotRegexp asserts that a specified regexp does not match a string. // // assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") // assert.NotRegexp(t, "^start", "it's not starting") // // Returns whether the assertion was successful (true) or not (false). func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { match := matchRegexp(rx, str) if match { Fail(t, fmt.Sprintf("Expect \"%v\" to NOT match \"%v\"", str, rx), msgAndArgs...) } return !match } // Zero asserts that i is the zero value for its type and returns the truth. func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { if i != nil && !reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { return Fail(t, fmt.Sprintf("Should be zero, but was %v", i), msgAndArgs...) } return true } // NotZero asserts that i is not the zero value for its type and returns the truth. func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { if i == nil || reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { return Fail(t, fmt.Sprintf("Should not be zero, but was %v", i), msgAndArgs...) } return true } // JSONEq asserts that two JSON strings are equivalent. // // assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) // // Returns whether the assertion was successful (true) or not (false). func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { var expectedJSONAsInterface, actualJSONAsInterface interface{} if err := json.Unmarshal([]byte(expected), &expectedJSONAsInterface); err != nil { return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid json.\nJSON parsing error: '%s'", expected, err.Error()), msgAndArgs...) } if err := json.Unmarshal([]byte(actual), &actualJSONAsInterface); err != nil { return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid json.\nJSON parsing error: '%s'", actual, err.Error()), msgAndArgs...) } return Equal(t, expectedJSONAsInterface, actualJSONAsInterface, msgAndArgs...) } func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { t := reflect.TypeOf(v) k := t.Kind() if k == reflect.Ptr { t = t.Elem() k = t.Kind() } return t, k } // diff returns a diff of both values as long as both are of the same type and // are a struct, map, slice or array. Otherwise it returns an empty string. func diff(expected interface{}, actual interface{}) string { if expected == nil || actual == nil { return "" } et, ek := typeAndKind(expected) at, _ := typeAndKind(actual) if et != at { return "" } if ek != reflect.Struct && ek != reflect.Map && ek != reflect.Slice && ek != reflect.Array { return "" } e := spew.Sdump(expected) a := spew.Sdump(actual) diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ A: difflib.SplitLines(e), B: difflib.SplitLines(a), FromFile: "Expected", FromDate: "", ToFile: "Actual", ToDate: "", Context: 1, }) return "\n\nDiff:\n" + diff } ================================================ FILE: vendor/github.com/stretchr/testify/assert/assertions_test.go ================================================ package assert import ( "errors" "io" "math" "os" "reflect" "regexp" "testing" "time" ) var ( i interface{} zeros = []interface{}{ false, byte(0), complex64(0), complex128(0), float32(0), float64(0), int(0), int8(0), int16(0), int32(0), int64(0), rune(0), uint(0), uint8(0), uint16(0), uint32(0), uint64(0), uintptr(0), "", [0]interface{}{}, []interface{}(nil), struct{ x int }{}, (*interface{})(nil), (func())(nil), nil, interface{}(nil), map[interface{}]interface{}(nil), (chan interface{})(nil), (<-chan interface{})(nil), (chan<- interface{})(nil), } nonZeros = []interface{}{ true, byte(1), complex64(1), complex128(1), float32(1), float64(1), int(1), int8(1), int16(1), int32(1), int64(1), rune(1), uint(1), uint8(1), uint16(1), uint32(1), uint64(1), uintptr(1), "s", [1]interface{}{1}, []interface{}{}, struct{ x int }{1}, (*interface{})(&i), (func())(func() {}), interface{}(1), map[interface{}]interface{}{}, (chan interface{})(make(chan interface{})), (<-chan interface{})(make(chan interface{})), (chan<- interface{})(make(chan interface{})), } ) // AssertionTesterInterface defines an interface to be used for testing assertion methods type AssertionTesterInterface interface { TestMethod() } // AssertionTesterConformingObject is an object that conforms to the AssertionTesterInterface interface type AssertionTesterConformingObject struct { } func (a *AssertionTesterConformingObject) TestMethod() { } // AssertionTesterNonConformingObject is an object that does not conform to the AssertionTesterInterface interface type AssertionTesterNonConformingObject struct { } func TestObjectsAreEqual(t *testing.T) { if !ObjectsAreEqual("Hello World", "Hello World") { t.Error("objectsAreEqual should return true") } if !ObjectsAreEqual(123, 123) { t.Error("objectsAreEqual should return true") } if !ObjectsAreEqual(123.5, 123.5) { t.Error("objectsAreEqual should return true") } if !ObjectsAreEqual([]byte("Hello World"), []byte("Hello World")) { t.Error("objectsAreEqual should return true") } if !ObjectsAreEqual(nil, nil) { t.Error("objectsAreEqual should return true") } if ObjectsAreEqual(map[int]int{5: 10}, map[int]int{10: 20}) { t.Error("objectsAreEqual should return false") } if ObjectsAreEqual('x', "x") { t.Error("objectsAreEqual should return false") } if ObjectsAreEqual("x", 'x') { t.Error("objectsAreEqual should return false") } if ObjectsAreEqual(0, 0.1) { t.Error("objectsAreEqual should return false") } if ObjectsAreEqual(0.1, 0) { t.Error("objectsAreEqual should return false") } if ObjectsAreEqual(uint32(10), int32(10)) { t.Error("objectsAreEqual should return false") } if !ObjectsAreEqualValues(uint32(10), int32(10)) { t.Error("ObjectsAreEqualValues should return true") } if ObjectsAreEqualValues(0, nil) { t.Fail() } if ObjectsAreEqualValues(nil, 0) { t.Fail() } } func TestImplements(t *testing.T) { mockT := new(testing.T) if !Implements(mockT, (*AssertionTesterInterface)(nil), new(AssertionTesterConformingObject)) { t.Error("Implements method should return true: AssertionTesterConformingObject implements AssertionTesterInterface") } if Implements(mockT, (*AssertionTesterInterface)(nil), new(AssertionTesterNonConformingObject)) { t.Error("Implements method should return false: AssertionTesterNonConformingObject does not implements AssertionTesterInterface") } } func TestIsType(t *testing.T) { mockT := new(testing.T) if !IsType(mockT, new(AssertionTesterConformingObject), new(AssertionTesterConformingObject)) { t.Error("IsType should return true: AssertionTesterConformingObject is the same type as AssertionTesterConformingObject") } if IsType(mockT, new(AssertionTesterConformingObject), new(AssertionTesterNonConformingObject)) { t.Error("IsType should return false: AssertionTesterConformingObject is not the same type as AssertionTesterNonConformingObject") } } func TestEqual(t *testing.T) { mockT := new(testing.T) if !Equal(mockT, "Hello World", "Hello World") { t.Error("Equal should return true") } if !Equal(mockT, 123, 123) { t.Error("Equal should return true") } if !Equal(mockT, 123.5, 123.5) { t.Error("Equal should return true") } if !Equal(mockT, []byte("Hello World"), []byte("Hello World")) { t.Error("Equal should return true") } if !Equal(mockT, nil, nil) { t.Error("Equal should return true") } if !Equal(mockT, int32(123), int32(123)) { t.Error("Equal should return true") } if !Equal(mockT, uint64(123), uint64(123)) { t.Error("Equal should return true") } } func TestFormatUnequalValues(t *testing.T) { expected, actual := formatUnequalValues("foo", "bar") Equal(t, `"foo"`, expected, "value should not include type") Equal(t, `"bar"`, actual, "value should not include type") expected, actual = formatUnequalValues(123, 123) Equal(t, `123`, expected, "value should not include type") Equal(t, `123`, actual, "value should not include type") expected, actual = formatUnequalValues(int64(123), int32(123)) Equal(t, `int64(123)`, expected, "value should include type") Equal(t, `int32(123)`, actual, "value should include type") type testStructType struct { Val string } expected, actual = formatUnequalValues(&testStructType{Val: "test"}, &testStructType{Val: "test"}) Equal(t, `&assert.testStructType{Val:"test"}`, expected, "value should not include type annotation") Equal(t, `&assert.testStructType{Val:"test"}`, actual, "value should not include type annotation") } func TestNotNil(t *testing.T) { mockT := new(testing.T) if !NotNil(mockT, new(AssertionTesterConformingObject)) { t.Error("NotNil should return true: object is not nil") } if NotNil(mockT, nil) { t.Error("NotNil should return false: object is nil") } if NotNil(mockT, (*struct{})(nil)) { t.Error("NotNil should return false: object is (*struct{})(nil)") } } func TestNil(t *testing.T) { mockT := new(testing.T) if !Nil(mockT, nil) { t.Error("Nil should return true: object is nil") } if !Nil(mockT, (*struct{})(nil)) { t.Error("Nil should return true: object is (*struct{})(nil)") } if Nil(mockT, new(AssertionTesterConformingObject)) { t.Error("Nil should return false: object is not nil") } } func TestTrue(t *testing.T) { mockT := new(testing.T) if !True(mockT, true) { t.Error("True should return true") } if True(mockT, false) { t.Error("True should return false") } } func TestFalse(t *testing.T) { mockT := new(testing.T) if !False(mockT, false) { t.Error("False should return true") } if False(mockT, true) { t.Error("False should return false") } } func TestExactly(t *testing.T) { mockT := new(testing.T) a := float32(1) b := float64(1) c := float32(1) d := float32(2) if Exactly(mockT, a, b) { t.Error("Exactly should return false") } if Exactly(mockT, a, d) { t.Error("Exactly should return false") } if !Exactly(mockT, a, c) { t.Error("Exactly should return true") } if Exactly(mockT, nil, a) { t.Error("Exactly should return false") } if Exactly(mockT, a, nil) { t.Error("Exactly should return false") } } func TestNotEqual(t *testing.T) { mockT := new(testing.T) if !NotEqual(mockT, "Hello World", "Hello World!") { t.Error("NotEqual should return true") } if !NotEqual(mockT, 123, 1234) { t.Error("NotEqual should return true") } if !NotEqual(mockT, 123.5, 123.55) { t.Error("NotEqual should return true") } if !NotEqual(mockT, []byte("Hello World"), []byte("Hello World!")) { t.Error("NotEqual should return true") } if !NotEqual(mockT, nil, new(AssertionTesterConformingObject)) { t.Error("NotEqual should return true") } funcA := func() int { return 23 } funcB := func() int { return 42 } if !NotEqual(mockT, funcA, funcB) { t.Error("NotEqual should return true") } if NotEqual(mockT, "Hello World", "Hello World") { t.Error("NotEqual should return false") } if NotEqual(mockT, 123, 123) { t.Error("NotEqual should return false") } if NotEqual(mockT, 123.5, 123.5) { t.Error("NotEqual should return false") } if NotEqual(mockT, []byte("Hello World"), []byte("Hello World")) { t.Error("NotEqual should return false") } if NotEqual(mockT, new(AssertionTesterConformingObject), new(AssertionTesterConformingObject)) { t.Error("NotEqual should return false") } } type A struct { Name, Value string } func TestContains(t *testing.T) { mockT := new(testing.T) list := []string{"Foo", "Bar"} complexList := []*A{ {"b", "c"}, {"d", "e"}, {"g", "h"}, {"j", "k"}, } simpleMap := map[interface{}]interface{}{"Foo": "Bar"} if !Contains(mockT, "Hello World", "Hello") { t.Error("Contains should return true: \"Hello World\" contains \"Hello\"") } if Contains(mockT, "Hello World", "Salut") { t.Error("Contains should return false: \"Hello World\" does not contain \"Salut\"") } if !Contains(mockT, list, "Bar") { t.Error("Contains should return true: \"[\"Foo\", \"Bar\"]\" contains \"Bar\"") } if Contains(mockT, list, "Salut") { t.Error("Contains should return false: \"[\"Foo\", \"Bar\"]\" does not contain \"Salut\"") } if !Contains(mockT, complexList, &A{"g", "h"}) { t.Error("Contains should return true: complexList contains {\"g\", \"h\"}") } if Contains(mockT, complexList, &A{"g", "e"}) { t.Error("Contains should return false: complexList contains {\"g\", \"e\"}") } if Contains(mockT, complexList, &A{"g", "e"}) { t.Error("Contains should return false: complexList contains {\"g\", \"e\"}") } if !Contains(mockT, simpleMap, "Foo") { t.Error("Contains should return true: \"{\"Foo\": \"Bar\"}\" contains \"Foo\"") } if Contains(mockT, simpleMap, "Bar") { t.Error("Contains should return false: \"{\"Foo\": \"Bar\"}\" does not contains \"Bar\"") } } func TestNotContains(t *testing.T) { mockT := new(testing.T) list := []string{"Foo", "Bar"} simpleMap := map[interface{}]interface{}{"Foo": "Bar"} if !NotContains(mockT, "Hello World", "Hello!") { t.Error("NotContains should return true: \"Hello World\" does not contain \"Hello!\"") } if NotContains(mockT, "Hello World", "Hello") { t.Error("NotContains should return false: \"Hello World\" contains \"Hello\"") } if !NotContains(mockT, list, "Foo!") { t.Error("NotContains should return true: \"[\"Foo\", \"Bar\"]\" does not contain \"Foo!\"") } if NotContains(mockT, list, "Foo") { t.Error("NotContains should return false: \"[\"Foo\", \"Bar\"]\" contains \"Foo\"") } if NotContains(mockT, simpleMap, "Foo") { t.Error("Contains should return true: \"{\"Foo\": \"Bar\"}\" contains \"Foo\"") } if !NotContains(mockT, simpleMap, "Bar") { t.Error("Contains should return false: \"{\"Foo\": \"Bar\"}\" does not contains \"Bar\"") } } func Test_includeElement(t *testing.T) { list1 := []string{"Foo", "Bar"} list2 := []int{1, 2} simpleMap := map[interface{}]interface{}{"Foo": "Bar"} ok, found := includeElement("Hello World", "World") True(t, ok) True(t, found) ok, found = includeElement(list1, "Foo") True(t, ok) True(t, found) ok, found = includeElement(list1, "Bar") True(t, ok) True(t, found) ok, found = includeElement(list2, 1) True(t, ok) True(t, found) ok, found = includeElement(list2, 2) True(t, ok) True(t, found) ok, found = includeElement(list1, "Foo!") True(t, ok) False(t, found) ok, found = includeElement(list2, 3) True(t, ok) False(t, found) ok, found = includeElement(list2, "1") True(t, ok) False(t, found) ok, found = includeElement(simpleMap, "Foo") True(t, ok) True(t, found) ok, found = includeElement(simpleMap, "Bar") True(t, ok) False(t, found) ok, found = includeElement(1433, "1") False(t, ok) False(t, found) } func TestCondition(t *testing.T) { mockT := new(testing.T) if !Condition(mockT, func() bool { return true }, "Truth") { t.Error("Condition should return true") } if Condition(mockT, func() bool { return false }, "Lie") { t.Error("Condition should return false") } } func TestDidPanic(t *testing.T) { if funcDidPanic, _ := didPanic(func() { panic("Panic!") }); !funcDidPanic { t.Error("didPanic should return true") } if funcDidPanic, _ := didPanic(func() { }); funcDidPanic { t.Error("didPanic should return false") } } func TestPanics(t *testing.T) { mockT := new(testing.T) if !Panics(mockT, func() { panic("Panic!") }) { t.Error("Panics should return true") } if Panics(mockT, func() { }) { t.Error("Panics should return false") } } func TestNotPanics(t *testing.T) { mockT := new(testing.T) if !NotPanics(mockT, func() { }) { t.Error("NotPanics should return true") } if NotPanics(mockT, func() { panic("Panic!") }) { t.Error("NotPanics should return false") } } func TestNoError(t *testing.T) { mockT := new(testing.T) // start with a nil error var err error True(t, NoError(mockT, err), "NoError should return True for nil arg") // now set an error err = errors.New("some error") False(t, NoError(mockT, err), "NoError with error should return False") // returning an empty error interface err = func() error { var err *customError if err != nil { t.Fatal("err should be nil here") } return err }() if err == nil { // err is not nil here! t.Errorf("Error should be nil due to empty interface", err) } False(t, NoError(mockT, err), "NoError should fail with empty error interface") } type customError struct{} func (*customError) Error() string { return "fail" } func TestError(t *testing.T) { mockT := new(testing.T) // start with a nil error var err error False(t, Error(mockT, err), "Error should return False for nil arg") // now set an error err = errors.New("some error") True(t, Error(mockT, err), "Error with error should return True") // returning an empty error interface err = func() error { var err *customError if err != nil { t.Fatal("err should be nil here") } return err }() if err == nil { // err is not nil here! t.Errorf("Error should be nil due to empty interface", err) } True(t, Error(mockT, err), "Error should pass with empty error interface") } func TestEqualError(t *testing.T) { mockT := new(testing.T) // start with a nil error var err error False(t, EqualError(mockT, err, ""), "EqualError should return false for nil arg") // now set an error err = errors.New("some error") False(t, EqualError(mockT, err, "Not some error"), "EqualError should return false for different error string") True(t, EqualError(mockT, err, "some error"), "EqualError should return true") } func Test_isEmpty(t *testing.T) { chWithValue := make(chan struct{}, 1) chWithValue <- struct{}{} True(t, isEmpty("")) True(t, isEmpty(nil)) True(t, isEmpty([]string{})) True(t, isEmpty(0)) True(t, isEmpty(int32(0))) True(t, isEmpty(int64(0))) True(t, isEmpty(false)) True(t, isEmpty(map[string]string{})) True(t, isEmpty(new(time.Time))) True(t, isEmpty(time.Time{})) True(t, isEmpty(make(chan struct{}))) False(t, isEmpty("something")) False(t, isEmpty(errors.New("something"))) False(t, isEmpty([]string{"something"})) False(t, isEmpty(1)) False(t, isEmpty(true)) False(t, isEmpty(map[string]string{"Hello": "World"})) False(t, isEmpty(chWithValue)) } func TestEmpty(t *testing.T) { mockT := new(testing.T) chWithValue := make(chan struct{}, 1) chWithValue <- struct{}{} var tiP *time.Time var tiNP time.Time var s *string var f *os.File True(t, Empty(mockT, ""), "Empty string is empty") True(t, Empty(mockT, nil), "Nil is empty") True(t, Empty(mockT, []string{}), "Empty string array is empty") True(t, Empty(mockT, 0), "Zero int value is empty") True(t, Empty(mockT, false), "False value is empty") True(t, Empty(mockT, make(chan struct{})), "Channel without values is empty") True(t, Empty(mockT, s), "Nil string pointer is empty") True(t, Empty(mockT, f), "Nil os.File pointer is empty") True(t, Empty(mockT, tiP), "Nil time.Time pointer is empty") True(t, Empty(mockT, tiNP), "time.Time is empty") False(t, Empty(mockT, "something"), "Non Empty string is not empty") False(t, Empty(mockT, errors.New("something")), "Non nil object is not empty") False(t, Empty(mockT, []string{"something"}), "Non empty string array is not empty") False(t, Empty(mockT, 1), "Non-zero int value is not empty") False(t, Empty(mockT, true), "True value is not empty") False(t, Empty(mockT, chWithValue), "Channel with values is not empty") } func TestNotEmpty(t *testing.T) { mockT := new(testing.T) chWithValue := make(chan struct{}, 1) chWithValue <- struct{}{} False(t, NotEmpty(mockT, ""), "Empty string is empty") False(t, NotEmpty(mockT, nil), "Nil is empty") False(t, NotEmpty(mockT, []string{}), "Empty string array is empty") False(t, NotEmpty(mockT, 0), "Zero int value is empty") False(t, NotEmpty(mockT, false), "False value is empty") False(t, NotEmpty(mockT, make(chan struct{})), "Channel without values is empty") True(t, NotEmpty(mockT, "something"), "Non Empty string is not empty") True(t, NotEmpty(mockT, errors.New("something")), "Non nil object is not empty") True(t, NotEmpty(mockT, []string{"something"}), "Non empty string array is not empty") True(t, NotEmpty(mockT, 1), "Non-zero int value is not empty") True(t, NotEmpty(mockT, true), "True value is not empty") True(t, NotEmpty(mockT, chWithValue), "Channel with values is not empty") } func Test_getLen(t *testing.T) { falseCases := []interface{}{ nil, 0, true, false, 'A', struct{}{}, } for _, v := range falseCases { ok, l := getLen(v) False(t, ok, "Expected getLen fail to get length of %#v", v) Equal(t, 0, l, "getLen should return 0 for %#v", v) } ch := make(chan int, 5) ch <- 1 ch <- 2 ch <- 3 trueCases := []struct { v interface{} l int }{ {[]int{1, 2, 3}, 3}, {[...]int{1, 2, 3}, 3}, {"ABC", 3}, {map[int]int{1: 2, 2: 4, 3: 6}, 3}, {ch, 3}, {[]int{}, 0}, {map[int]int{}, 0}, {make(chan int), 0}, {[]int(nil), 0}, {map[int]int(nil), 0}, {(chan int)(nil), 0}, } for _, c := range trueCases { ok, l := getLen(c.v) True(t, ok, "Expected getLen success to get length of %#v", c.v) Equal(t, c.l, l) } } func TestLen(t *testing.T) { mockT := new(testing.T) False(t, Len(mockT, nil, 0), "nil does not have length") False(t, Len(mockT, 0, 0), "int does not have length") False(t, Len(mockT, true, 0), "true does not have length") False(t, Len(mockT, false, 0), "false does not have length") False(t, Len(mockT, 'A', 0), "Rune does not have length") False(t, Len(mockT, struct{}{}, 0), "Struct does not have length") ch := make(chan int, 5) ch <- 1 ch <- 2 ch <- 3 cases := []struct { v interface{} l int }{ {[]int{1, 2, 3}, 3}, {[...]int{1, 2, 3}, 3}, {"ABC", 3}, {map[int]int{1: 2, 2: 4, 3: 6}, 3}, {ch, 3}, {[]int{}, 0}, {map[int]int{}, 0}, {make(chan int), 0}, {[]int(nil), 0}, {map[int]int(nil), 0}, {(chan int)(nil), 0}, } for _, c := range cases { True(t, Len(mockT, c.v, c.l), "%#v have %d items", c.v, c.l) } cases = []struct { v interface{} l int }{ {[]int{1, 2, 3}, 4}, {[...]int{1, 2, 3}, 2}, {"ABC", 2}, {map[int]int{1: 2, 2: 4, 3: 6}, 4}, {ch, 2}, {[]int{}, 1}, {map[int]int{}, 1}, {make(chan int), 1}, {[]int(nil), 1}, {map[int]int(nil), 1}, {(chan int)(nil), 1}, } for _, c := range cases { False(t, Len(mockT, c.v, c.l), "%#v have %d items", c.v, c.l) } } func TestWithinDuration(t *testing.T) { mockT := new(testing.T) a := time.Now() b := a.Add(10 * time.Second) True(t, WithinDuration(mockT, a, b, 10*time.Second), "A 10s difference is within a 10s time difference") True(t, WithinDuration(mockT, b, a, 10*time.Second), "A 10s difference is within a 10s time difference") False(t, WithinDuration(mockT, a, b, 9*time.Second), "A 10s difference is not within a 9s time difference") False(t, WithinDuration(mockT, b, a, 9*time.Second), "A 10s difference is not within a 9s time difference") False(t, WithinDuration(mockT, a, b, -9*time.Second), "A 10s difference is not within a 9s time difference") False(t, WithinDuration(mockT, b, a, -9*time.Second), "A 10s difference is not within a 9s time difference") False(t, WithinDuration(mockT, a, b, -11*time.Second), "A 10s difference is not within a 9s time difference") False(t, WithinDuration(mockT, b, a, -11*time.Second), "A 10s difference is not within a 9s time difference") } func TestInDelta(t *testing.T) { mockT := new(testing.T) True(t, InDelta(mockT, 1.001, 1, 0.01), "|1.001 - 1| <= 0.01") True(t, InDelta(mockT, 1, 1.001, 0.01), "|1 - 1.001| <= 0.01") True(t, InDelta(mockT, 1, 2, 1), "|1 - 2| <= 1") False(t, InDelta(mockT, 1, 2, 0.5), "Expected |1 - 2| <= 0.5 to fail") False(t, InDelta(mockT, 2, 1, 0.5), "Expected |2 - 1| <= 0.5 to fail") False(t, InDelta(mockT, "", nil, 1), "Expected non numerals to fail") False(t, InDelta(mockT, 42, math.NaN(), 0.01), "Expected NaN for actual to fail") False(t, InDelta(mockT, math.NaN(), 42, 0.01), "Expected NaN for expected to fail") cases := []struct { a, b interface{} delta float64 }{ {uint8(2), uint8(1), 1}, {uint16(2), uint16(1), 1}, {uint32(2), uint32(1), 1}, {uint64(2), uint64(1), 1}, {int(2), int(1), 1}, {int8(2), int8(1), 1}, {int16(2), int16(1), 1}, {int32(2), int32(1), 1}, {int64(2), int64(1), 1}, {float32(2), float32(1), 1}, {float64(2), float64(1), 1}, } for _, tc := range cases { True(t, InDelta(mockT, tc.a, tc.b, tc.delta), "Expected |%V - %V| <= %v", tc.a, tc.b, tc.delta) } } func TestInDeltaSlice(t *testing.T) { mockT := new(testing.T) True(t, InDeltaSlice(mockT, []float64{1.001, 0.999}, []float64{1, 1}, 0.1), "{1.001, 0.009} is element-wise close to {1, 1} in delta=0.1") True(t, InDeltaSlice(mockT, []float64{1, 2}, []float64{0, 3}, 1), "{1, 2} is element-wise close to {0, 3} in delta=1") False(t, InDeltaSlice(mockT, []float64{1, 2}, []float64{0, 3}, 0.1), "{1, 2} is not element-wise close to {0, 3} in delta=0.1") False(t, InDeltaSlice(mockT, "", nil, 1), "Expected non numeral slices to fail") } func TestInEpsilon(t *testing.T) { mockT := new(testing.T) cases := []struct { a, b interface{} epsilon float64 }{ {uint8(2), uint16(2), .001}, {2.1, 2.2, 0.1}, {2.2, 2.1, 0.1}, {-2.1, -2.2, 0.1}, {-2.2, -2.1, 0.1}, {uint64(100), uint8(101), 0.01}, {0.1, -0.1, 2}, {0.1, 0, 2}, } for _, tc := range cases { True(t, InEpsilon(t, tc.a, tc.b, tc.epsilon, "Expected %V and %V to have a relative difference of %v", tc.a, tc.b, tc.epsilon), "test: %q", tc) } cases = []struct { a, b interface{} epsilon float64 }{ {uint8(2), int16(-2), .001}, {uint64(100), uint8(102), 0.01}, {2.1, 2.2, 0.001}, {2.2, 2.1, 0.001}, {2.1, -2.2, 1}, {2.1, "bla-bla", 0}, {0.1, -0.1, 1.99}, {0, 0.1, 2}, // expected must be different to zero } for _, tc := range cases { False(t, InEpsilon(mockT, tc.a, tc.b, tc.epsilon, "Expected %V and %V to have a relative difference of %v", tc.a, tc.b, tc.epsilon)) } } func TestInEpsilonSlice(t *testing.T) { mockT := new(testing.T) True(t, InEpsilonSlice(mockT, []float64{2.2, 2.0}, []float64{2.1, 2.1}, 0.06), "{2.2, 2.0} is element-wise close to {2.1, 2.1} in espilon=0.06") False(t, InEpsilonSlice(mockT, []float64{2.2, 2.0}, []float64{2.1, 2.1}, 0.04), "{2.2, 2.0} is not element-wise close to {2.1, 2.1} in espilon=0.04") False(t, InEpsilonSlice(mockT, "", nil, 1), "Expected non numeral slices to fail") } func TestRegexp(t *testing.T) { mockT := new(testing.T) cases := []struct { rx, str string }{ {"^start", "start of the line"}, {"end$", "in the end"}, {"[0-9]{3}[.-]?[0-9]{2}[.-]?[0-9]{2}", "My phone number is 650.12.34"}, } for _, tc := range cases { True(t, Regexp(mockT, tc.rx, tc.str)) True(t, Regexp(mockT, regexp.MustCompile(tc.rx), tc.str)) False(t, NotRegexp(mockT, tc.rx, tc.str)) False(t, NotRegexp(mockT, regexp.MustCompile(tc.rx), tc.str)) } cases = []struct { rx, str string }{ {"^asdfastart", "Not the start of the line"}, {"end$", "in the end."}, {"[0-9]{3}[.-]?[0-9]{2}[.-]?[0-9]{2}", "My phone number is 650.12a.34"}, } for _, tc := range cases { False(t, Regexp(mockT, tc.rx, tc.str), "Expected \"%s\" to not match \"%s\"", tc.rx, tc.str) False(t, Regexp(mockT, regexp.MustCompile(tc.rx), tc.str)) True(t, NotRegexp(mockT, tc.rx, tc.str)) True(t, NotRegexp(mockT, regexp.MustCompile(tc.rx), tc.str)) } } func testAutogeneratedFunction() { defer func() { if err := recover(); err == nil { panic("did not panic") } CallerInfo() }() t := struct { io.Closer }{} var c io.Closer c = t c.Close() } func TestCallerInfoWithAutogeneratedFunctions(t *testing.T) { NotPanics(t, func() { testAutogeneratedFunction() }) } func TestZero(t *testing.T) { mockT := new(testing.T) for _, test := range zeros { True(t, Zero(mockT, test, "%#v is not the %v zero value", test, reflect.TypeOf(test))) } for _, test := range nonZeros { False(t, Zero(mockT, test, "%#v is not the %v zero value", test, reflect.TypeOf(test))) } } func TestNotZero(t *testing.T) { mockT := new(testing.T) for _, test := range zeros { False(t, NotZero(mockT, test, "%#v is not the %v zero value", test, reflect.TypeOf(test))) } for _, test := range nonZeros { True(t, NotZero(mockT, test, "%#v is not the %v zero value", test, reflect.TypeOf(test))) } } func TestJSONEq_EqualSONString(t *testing.T) { mockT := new(testing.T) True(t, JSONEq(mockT, `{"hello": "world", "foo": "bar"}`, `{"hello": "world", "foo": "bar"}`)) } func TestJSONEq_EquivalentButNotEqual(t *testing.T) { mockT := new(testing.T) True(t, JSONEq(mockT, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`)) } func TestJSONEq_HashOfArraysAndHashes(t *testing.T) { mockT := new(testing.T) True(t, JSONEq(mockT, "{\r\n\t\"numeric\": 1.5,\r\n\t\"array\": [{\"foo\": \"bar\"}, 1, \"string\", [\"nested\", \"array\", 5.5]],\r\n\t\"hash\": {\"nested\": \"hash\", \"nested_slice\": [\"this\", \"is\", \"nested\"]},\r\n\t\"string\": \"foo\"\r\n}", "{\r\n\t\"numeric\": 1.5,\r\n\t\"hash\": {\"nested\": \"hash\", \"nested_slice\": [\"this\", \"is\", \"nested\"]},\r\n\t\"string\": \"foo\",\r\n\t\"array\": [{\"foo\": \"bar\"}, 1, \"string\", [\"nested\", \"array\", 5.5]]\r\n}")) } func TestJSONEq_Array(t *testing.T) { mockT := new(testing.T) True(t, JSONEq(mockT, `["foo", {"hello": "world", "nested": "hash"}]`, `["foo", {"nested": "hash", "hello": "world"}]`)) } func TestJSONEq_HashAndArrayNotEquivalent(t *testing.T) { mockT := new(testing.T) False(t, JSONEq(mockT, `["foo", {"hello": "world", "nested": "hash"}]`, `{"foo": "bar", {"nested": "hash", "hello": "world"}}`)) } func TestJSONEq_HashesNotEquivalent(t *testing.T) { mockT := new(testing.T) False(t, JSONEq(mockT, `{"foo": "bar"}`, `{"foo": "bar", "hello": "world"}`)) } func TestJSONEq_ActualIsNotJSON(t *testing.T) { mockT := new(testing.T) False(t, JSONEq(mockT, `{"foo": "bar"}`, "Not JSON")) } func TestJSONEq_ExpectedIsNotJSON(t *testing.T) { mockT := new(testing.T) False(t, JSONEq(mockT, "Not JSON", `{"foo": "bar", "hello": "world"}`)) } func TestJSONEq_ExpectedAndActualNotJSON(t *testing.T) { mockT := new(testing.T) False(t, JSONEq(mockT, "Not JSON", "Not JSON")) } func TestJSONEq_ArraysOfDifferentOrder(t *testing.T) { mockT := new(testing.T) False(t, JSONEq(mockT, `["foo", {"hello": "world", "nested": "hash"}]`, `[{ "hello": "world", "nested": "hash"}, "foo"]`)) } func TestDiff(t *testing.T) { expected := ` Diff: --- Expected +++ Actual @@ -1,3 +1,3 @@ (struct { foo string }) { - foo: (string) (len=5) "hello" + foo: (string) (len=3) "bar" } ` actual := diff( struct{ foo string }{"hello"}, struct{ foo string }{"bar"}, ) Equal(t, expected, actual) expected = ` Diff: --- Expected +++ Actual @@ -2,5 +2,5 @@ (int) 1, - (int) 2, (int) 3, - (int) 4 + (int) 5, + (int) 7 } ` actual = diff( []int{1, 2, 3, 4}, []int{1, 3, 5, 7}, ) Equal(t, expected, actual) expected = ` Diff: --- Expected +++ Actual @@ -2,4 +2,4 @@ (int) 1, - (int) 2, - (int) 3 + (int) 3, + (int) 5 } ` actual = diff( []int{1, 2, 3, 4}[0:3], []int{1, 3, 5, 7}[0:3], ) Equal(t, expected, actual) expected = ` Diff: --- Expected +++ Actual @@ -1,6 +1,6 @@ (map[string]int) (len=4) { - (string) (len=4) "four": (int) 4, + (string) (len=4) "five": (int) 5, (string) (len=3) "one": (int) 1, - (string) (len=5) "three": (int) 3, - (string) (len=3) "two": (int) 2 + (string) (len=5) "seven": (int) 7, + (string) (len=5) "three": (int) 3 } ` actual = diff( map[string]int{"one": 1, "two": 2, "three": 3, "four": 4}, map[string]int{"one": 1, "three": 3, "five": 5, "seven": 7}, ) Equal(t, expected, actual) } func TestDiffEmptyCases(t *testing.T) { Equal(t, "", diff(nil, nil)) Equal(t, "", diff(struct{ foo string }{}, nil)) Equal(t, "", diff(nil, struct{ foo string }{})) Equal(t, "", diff(1, 2)) Equal(t, "", diff(1, 2)) Equal(t, "", diff([]int{1}, []bool{true})) } // Ensure there are no data races func TestDiffRace(t *testing.T) { t.Parallel() expected := map[string]string{ "a": "A", "b": "B", "c": "C", } actual := map[string]string{ "d": "D", "e": "E", "f": "F", } // run diffs in parallel simulating tests with t.Parallel() numRoutines := 10 rChans := make([]chan string, numRoutines) for idx := range rChans { rChans[idx] = make(chan string) go func(ch chan string) { defer close(ch) ch <- diff(expected, actual) }(rChans[idx]) } for _, ch := range rChans { for msg := range ch { NotZero(t, msg) // dummy assert } } } type mockTestingT struct { } func (m *mockTestingT) Errorf(format string, args ...interface{}) {} func TestFailNowWithPlainTestingT(t *testing.T) { mockT := &mockTestingT{} Panics(t, func() { FailNow(mockT, "failed") }, "should panic since mockT is missing FailNow()") } type mockFailNowTestingT struct { } func (m *mockFailNowTestingT) Errorf(format string, args ...interface{}) {} func (m *mockFailNowTestingT) FailNow() {} func TestFailNowWithFullTestingT(t *testing.T) { mockT := &mockFailNowTestingT{} NotPanics(t, func() { FailNow(mockT, "failed") }, "should call mockT.FailNow() rather than panicking") } ================================================ FILE: vendor/github.com/stretchr/testify/assert/doc.go ================================================ // Package assert provides a set of comprehensive testing tools for use with the normal Go testing system. // // Example Usage // // The following is a complete example using assert in a standard test function: // import ( // "testing" // "github.com/stretchr/testify/assert" // ) // // func TestSomething(t *testing.T) { // // var a string = "Hello" // var b string = "Hello" // // assert.Equal(t, a, b, "The two words should be the same.") // // } // // if you assert many times, use the format below: // // import ( // "testing" // "github.com/stretchr/testify/assert" // ) // // func TestSomething(t *testing.T) { // assert := assert.New(t) // // var a string = "Hello" // var b string = "Hello" // // assert.Equal(a, b, "The two words should be the same.") // } // // Assertions // // Assertions allow you to easily write test code, and are global funcs in the `assert` package. // All assertion functions take, as the first argument, the `*testing.T` object provided by the // testing framework. This allows the assertion funcs to write the failings and other details to // the correct place. // // Every assertion function also takes an optional string message as the final argument, // allowing custom error messages to be appended to the message the assertion method outputs. package assert ================================================ FILE: vendor/github.com/stretchr/testify/assert/errors.go ================================================ package assert import ( "errors" ) // AnError is an error instance useful for testing. If the code does not care // about error specifics, and only needs to return the error for example, this // error should be used to make the test code more readable. var AnError = errors.New("assert.AnError general error for testing") ================================================ FILE: vendor/github.com/stretchr/testify/assert/forward_assertions.go ================================================ package assert // Assertions provides assertion methods around the // TestingT interface. type Assertions struct { t TestingT } // New makes a new Assertions object for the specified TestingT. func New(t TestingT) *Assertions { return &Assertions{ t: t, } } //go:generate go run ../_codegen/main.go -output-package=assert -template=assertion_forward.go.tmpl ================================================ FILE: vendor/github.com/stretchr/testify/assert/forward_assertions_test.go ================================================ package assert import ( "errors" "regexp" "testing" "time" ) func TestImplementsWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.Implements((*AssertionTesterInterface)(nil), new(AssertionTesterConformingObject)) { t.Error("Implements method should return true: AssertionTesterConformingObject implements AssertionTesterInterface") } if assert.Implements((*AssertionTesterInterface)(nil), new(AssertionTesterNonConformingObject)) { t.Error("Implements method should return false: AssertionTesterNonConformingObject does not implements AssertionTesterInterface") } } func TestIsTypeWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.IsType(new(AssertionTesterConformingObject), new(AssertionTesterConformingObject)) { t.Error("IsType should return true: AssertionTesterConformingObject is the same type as AssertionTesterConformingObject") } if assert.IsType(new(AssertionTesterConformingObject), new(AssertionTesterNonConformingObject)) { t.Error("IsType should return false: AssertionTesterConformingObject is not the same type as AssertionTesterNonConformingObject") } } func TestEqualWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.Equal("Hello World", "Hello World") { t.Error("Equal should return true") } if !assert.Equal(123, 123) { t.Error("Equal should return true") } if !assert.Equal(123.5, 123.5) { t.Error("Equal should return true") } if !assert.Equal([]byte("Hello World"), []byte("Hello World")) { t.Error("Equal should return true") } if !assert.Equal(nil, nil) { t.Error("Equal should return true") } } func TestEqualValuesWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.EqualValues(uint32(10), int32(10)) { t.Error("EqualValues should return true") } } func TestNotNilWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.NotNil(new(AssertionTesterConformingObject)) { t.Error("NotNil should return true: object is not nil") } if assert.NotNil(nil) { t.Error("NotNil should return false: object is nil") } } func TestNilWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.Nil(nil) { t.Error("Nil should return true: object is nil") } if assert.Nil(new(AssertionTesterConformingObject)) { t.Error("Nil should return false: object is not nil") } } func TestTrueWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.True(true) { t.Error("True should return true") } if assert.True(false) { t.Error("True should return false") } } func TestFalseWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.False(false) { t.Error("False should return true") } if assert.False(true) { t.Error("False should return false") } } func TestExactlyWrapper(t *testing.T) { assert := New(new(testing.T)) a := float32(1) b := float64(1) c := float32(1) d := float32(2) if assert.Exactly(a, b) { t.Error("Exactly should return false") } if assert.Exactly(a, d) { t.Error("Exactly should return false") } if !assert.Exactly(a, c) { t.Error("Exactly should return true") } if assert.Exactly(nil, a) { t.Error("Exactly should return false") } if assert.Exactly(a, nil) { t.Error("Exactly should return false") } } func TestNotEqualWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.NotEqual("Hello World", "Hello World!") { t.Error("NotEqual should return true") } if !assert.NotEqual(123, 1234) { t.Error("NotEqual should return true") } if !assert.NotEqual(123.5, 123.55) { t.Error("NotEqual should return true") } if !assert.NotEqual([]byte("Hello World"), []byte("Hello World!")) { t.Error("NotEqual should return true") } if !assert.NotEqual(nil, new(AssertionTesterConformingObject)) { t.Error("NotEqual should return true") } } func TestContainsWrapper(t *testing.T) { assert := New(new(testing.T)) list := []string{"Foo", "Bar"} if !assert.Contains("Hello World", "Hello") { t.Error("Contains should return true: \"Hello World\" contains \"Hello\"") } if assert.Contains("Hello World", "Salut") { t.Error("Contains should return false: \"Hello World\" does not contain \"Salut\"") } if !assert.Contains(list, "Foo") { t.Error("Contains should return true: \"[\"Foo\", \"Bar\"]\" contains \"Foo\"") } if assert.Contains(list, "Salut") { t.Error("Contains should return false: \"[\"Foo\", \"Bar\"]\" does not contain \"Salut\"") } } func TestNotContainsWrapper(t *testing.T) { assert := New(new(testing.T)) list := []string{"Foo", "Bar"} if !assert.NotContains("Hello World", "Hello!") { t.Error("NotContains should return true: \"Hello World\" does not contain \"Hello!\"") } if assert.NotContains("Hello World", "Hello") { t.Error("NotContains should return false: \"Hello World\" contains \"Hello\"") } if !assert.NotContains(list, "Foo!") { t.Error("NotContains should return true: \"[\"Foo\", \"Bar\"]\" does not contain \"Foo!\"") } if assert.NotContains(list, "Foo") { t.Error("NotContains should return false: \"[\"Foo\", \"Bar\"]\" contains \"Foo\"") } } func TestConditionWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.Condition(func() bool { return true }, "Truth") { t.Error("Condition should return true") } if assert.Condition(func() bool { return false }, "Lie") { t.Error("Condition should return false") } } func TestDidPanicWrapper(t *testing.T) { if funcDidPanic, _ := didPanic(func() { panic("Panic!") }); !funcDidPanic { t.Error("didPanic should return true") } if funcDidPanic, _ := didPanic(func() { }); funcDidPanic { t.Error("didPanic should return false") } } func TestPanicsWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.Panics(func() { panic("Panic!") }) { t.Error("Panics should return true") } if assert.Panics(func() { }) { t.Error("Panics should return false") } } func TestNotPanicsWrapper(t *testing.T) { assert := New(new(testing.T)) if !assert.NotPanics(func() { }) { t.Error("NotPanics should return true") } if assert.NotPanics(func() { panic("Panic!") }) { t.Error("NotPanics should return false") } } func TestNoErrorWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) // start with a nil error var err error assert.True(mockAssert.NoError(err), "NoError should return True for nil arg") // now set an error err = errors.New("Some error") assert.False(mockAssert.NoError(err), "NoError with error should return False") } func TestErrorWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) // start with a nil error var err error assert.False(mockAssert.Error(err), "Error should return False for nil arg") // now set an error err = errors.New("Some error") assert.True(mockAssert.Error(err), "Error with error should return True") } func TestEqualErrorWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) // start with a nil error var err error assert.False(mockAssert.EqualError(err, ""), "EqualError should return false for nil arg") // now set an error err = errors.New("some error") assert.False(mockAssert.EqualError(err, "Not some error"), "EqualError should return false for different error string") assert.True(mockAssert.EqualError(err, "some error"), "EqualError should return true") } func TestEmptyWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) assert.True(mockAssert.Empty(""), "Empty string is empty") assert.True(mockAssert.Empty(nil), "Nil is empty") assert.True(mockAssert.Empty([]string{}), "Empty string array is empty") assert.True(mockAssert.Empty(0), "Zero int value is empty") assert.True(mockAssert.Empty(false), "False value is empty") assert.False(mockAssert.Empty("something"), "Non Empty string is not empty") assert.False(mockAssert.Empty(errors.New("something")), "Non nil object is not empty") assert.False(mockAssert.Empty([]string{"something"}), "Non empty string array is not empty") assert.False(mockAssert.Empty(1), "Non-zero int value is not empty") assert.False(mockAssert.Empty(true), "True value is not empty") } func TestNotEmptyWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) assert.False(mockAssert.NotEmpty(""), "Empty string is empty") assert.False(mockAssert.NotEmpty(nil), "Nil is empty") assert.False(mockAssert.NotEmpty([]string{}), "Empty string array is empty") assert.False(mockAssert.NotEmpty(0), "Zero int value is empty") assert.False(mockAssert.NotEmpty(false), "False value is empty") assert.True(mockAssert.NotEmpty("something"), "Non Empty string is not empty") assert.True(mockAssert.NotEmpty(errors.New("something")), "Non nil object is not empty") assert.True(mockAssert.NotEmpty([]string{"something"}), "Non empty string array is not empty") assert.True(mockAssert.NotEmpty(1), "Non-zero int value is not empty") assert.True(mockAssert.NotEmpty(true), "True value is not empty") } func TestLenWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) assert.False(mockAssert.Len(nil, 0), "nil does not have length") assert.False(mockAssert.Len(0, 0), "int does not have length") assert.False(mockAssert.Len(true, 0), "true does not have length") assert.False(mockAssert.Len(false, 0), "false does not have length") assert.False(mockAssert.Len('A', 0), "Rune does not have length") assert.False(mockAssert.Len(struct{}{}, 0), "Struct does not have length") ch := make(chan int, 5) ch <- 1 ch <- 2 ch <- 3 cases := []struct { v interface{} l int }{ {[]int{1, 2, 3}, 3}, {[...]int{1, 2, 3}, 3}, {"ABC", 3}, {map[int]int{1: 2, 2: 4, 3: 6}, 3}, {ch, 3}, {[]int{}, 0}, {map[int]int{}, 0}, {make(chan int), 0}, {[]int(nil), 0}, {map[int]int(nil), 0}, {(chan int)(nil), 0}, } for _, c := range cases { assert.True(mockAssert.Len(c.v, c.l), "%#v have %d items", c.v, c.l) } } func TestWithinDurationWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) a := time.Now() b := a.Add(10 * time.Second) assert.True(mockAssert.WithinDuration(a, b, 10*time.Second), "A 10s difference is within a 10s time difference") assert.True(mockAssert.WithinDuration(b, a, 10*time.Second), "A 10s difference is within a 10s time difference") assert.False(mockAssert.WithinDuration(a, b, 9*time.Second), "A 10s difference is not within a 9s time difference") assert.False(mockAssert.WithinDuration(b, a, 9*time.Second), "A 10s difference is not within a 9s time difference") assert.False(mockAssert.WithinDuration(a, b, -9*time.Second), "A 10s difference is not within a 9s time difference") assert.False(mockAssert.WithinDuration(b, a, -9*time.Second), "A 10s difference is not within a 9s time difference") assert.False(mockAssert.WithinDuration(a, b, -11*time.Second), "A 10s difference is not within a 9s time difference") assert.False(mockAssert.WithinDuration(b, a, -11*time.Second), "A 10s difference is not within a 9s time difference") } func TestInDeltaWrapper(t *testing.T) { assert := New(new(testing.T)) True(t, assert.InDelta(1.001, 1, 0.01), "|1.001 - 1| <= 0.01") True(t, assert.InDelta(1, 1.001, 0.01), "|1 - 1.001| <= 0.01") True(t, assert.InDelta(1, 2, 1), "|1 - 2| <= 1") False(t, assert.InDelta(1, 2, 0.5), "Expected |1 - 2| <= 0.5 to fail") False(t, assert.InDelta(2, 1, 0.5), "Expected |2 - 1| <= 0.5 to fail") False(t, assert.InDelta("", nil, 1), "Expected non numerals to fail") cases := []struct { a, b interface{} delta float64 }{ {uint8(2), uint8(1), 1}, {uint16(2), uint16(1), 1}, {uint32(2), uint32(1), 1}, {uint64(2), uint64(1), 1}, {int(2), int(1), 1}, {int8(2), int8(1), 1}, {int16(2), int16(1), 1}, {int32(2), int32(1), 1}, {int64(2), int64(1), 1}, {float32(2), float32(1), 1}, {float64(2), float64(1), 1}, } for _, tc := range cases { True(t, assert.InDelta(tc.a, tc.b, tc.delta), "Expected |%V - %V| <= %v", tc.a, tc.b, tc.delta) } } func TestInEpsilonWrapper(t *testing.T) { assert := New(new(testing.T)) cases := []struct { a, b interface{} epsilon float64 }{ {uint8(2), uint16(2), .001}, {2.1, 2.2, 0.1}, {2.2, 2.1, 0.1}, {-2.1, -2.2, 0.1}, {-2.2, -2.1, 0.1}, {uint64(100), uint8(101), 0.01}, {0.1, -0.1, 2}, } for _, tc := range cases { True(t, assert.InEpsilon(tc.a, tc.b, tc.epsilon, "Expected %V and %V to have a relative difference of %v", tc.a, tc.b, tc.epsilon)) } cases = []struct { a, b interface{} epsilon float64 }{ {uint8(2), int16(-2), .001}, {uint64(100), uint8(102), 0.01}, {2.1, 2.2, 0.001}, {2.2, 2.1, 0.001}, {2.1, -2.2, 1}, {2.1, "bla-bla", 0}, {0.1, -0.1, 1.99}, } for _, tc := range cases { False(t, assert.InEpsilon(tc.a, tc.b, tc.epsilon, "Expected %V and %V to have a relative difference of %v", tc.a, tc.b, tc.epsilon)) } } func TestRegexpWrapper(t *testing.T) { assert := New(new(testing.T)) cases := []struct { rx, str string }{ {"^start", "start of the line"}, {"end$", "in the end"}, {"[0-9]{3}[.-]?[0-9]{2}[.-]?[0-9]{2}", "My phone number is 650.12.34"}, } for _, tc := range cases { True(t, assert.Regexp(tc.rx, tc.str)) True(t, assert.Regexp(regexp.MustCompile(tc.rx), tc.str)) False(t, assert.NotRegexp(tc.rx, tc.str)) False(t, assert.NotRegexp(regexp.MustCompile(tc.rx), tc.str)) } cases = []struct { rx, str string }{ {"^asdfastart", "Not the start of the line"}, {"end$", "in the end."}, {"[0-9]{3}[.-]?[0-9]{2}[.-]?[0-9]{2}", "My phone number is 650.12a.34"}, } for _, tc := range cases { False(t, assert.Regexp(tc.rx, tc.str), "Expected \"%s\" to not match \"%s\"", tc.rx, tc.str) False(t, assert.Regexp(regexp.MustCompile(tc.rx), tc.str)) True(t, assert.NotRegexp(tc.rx, tc.str)) True(t, assert.NotRegexp(regexp.MustCompile(tc.rx), tc.str)) } } func TestZeroWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) for _, test := range zeros { assert.True(mockAssert.Zero(test), "Zero should return true for %v", test) } for _, test := range nonZeros { assert.False(mockAssert.Zero(test), "Zero should return false for %v", test) } } func TestNotZeroWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) for _, test := range zeros { assert.False(mockAssert.NotZero(test), "Zero should return true for %v", test) } for _, test := range nonZeros { assert.True(mockAssert.NotZero(test), "Zero should return false for %v", test) } } func TestJSONEqWrapper_EqualSONString(t *testing.T) { assert := New(new(testing.T)) if !assert.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"hello": "world", "foo": "bar"}`) { t.Error("JSONEq should return true") } } func TestJSONEqWrapper_EquivalentButNotEqual(t *testing.T) { assert := New(new(testing.T)) if !assert.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) { t.Error("JSONEq should return true") } } func TestJSONEqWrapper_HashOfArraysAndHashes(t *testing.T) { assert := New(new(testing.T)) if !assert.JSONEq("{\r\n\t\"numeric\": 1.5,\r\n\t\"array\": [{\"foo\": \"bar\"}, 1, \"string\", [\"nested\", \"array\", 5.5]],\r\n\t\"hash\": {\"nested\": \"hash\", \"nested_slice\": [\"this\", \"is\", \"nested\"]},\r\n\t\"string\": \"foo\"\r\n}", "{\r\n\t\"numeric\": 1.5,\r\n\t\"hash\": {\"nested\": \"hash\", \"nested_slice\": [\"this\", \"is\", \"nested\"]},\r\n\t\"string\": \"foo\",\r\n\t\"array\": [{\"foo\": \"bar\"}, 1, \"string\", [\"nested\", \"array\", 5.5]]\r\n}") { t.Error("JSONEq should return true") } } func TestJSONEqWrapper_Array(t *testing.T) { assert := New(new(testing.T)) if !assert.JSONEq(`["foo", {"hello": "world", "nested": "hash"}]`, `["foo", {"nested": "hash", "hello": "world"}]`) { t.Error("JSONEq should return true") } } func TestJSONEqWrapper_HashAndArrayNotEquivalent(t *testing.T) { assert := New(new(testing.T)) if assert.JSONEq(`["foo", {"hello": "world", "nested": "hash"}]`, `{"foo": "bar", {"nested": "hash", "hello": "world"}}`) { t.Error("JSONEq should return false") } } func TestJSONEqWrapper_HashesNotEquivalent(t *testing.T) { assert := New(new(testing.T)) if assert.JSONEq(`{"foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) { t.Error("JSONEq should return false") } } func TestJSONEqWrapper_ActualIsNotJSON(t *testing.T) { assert := New(new(testing.T)) if assert.JSONEq(`{"foo": "bar"}`, "Not JSON") { t.Error("JSONEq should return false") } } func TestJSONEqWrapper_ExpectedIsNotJSON(t *testing.T) { assert := New(new(testing.T)) if assert.JSONEq("Not JSON", `{"foo": "bar", "hello": "world"}`) { t.Error("JSONEq should return false") } } func TestJSONEqWrapper_ExpectedAndActualNotJSON(t *testing.T) { assert := New(new(testing.T)) if assert.JSONEq("Not JSON", "Not JSON") { t.Error("JSONEq should return false") } } func TestJSONEqWrapper_ArraysOfDifferentOrder(t *testing.T) { assert := New(new(testing.T)) if assert.JSONEq(`["foo", {"hello": "world", "nested": "hash"}]`, `[{ "hello": "world", "nested": "hash"}, "foo"]`) { t.Error("JSONEq should return false") } } ================================================ FILE: vendor/github.com/stretchr/testify/assert/http_assertions.go ================================================ package assert import ( "fmt" "net/http" "net/http/httptest" "net/url" "strings" ) // httpCode is a helper that returns HTTP code of the response. It returns -1 // if building a new request fails. func httpCode(handler http.HandlerFunc, method, url string, values url.Values) int { w := httptest.NewRecorder() req, err := http.NewRequest(method, url+"?"+values.Encode(), nil) if err != nil { return -1 } handler(w, req) return w.Code } // HTTPSuccess asserts that a specified handler returns a success status code. // // assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) // // Returns whether the assertion was successful (true) or not (false). func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool { code := httpCode(handler, method, url, values) if code == -1 { return false } return code >= http.StatusOK && code <= http.StatusPartialContent } // HTTPRedirect asserts that a specified handler returns a redirect status code. // // assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool { code := httpCode(handler, method, url, values) if code == -1 { return false } return code >= http.StatusMultipleChoices && code <= http.StatusTemporaryRedirect } // HTTPError asserts that a specified handler returns an error status code. // // assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool { code := httpCode(handler, method, url, values) if code == -1 { return false } return code >= http.StatusBadRequest } // HTTPBody is a helper that returns HTTP body of the response. It returns // empty string if building a new request fails. func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) string { w := httptest.NewRecorder() req, err := http.NewRequest(method, url+"?"+values.Encode(), nil) if err != nil { return "" } handler(w, req) return w.Body.String() } // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // // assert.HTTPBodyContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}) bool { body := HTTPBody(handler, method, url, values) contains := strings.Contains(body, fmt.Sprint(str)) if !contains { Fail(t, fmt.Sprintf("Expected response body for \"%s\" to contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body)) } return contains } // HTTPBodyNotContains asserts that a specified handler returns a // body that does not contain a string. // // assert.HTTPBodyNotContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}) bool { body := HTTPBody(handler, method, url, values) contains := strings.Contains(body, fmt.Sprint(str)) if contains { Fail(t, fmt.Sprintf("Expected response body for \"%s\" to NOT contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body)) } return !contains } ================================================ FILE: vendor/github.com/stretchr/testify/assert/http_assertions_test.go ================================================ package assert import ( "fmt" "net/http" "net/url" "testing" ) func httpOK(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } func httpRedirect(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTemporaryRedirect) } func httpError(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) } func TestHTTPStatuses(t *testing.T) { assert := New(t) mockT := new(testing.T) assert.Equal(HTTPSuccess(mockT, httpOK, "GET", "/", nil), true) assert.Equal(HTTPSuccess(mockT, httpRedirect, "GET", "/", nil), false) assert.Equal(HTTPSuccess(mockT, httpError, "GET", "/", nil), false) assert.Equal(HTTPRedirect(mockT, httpOK, "GET", "/", nil), false) assert.Equal(HTTPRedirect(mockT, httpRedirect, "GET", "/", nil), true) assert.Equal(HTTPRedirect(mockT, httpError, "GET", "/", nil), false) assert.Equal(HTTPError(mockT, httpOK, "GET", "/", nil), false) assert.Equal(HTTPError(mockT, httpRedirect, "GET", "/", nil), false) assert.Equal(HTTPError(mockT, httpError, "GET", "/", nil), true) } func TestHTTPStatusesWrapper(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) assert.Equal(mockAssert.HTTPSuccess(httpOK, "GET", "/", nil), true) assert.Equal(mockAssert.HTTPSuccess(httpRedirect, "GET", "/", nil), false) assert.Equal(mockAssert.HTTPSuccess(httpError, "GET", "/", nil), false) assert.Equal(mockAssert.HTTPRedirect(httpOK, "GET", "/", nil), false) assert.Equal(mockAssert.HTTPRedirect(httpRedirect, "GET", "/", nil), true) assert.Equal(mockAssert.HTTPRedirect(httpError, "GET", "/", nil), false) assert.Equal(mockAssert.HTTPError(httpOK, "GET", "/", nil), false) assert.Equal(mockAssert.HTTPError(httpRedirect, "GET", "/", nil), false) assert.Equal(mockAssert.HTTPError(httpError, "GET", "/", nil), true) } func httpHelloName(w http.ResponseWriter, r *http.Request) { name := r.FormValue("name") w.Write([]byte(fmt.Sprintf("Hello, %s!", name))) } func TestHttpBody(t *testing.T) { assert := New(t) mockT := new(testing.T) assert.True(HTTPBodyContains(mockT, httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "Hello, World!")) assert.True(HTTPBodyContains(mockT, httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "World")) assert.False(HTTPBodyContains(mockT, httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "world")) assert.False(HTTPBodyNotContains(mockT, httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "Hello, World!")) assert.False(HTTPBodyNotContains(mockT, httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "World")) assert.True(HTTPBodyNotContains(mockT, httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "world")) } func TestHttpBodyWrappers(t *testing.T) { assert := New(t) mockAssert := New(new(testing.T)) assert.True(mockAssert.HTTPBodyContains(httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "Hello, World!")) assert.True(mockAssert.HTTPBodyContains(httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "World")) assert.False(mockAssert.HTTPBodyContains(httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "world")) assert.False(mockAssert.HTTPBodyNotContains(httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "Hello, World!")) assert.False(mockAssert.HTTPBodyNotContains(httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "World")) assert.True(mockAssert.HTTPBodyNotContains(httpHelloName, "GET", "/", url.Values{"name": []string{"World"}}, "world")) } ================================================ FILE: vendor/github.com/stretchr/testify/doc.go ================================================ // Package testify is a set of packages that provide many tools for testifying that your code will behave as you intend. // // testify contains the following packages: // // The assert package provides a comprehensive set of assertion functions that tie in to the Go testing system. // // The http package contains tools to make it easier to test http activity using the Go testing system. // // The mock package provides a system by which it is possible to mock your objects and verify calls are happening as expected. // // The suite package provides a basic structure for using structs as testing suites, and methods on those structs as tests. It includes setup/teardown functionality in the way of interfaces. package testify // blank imports help docs. import ( // assert package _ "github.com/stretchr/testify/assert" // http package _ "github.com/stretchr/testify/http" // mock package _ "github.com/stretchr/testify/mock" ) ================================================ FILE: vendor/github.com/stretchr/testify/package_test.go ================================================ package testify import ( "github.com/stretchr/testify/assert" "testing" ) func TestImports(t *testing.T) { if assert.Equal(t, 1, 1) != true { t.Error("Something is wrong.") } } ================================================ FILE: window.go ================================================ package morgoth type Window struct { Data []float64 } func (self *Window) Copy() *Window { data := make([]float64, len(self.Data)) copy(data, self.Data) return &Window{ Data: data, } }